[Pkg-ofed-commits] [libfabric] 02/06: New upstream version 1.4.0~rc2
Ana Beatriz Guerrero López
ana at moszumanska.debian.org
Sat Oct 22 13:44:59 UTC 2016
This is an automated email from the git hooks/post-receive script.
ana pushed a commit to branch master
in repository libfabric.
commit 44ffeafb274fef0bc180b00af9f96f42f2ac8ae9
Author: Ana Guerrero López <ana at ekaia.org>
Date: Sat Oct 22 15:38:43 2016 +0200
New upstream version 1.4.0~rc2
---
.gitignore | 35 -
.mailmap | 10 -
.travis.yml | 52 -
AUTHORS | 11 +
CONTRIBUTORS | 23 -
COPYING | 2 +-
Makefile.am | 76 +-
Makefile.in | 10942 +++++++++
NEWS.md | 145 +
README | 9 +-
README.md | 252 -
aclocal.m4 | 1036 +
autogen.sh | 10 -
config.h.in | 207 +
config/compile | 143 +
config/config.guess | 1501 ++
config/config.sub | 1705 ++
config/cron-make-nightly-tarball.pl | 292 -
config/cron-run-all-md2nroff.pl | 211 -
config/cron-submit-coverity.pl | 147 -
config/depcomp | 630 +
config/github-webhook/README | 129 -
config/github-webhook/apache-config.txt | 9 -
.../github-webhook-ofiwg-init.d-rhel | 117 -
.../github-webhook-ofiwg-init.d-ubuntu | 85 -
config/github-webhook/verify-signed-off.rb | 127 -
config/install-sh | 520 +
config/libtool.m4 | 7360 ++++++
config/ltmain.sh | 8406 +++++++
config/ltoptions.m4 | 368 +
config/ltsugar.m4 | 123 +
config/ltversion.m4 | 23 +
config/lt~obsolete.m4 | 92 +
config/md2nroff.pl | 164 -
config/missing | 376 +
config/ofiwg-bot at flatbed-.bashrc | 12 -
config/ofiwg-bot at flatbed-crontab.txt | 41 -
configure | 24112 +++++++++++++++++++
configure.ac | 36 +-
docs/providers | 111 -
include/fi.h | 30 +-
include/fi_enosys.h | 4 +-
include/fi_list.h | 15 +-
include/fi_mem.h | 48 +-
include/fi_osd.h | 5 +
include/fi_proto.h | 24 +-
include/fi_rbuf.h | 20 +-
include/fi_signal.h | 17 +-
include/fi_util.h | 260 +-
include/freebsd/osd.h | 13 +
include/linux/osd.h | 21 +
include/osx/osd.h | 8 +
include/prov.h | 25 +-
include/rdma/fabric.h | 30 +-
include/rdma/fi_atomic.h | 6 +-
include/rdma/fi_cm.h | 6 +-
include/rdma/fi_domain.h | 23 +-
include/rdma/fi_endpoint.h | 19 +-
include/rdma/fi_eq.h | 13 +-
include/rdma/fi_errno.h | 7 +-
include/rdma/fi_rma.h | 6 +-
include/rdma/fi_tagged.h | 6 +-
include/rdma/fi_trigger.h | 8 +-
include/rdma/{ => providers}/fi_log.h | 14 +-
include/rdma/{ => providers}/fi_prov.h | 8 +-
prov/gni/test/pmi_utils.c => include/unix/osd.h | 66 +-
libfabric.spec.in => libfabric.spec | 7 +-
libfabric.spec.in | 5 +-
man/README | 168 -
man/fabric.7.md | 251 -
man/fi_atomic.3.md | 539 -
man/fi_av.3.md | 435 -
man/fi_cm.3.md | 219 -
man/fi_cntr.3.md | 250 -
man/fi_control.3.md | 47 -
man/fi_cq.3.md | 553 -
man/fi_direct.7.md | 75 -
man/fi_domain.3.md | 516 -
man/fi_endpoint.3.md | 1226 -
man/fi_eq.3.md | 433 -
man/fi_errno.3.md | 159 -
man/fi_fabric.3.md | 189 -
man/fi_getinfo.3.md | 576 -
man/fi_gni.7.md | 197 -
man/fi_mr.3.md | 320 -
man/fi_msg.3.md | 294 -
man/fi_mxm.7.md | 60 -
man/fi_poll.3.md | 303 -
man/fi_provider.7.md | 166 -
man/fi_psm.7.md | 169 -
man/fi_psm2.7.md | 151 -
man/fi_rma.3.md | 293 -
man/fi_sockets.7.md | 90 -
man/fi_tagged.3.md | 353 -
man/fi_trigger.3.md | 93 -
man/fi_udp.7.md | 60 -
man/fi_usnic.7.md | 288 -
man/fi_verbs.7.md | 86 -
man/fi_version.3.md | 43 -
man/man1/fi_info.1 | 109 +
man/man1/fi_pingpong.1 | 168 +
man/man1/fi_strerror.1 | 25 +
man/man3/fi_atomic.3 | 31 +-
man/man3/fi_av.3 | 26 +-
man/man3/fi_cm.3 | 3 +-
man/man3/fi_cntr.3 | 2 +-
man/man3/fi_control.3 | 2 +-
man/man3/fi_cq.3 | 21 +-
man/man3/fi_domain.3 | 20 +-
man/man3/fi_endpoint.3 | 142 +-
man/man3/fi_eq.3 | 16 +-
man/man3/fi_errno.3 | 2 +-
man/man3/fi_fabric.3 | 2 +-
man/man3/fi_getinfo.3 | 8 +-
man/man3/fi_mr.3 | 2 +-
man/man3/fi_msg.3 | 35 +-
man/man3/fi_poll.3 | 21 +-
man/man3/fi_rma.3 | 14 +-
man/man3/fi_tagged.3 | 14 +-
man/man3/fi_trigger.3 | 2 +-
man/man3/fi_version.3 | 2 +-
man/man7/fabric.7 | 21 +-
man/man7/fi_direct.7 | 2 +-
man/man7/fi_gni.7 | 76 +-
man/man7/fi_mxm.7 | 5 +-
man/man7/fi_provider.7 | 2 +-
man/man7/fi_psm.7 | 2 +-
man/man7/fi_psm2.7 | 2 +-
man/man7/fi_sockets.7 | 2 +-
man/man7/fi_udp.7 | 2 +-
man/man7/fi_usnic.7 | 59 +-
man/man7/fi_verbs.7 | 170 +-
prov/gni/Makefile.include | 20 +-
prov/gni/README.md | 21 -
prov/gni/configure.m4 | 60 +-
prov/gni/contrib/gnitest.supp | 463 -
prov/gni/gnix.map | 108 -
prov/gni/include/fi_ext_gni.h | 30 +-
prov/gni/include/gnix.h | 229 +-
prov/gni/include/gnix_atomic.h | 5 +-
prov/gni/include/gnix_av.h | 67 +-
prov/gni/include/gnix_buddy_allocator.h | 13 +-
prov/gni/include/gnix_cm_nic.h | 25 +-
prov/gni/include/gnix_cntr.h | 13 +-
prov/gni/include/gnix_cq.h | 12 +-
prov/gni/include/gnix_datagram.h | 24 +-
prov/gni/include/gnix_ep.h | 62 +-
prov/gni/include/gnix_eq.h | 15 +-
prov/gni/include/gnix_freelist.h | 62 +-
prov/gni/include/gnix_mbox_allocator.h | 10 +-
prov/gni/include/gnix_mr.h | 43 +
prov/gni/include/gnix_mr_cache.h | 11 +-
prov/gni/include/gnix_mr_notifier.h | 183 +
prov/gni/include/gnix_msg.h | 12 +-
prov/gni/include/gnix_nameserver.h | 10 +-
prov/gni/include/gnix_nic.h | 72 +-
prov/gni/include/gnix_poll.h | 10 +-
prov/gni/include/gnix_priv.h | 26 +-
prov/gni/include/gnix_queue.h | 9 +-
prov/gni/include/gnix_rma.h | 2 +
prov/gni/{test/common.c => include/gnix_trigger.h} | 30 +-
prov/gni/include/gnix_util.h | 38 +-
prov/gni/include/gnix_vc.h | 15 +-
prov/gni/include/gnix_vector.h | 172 +-
prov/gni/include/gnix_wait.h | 9 +-
prov/gni/include/gnix_xpmem.h | 207 +
prov/gni/include/rdma/fi_direct.h | 45 -
prov/gni/include/rdma/fi_direct_atomic.h | 267 -
prov/gni/include/rdma/fi_direct_atomic_def.h | 81 -
prov/gni/include/rdma/fi_direct_cm.h | 107 -
prov/gni/include/rdma/fi_direct_domain.h | 218 -
prov/gni/include/rdma/fi_direct_endpoint.h | 275 -
prov/gni/include/rdma/fi_direct_eq.h | 243 -
prov/gni/include/rdma/fi_direct_rma.h | 147 -
prov/gni/include/rdma/fi_direct_tagged.h | 145 -
prov/gni/include/rdma/fi_direct_trigger.h | 49 -
prov/gni/provider_FABRIC_1.0.map | 111 -
prov/gni/src/gnix_atomic.c | 257 +-
prov/gni/src/gnix_av.c | 417 +-
prov/gni/src/gnix_buddy_allocator.c | 9 +-
prov/gni/src/gnix_cm.c | 58 +-
prov/gni/src/gnix_cm_nic.c | 155 +-
prov/gni/src/gnix_cntr.c | 100 +-
prov/gni/src/gnix_cq.c | 48 +-
prov/gni/src/gnix_datagram.c | 86 +-
prov/gni/src/gnix_dom.c | 129 +-
prov/gni/src/gnix_ep.c | 783 +-
prov/gni/src/gnix_eq.c | 61 +-
prov/gni/src/gnix_fabric.c | 187 +-
prov/gni/src/gnix_freelist.c | 53 +-
prov/gni/src/gnix_hashtable.c | 45 +-
prov/gni/src/gnix_init.c | 37 +
prov/gni/src/gnix_mbox_allocator.c | 89 +-
prov/gni/src/gnix_mr.c | 448 +-
prov/gni/src/gnix_mr_cache.c | 762 +-
prov/gni/src/gnix_mr_notifier.c | 247 +
prov/gni/src/gnix_msg.c | 2320 +-
prov/gni/src/gnix_nameserver.c | 35 +-
prov/gni/src/gnix_nic.c | 144 +-
prov/gni/src/gnix_queue.c | 2 +-
prov/gni/src/gnix_rma.c | 213 +-
prov/gni/src/gnix_tags.c | 22 +-
prov/gni/src/gnix_trigger.c | 128 +
prov/gni/src/gnix_util.c | 313 +-
prov/gni/src/gnix_vc.c | 975 +-
prov/gni/src/gnix_vector.c | 291 +-
prov/gni/src/gnix_wait.c | 5 +-
prov/gni/src/gnix_xpmem.c | 611 +
prov/gni/test/allocator.c | 517 -
prov/gni/test/api.c | 817 -
prov/gni/test/api_cq.c | 460 -
prov/gni/test/av.c | 486 -
prov/gni/test/bitmap.c | 576 -
prov/gni/test/buddy_allocator.c | 269 -
prov/gni/test/cancel.c | 346 -
prov/gni/test/cntr.c | 797 -
prov/gni/test/common.h | 43 -
prov/gni/test/cq.c | 662 -
prov/gni/test/datagram.c | 453 -
prov/gni/test/dlist-utils.c | 222 -
prov/gni/test/dom.c | 171 -
prov/gni/test/ep.c | 278 -
prov/gni/test/eq.c | 246 -
prov/gni/test/freelist.c | 172 -
prov/gni/test/hashtable.c | 757 -
prov/gni/test/mr.c | 933 -
prov/gni/test/nic.c | 110 -
prov/gni/test/queue.c | 223 -
prov/gni/test/rdm_atomic.c | 4420 ----
prov/gni/test/rdm_dgram_rma.c | 1624 --
prov/gni/test/rdm_fi_pcd_trecv_msg.c | 1884 --
prov/gni/test/rdm_rx_overrun.c | 283 -
prov/gni/test/rdm_sr.c | 1587 --
prov/gni/test/rdm_tagged_sr.c | 810 -
prov/gni/test/run_gnitest | 63 -
prov/gni/test/tags.c | 1876 --
prov/gni/test/utils.c | 154 -
prov/gni/test/vc.c | 388 -
prov/gni/test/vector.c | 358 -
prov/gni/test/wait.c | 256 -
prov/mxm/AUTHORS | 2 -
prov/mxm/src/mlxm.h | 7 +-
prov/mxm/src/mlxm_av.c | 6 +-
prov/mxm/src/mlxm_domain.c | 1 +
prov/mxm/src/mlxm_ep.c | 8 +-
prov/psm/Makefile.include | 2 -
prov/psm/src/psmx.h | 131 +-
prov/psm/src/psmx_atomic.c | 44 +-
prov/psm/src/psmx_av.c | 84 +-
prov/psm/src/psmx_cntr.c | 71 +-
prov/psm/src/psmx_cq.c | 113 +-
prov/psm/src/psmx_domain.c | 200 +-
prov/psm/src/psmx_ep.c | 97 +-
prov/psm/src/psmx_eq.c | 444 -
prov/psm/src/psmx_fabric.c | 61 +-
prov/psm/src/psmx_init.c | 15 +-
prov/psm/src/psmx_mr.c | 27 +-
prov/psm/src/psmx_msg.c | 16 +-
prov/psm/src/psmx_msg2.c | 16 +-
prov/psm/src/psmx_poll.c | 176 -
prov/psm/src/psmx_rma.c | 28 +-
prov/psm/src/psmx_tagged.c | 26 +-
prov/psm/src/psmx_wait.c | 241 +-
prov/psm2/src/psmx2.h | 123 +-
prov/psm2/src/psmx2_atomic.c | 819 +-
prov/psm2/src/psmx2_av.c | 22 +-
prov/psm2/src/psmx2_cm.c | 1 +
prov/psm2/src/psmx2_cntr.c | 37 +-
prov/psm2/src/psmx2_cq.c | 89 +-
prov/psm2/src/psmx2_domain.c | 8 +-
prov/psm2/src/psmx2_ep.c | 91 +-
prov/psm2/src/psmx2_fabric.c | 15 +-
prov/psm2/src/psmx2_init.c | 16 +-
prov/psm2/src/psmx2_mr.c | 18 +-
prov/psm2/src/psmx2_msg.c | 20 +-
prov/psm2/src/psmx2_rma.c | 288 +-
prov/psm2/src/psmx2_tagged.c | 14 +-
prov/psm2/src/psmx2_wait.c | 35 +-
prov/rxd/Makefile.include | 28 +
prov/rxd/configure.m4 | 15 +
prov/rxd/src/rxd.h | 471 +
prov/{udp/src/udpx_attr.c => rxd/src/rxd_attr.c} | 77 +-
prov/rxd/src/rxd_av.c | 335 +
prov/rxd/src/rxd_cq.c | 1355 ++
prov/rxd/src/rxd_domain.c | 303 +
prov/rxd/src/rxd_ep.c | 1905 ++
.../src/udpx_fabric.c => rxd/src/rxd_fabric.c} | 86 +-
prov/{udp/src/udpx_init.c => rxd/src/rxd_init.c} | 46 +-
prov/rxd/src/rxd_rma.c | 365 +
prov/rxm/Makefile.include | 26 +
prov/rxm/configure.m4 | 16 +
prov/rxm/src/rxm.h | 124 +
prov/{udp/src/udpx_attr.c => rxm/src/rxm_attr.c} | 64 +-
prov/rxm/src/rxm_conn.c | 355 +
prov/rxm/src/rxm_cq.c | 148 +
prov/rxm/src/rxm_domain.c | 185 +
prov/rxm/src/rxm_ep.c | 446 +
prov/rxm/src/rxm_fabric.c | 164 +
.../rdma/fi_direct.h => rxm/src/rxm_init.c} | 61 +-
prov/sockets/Makefile.include | 5 +-
prov/sockets/include/rdma/fi_direct_atomic.h | 33 -
prov/sockets/include/rdma/fi_direct_atomic_def.h | 33 -
prov/sockets/include/rdma/fi_direct_domain.h | 33 -
prov/sockets/include/rdma/fi_direct_endpoint.h | 33 -
prov/sockets/include/rdma/fi_direct_eq.h | 33 -
prov/sockets/include/rdma/fi_direct_rma.h | 33 -
prov/sockets/include/rdma/fi_direct_tagged.h | 33 -
prov/sockets/include/rdma/fi_direct_trigger.h | 33 -
prov/sockets/include/sock.h | 174 +-
prov/sockets/include/sock_util.h | 11 +-
prov/sockets/provider_FABRIC_1.0.map | 1 -
prov/sockets/src/sock_atomic.c | 71 +-
prov/sockets/src/sock_av.c | 259 +-
prov/sockets/src/sock_cntr.c | 138 +-
prov/sockets/src/sock_comm.c | 23 +-
prov/sockets/src/sock_conn.c | 210 +-
prov/sockets/src/sock_cq.c | 76 +-
prov/sockets/src/sock_ctx.c | 52 +-
prov/sockets/src/sock_dom.c | 114 +-
prov/sockets/src/sock_ep.c | 1003 +-
prov/sockets/src/sock_ep_dgram.c | 10 +-
prov/sockets/src/sock_ep_msg.c | 1035 +-
prov/sockets/src/sock_ep_rdm.c | 10 +-
prov/sockets/src/sock_epoll.c | 25 +-
prov/sockets/src/sock_eq.c | 1 +
prov/sockets/src/sock_fabric.c | 324 +-
prov/sockets/src/sock_msg.c | 52 +-
prov/sockets/src/sock_poll.c | 6 +-
prov/sockets/src/sock_progress.c | 319 +-
prov/sockets/src/sock_rma.c | 34 +-
prov/sockets/src/sock_rx_entry.c | 5 -
prov/sockets/src/sock_trigger.c | 28 +-
prov/sockets/src/sock_wait.c | 20 +-
prov/udp/src/udpx.h | 4 +-
prov/udp/src/udpx_attr.c | 14 +-
prov/udp/src/udpx_cq.c | 38 +-
prov/udp/src/udpx_domain.c | 4 +-
prov/udp/src/udpx_ep.c | 45 +-
prov/udp/src/udpx_fabric.c | 12 +-
prov/udp/src/udpx_init.c | 71 +-
prov/usnic/src/fi_ext_usnic.h | 28 +-
prov/usnic/src/usdf.h | 22 +-
prov/usnic/src/usdf_av.c | 241 +-
prov/usnic/src/usdf_av.h | 6 +-
prov/usnic/src/usdf_cm.c | 208 +-
prov/usnic/src/usdf_cm.h | 8 +-
prov/usnic/src/usdf_cq.c | 4 +-
prov/usnic/src/usdf_cq.h | 2 +-
prov/usnic/src/usdf_dgram.c | 28 +-
prov/usnic/src/usdf_dgram.h | 3 +-
prov/usnic/src/usdf_domain.c | 93 +-
prov/usnic/src/usdf_endpoint.c | 53 +-
prov/usnic/src/usdf_endpoint.h | 9 +-
prov/usnic/src/usdf_ep_dgram.c | 20 +-
prov/usnic/src/usdf_ep_msg.c | 63 +-
prov/usnic/src/usdf_ep_rdm.c | 70 +-
prov/usnic/src/usdf_eq.c | 44 +-
prov/usnic/src/usdf_fabric.c | 127 +-
prov/usnic/src/usdf_mem.c | 3 +-
prov/usnic/src/usdf_msg.c | 23 +-
prov/usnic/src/usdf_msg.h | 5 +-
prov/usnic/src/usdf_pep.c | 166 +-
prov/usnic/src/usdf_progress.c | 1 -
prov/usnic/src/usdf_rdm.c | 22 +-
prov/usnic/src/usdf_rdm.h | 6 +-
prov/usnic/src/usdf_socket.c | 68 -
prov/usnic/src/usdf_socket.h | 43 -
prov/usnic/src/usnic_direct/vnic_dev.c | 6 +-
prov/util/src/util_attr.c | 282 +-
prov/util/src/util_av.c | 326 +-
prov/util/src/util_buf.c | 5 +-
prov/util/src/util_cq.c | 135 +-
prov/util/src/util_domain.c | 5 +-
.../rdma/fi_direct_cm.h => util/src/util_ep.c} | 35 +-
prov/util/src/util_eq.c | 12 +-
prov/util/src/util_fabric.c | 9 +-
prov/util/src/util_main.c | 58 +-
prov/util/src/util_mr.c | 256 +
prov/util/src/util_poll.c | 9 +-
prov/util/src/util_wait.c | 4 +-
prov/verbs/Makefile.include | 3 +
prov/verbs/configure.m4 | 36 +-
prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c | 141 +-
prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c | 263 +-
prov/verbs/src/ep_rdm/verbs_ep_rdm.c | 375 +-
prov/verbs/src/ep_rdm/verbs_queuing.h | 211 +-
prov/verbs/src/ep_rdm/verbs_rdm.h | 456 +-
prov/verbs/src/ep_rdm/verbs_rdm_cm.c | 525 +-
prov/verbs/src/ep_rdm/verbs_rdm_cntr.c | 149 +
prov/verbs/src/ep_rdm/verbs_rdm_msg.c | 312 +
prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c | 824 +-
prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c | 1474 +-
prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.h | 74 +-
prov/verbs/src/ep_rdm/verbs_utils.c | 322 +-
prov/verbs/src/ep_rdm/verbs_utils.h | 127 +-
prov/verbs/src/fi_verbs.c | 173 +-
prov/verbs/src/fi_verbs.h | 72 +-
prov/verbs/src/verbs_av.c | 93 +-
prov/verbs/src/verbs_cm.c | 10 +-
prov/verbs/src/verbs_cq.c | 129 +-
prov/verbs/src/verbs_domain.c | 59 +-
prov/verbs/src/verbs_eq.c | 2 +-
prov/verbs/src/verbs_info.c | 214 +-
prov/verbs/src/verbs_msg.c | 19 +
prov/verbs/src/verbs_msg_ep.c | 44 +-
prov/verbs/src/verbs_rma.c | 290 +-
prov/verbs/src/verbs_srq.c | 240 +
src/common.c | 97 +-
src/enosys.c | 5 +-
src/fabric.c | 59 +-
src/fasthash.c | 13 +-
src/fi_tostr.c | 7 +-
src/log.c | 5 +-
src/unix/osd.c | 202 +
src/var.c | 11 +-
util/info.c | 102 +-
util/pingpong.c | 2208 ++
util/strerror.c | 80 +
418 files changed, 87009 insertions(+), 44393 deletions(-)
diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 26640d0..0000000
--- a/.gitignore
+++ /dev/null
@@ -1,35 +0,0 @@
-.libs
-.deps
-.dirstamp
-*.o
-*.lo
-*.la
-stamp-h?
-Makefile
-Makefile.in
-aclocal.m4
-autom4te.cache
-config.h
-config.h.in
-config.h.in~
-config.log
-config.status
-configure
-libtool
-
-libfabric.spec
-libfabric.pc
-libfabric.map
-
-config/ar-lib
-config/compile
-config/config.guess
-config/config.sub
-config/depcomp
-config/install-sh
-config/libtool.m4
-config/ltmain.sh
-config/lt*.m4
-config/missing
-
-util/fi_info
diff --git a/.mailmap b/.mailmap
deleted file mode 100644
index 2ff1da6..0000000
--- a/.mailmap
+++ /dev/null
@@ -1,10 +0,0 @@
-Ben Turrubiates <bturrubi at cisco.com> <ben at turrubiat.es>
-Dave Goodell <dgoodell at cisco.com> <davidjgoodell at gmail.com>
-Howard Pritchard <howardp at lanl.gov> <hppritcha at gmail.com>
-Jeff Squyres <jsquyres at cisco.com> <jsquyres at users.noreply.github.com>
-Patrick McCormick <patrick.m.mccormick at intel.com>
-Reese Faucette <rfaucett at cisco.com> <rfaucett at users.noreply.github.com>
-Miao Luo <miao.luo at intel.com>
-Jianxin Xiong <jianxin.xiong at intel.com>
-Shantonu Hossain <shantonu.hossain at intel.com>
-Chen Zhao <soniczhao at gmail.com>
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 74cdbc3..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-sudo: false
-language: c
-compiler:
- - clang
- - gcc
-os:
- - linux
- - osx
-addons:
- apt:
- packages:
- - rpm
- - libnl-3-200
- - libnl-3-dev
- - libnl-route-3-200
- - libnl-route-3-dev
- ssh_known_hosts:
- - www.openfabrics.org
- - git.kernel.org
-
-env:
- global:
- - CPPFLAGS=-I$HOME/include
- - LDFLAGS=-L$HOME/lib
- - LD_LIBRARY_PATH=$HOME/lib
- - LIBFABRIC_CONFIGURE_ARGS="--prefix=$HOME --enable-sockets"
-
-# install dependencies for the verbs and usnic providers
-before_install:
- - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then git clone https://git.kernel.org/pub/scm/libs/infiniband/libibverbs.git ; fi
- - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then cd libibverbs && ./autogen.sh && ./configure --prefix=$HOME && make && make install && cd .. ; fi
- - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then git clone https://github.com/ofiwg/librdmacm.git ; fi
- - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then cd librdmacm && ./autogen.sh && ./configure --prefix=$HOME CC=gcc && make && make install && cd .. ; fi
-
-install:
- - ./autogen.sh
- - if [[ "$TRAVIS_OS_NAME" == "linux" && "$CC" == "gcc" ]]; then ./configure $LIBFABRIC_CONFIGURE_ARGS --enable-debug --enable-verbs && make -j2; fi
- - ./configure --prefix=$HOME --enable-direct=sockets --enable-udp=no --enable-psm=no --enable-mxm=no --enable-gni=no --enable-psm2=no --enable-verbs=no --enable-usnic=no && make -j2
- - ./configure $LIBFABRIC_CONFIGURE_ARGS
- - make -j2
- - make install
- - make test
- - make distcheck
- - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make rpm; fi
-script:
- - git clone https://github.com/ofiwg/fabtests.git
- - cd fabtests
- - ./autogen.sh
- - ./configure --prefix=$HOME --with-libfabric=$HOME
- - make -j2
- - make install
- - make test
diff --git a/AUTHORS b/AUTHORS
index cd92a6c..d74b127 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -32,3 +32,14 @@ Latchesar Ionkov <lionkov at lanl.gov>
James Dinan <james.dinan at intel.com>
Evgeny Leksikov <evgeny.leksikov at intel.com>
Dmitry Durnov <dmitry.durnov at intel.com>
+Thomas Smith <thomasm2 at cisco.com>
+Sergey Oblomov <sergey.oblomov at intel.com>
+Sylvain Didelot <didelot.sylvain at gmail.com>
+Wesley Bland <wesley.bland at intel.com>
+Kayla Seager <kayla.seager at intel.com>
+Pierre Roux <piroux at cisco.com>
+Joe Doyle <joseph.doyle at intel.com>
+Ana Guerrero López <ana at ekaia.org>
+Alexander Sannikov <alexander.sannikov at intel.com>
+Erik Paulson <erik.r.paulson at intel.com>
+
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
deleted file mode 100644
index d7a7fa8..0000000
--- a/CONTRIBUTORS
+++ /dev/null
@@ -1,23 +0,0 @@
-All patches submitted for inclusion must adhere to the OpenFabric
-bylaws for code submission. The OpenFabric bylaws are available at:
-https://www.openfabrics.org/images/membership/ofa_bylaws_march_2011.pdf
-
-In brief, these bylaws require:
-
-That all contributions be under a license that grants users of
-the contributions perpetual, irrevocable, worldwide, non-exclusive, royalty-
-free copyright license to reproduce, display, perform, prepare, and have
-prepared derivative works.
-
-All contributions are submitted under dual BSD and GPLv2 licenses.
-
-All contributions must be encumbrance free. Any intellectual property
-included as part of any contribution is granted royalty free in
-perpetuity.
-
-All contributions must contain a sign-off tag in each commit message
-indicating agreement to the above terms, otherwise the contribution
-cannot be accepted. An example sign-off tag looks like:
-"""
-Signed-off-by: Some Developer <developer at example.org>
-"""
diff --git a/COPYING b/COPYING
index 81f986d..9170682 100644
--- a/COPYING
+++ b/COPYING
@@ -3,7 +3,7 @@ licenses. You may choose to be licensed under the terms of the the
BSD license or the GNU General Public License (GPL) Version
2, both included below.
-Copyright (c) 2015 Intel Corporation. All rights reserved.
+Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
Copyright (c) 2015-2016 Cisco Systems, Inc. All rights reserved.
==================================================================
diff --git a/Makefile.am b/Makefile.am
index d8c8e36..82556f9 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -10,11 +10,17 @@ AM_CPPFLAGS = \
-DRDMADIR=\"@rdmadir@\" \
-DPROVDLDIR=\"$(pkglibdir)\"
-lib_LTLIBRARIES = src/libfabric.la
+noinst_LTLIBRARIES =
+lib_LTLIBRARIES =
+
+if EMBEDDED
+noinst_LTLIBRARIES += src/libfabric.la
+else
+lib_LTLIBRARIES += src/libfabric.la
+endif
pkglib_LTLIBRARIES = $(DL_PROVIDERS)
-noinst_LTLIBRARIES =
ACLOCAL_AMFLAGS = -I config
AM_CFLAGS = -g -Wall
@@ -40,18 +46,41 @@ common_srcs = \
prov/util/src/util_av.c \
prov/util/src/util_cq.c \
prov/util/src/util_domain.c \
+ prov/util/src/util_ep.c \
prov/util/src/util_eq.c \
prov/util/src/util_fabric.c \
prov/util/src/util_main.c \
prov/util/src/util_poll.c \
prov/util/src/util_wait.c \
- prov/util/src/util_buf.c
+ prov/util/src/util_buf.c \
+ prov/util/src/util_mr.c
+
+if MACOS
+common_srcs += src/osx/osd.c
+common_srcs += src/unix/osd.c
+common_srcs += include/osx/osd.h
+common_srcs += include/unix/osd.h
+endif
+
+if FREEBSD
+common_srcs += src/unix/osd.c
+common_srcs += include/freebsd/osd.h
+common_srcs += include/unix/osd.h
+endif
+
+if LINUX
+common_srcs += src/unix/osd.c
+common_srcs += include/linux/osd.h
+common_srcs += include/unix/osd.h
+endif
# ensure dl-built providers link back to libfabric
-linkback = $(top_builddir)/src/libfabric.la
+linkback = src/libfabric.la
bin_PROGRAMS = \
- util/fi_info
+ util/fi_info \
+ util/fi_strerror \
+ util/fi_pingpong
bin_SCRIPTS =
@@ -59,6 +88,14 @@ util_fi_info_SOURCES = \
util/info.c
util_fi_info_LDADD = $(linkback)
+util_fi_strerror_SOURCES = \
+ util/strerror.c
+util_fi_strerror_LDADD = $(linkback)
+
+util_fi_pingpong_SOURCES = \
+ util/pingpong.c
+util_fi_pingpong_LDADD = $(linkback)
+
src_libfabric_la_SOURCES = \
include/fi.h \
include/fi_abi.h \
@@ -77,31 +114,20 @@ src_libfabric_la_SOURCES = \
include/fasthash.h \
include/rbtree.h \
include/prov.h \
+ include/rdma/providers/fi_log.h \
+ include/rdma/providers/fi_prov.h \
src/fabric.c \
src/fi_tostr.c \
src/log.c \
src/var.c \
$(common_srcs)
-if MACOS
-src_libfabric_la_SOURCES += src/osx/osd.c
-src_libfabric_la_SOURCES += include/osx/osd.h
-endif
-
-if FREEBSD
-src_libfabric_la_SOURCES += include/freebsd/osd.h
-endif
-
-if LINUX
-src_libfabric_la_SOURCES += include/linux/osd.h
-endif
-
src_libfabric_la_CPPFLAGS = $(AM_CPPFLAGS)
src_libfabric_la_LDFLAGS =
src_libfabric_la_LIBADD =
src_libfabric_la_DEPENDENCIES = libfabric.map
-src_libfabric_la_LDFLAGS += -version-info 3:0:2 -export-dynamic \
+src_libfabric_la_LDFLAGS += -version-info 3:1:2 -export-dynamic \
$(libfabric_version_script)
rdmainclude_HEADERS += \
$(top_srcdir)/include/rdma/fabric.h \
@@ -109,8 +135,6 @@ rdmainclude_HEADERS += \
$(top_srcdir)/include/rdma/fi_cm.h \
$(top_srcdir)/include/rdma/fi_domain.h \
$(top_srcdir)/include/rdma/fi_eq.h \
- $(top_srcdir)/include/rdma/fi_log.h \
- $(top_srcdir)/include/rdma/fi_prov.h \
$(top_srcdir)/include/rdma/fi_rma.h \
$(top_srcdir)/include/rdma/fi_endpoint.h \
$(top_srcdir)/include/rdma/fi_errno.h \
@@ -132,6 +156,9 @@ nodist_rdmainclude_HEADERS = \
endif HAVE_DIRECT
real_man_pages = \
+ man/man1/fi_info.1 \
+ man/man1/fi_pingpong.1 \
+ man/man1/fi_strerror.1 \
man/man3/fi_av.3 \
man/man3/fi_cm.3 \
man/man3/fi_cntr.3 \
@@ -270,7 +297,7 @@ pkgconfigdir = $(libdir)/pkgconfig
pkgconfig_DATA = libfabric.pc
nroff:
- @for file in $(real_man_pages); do \
+ @for file in $(real_man_pages) $(prov_install_man_pages); do \
source=`echo $$file | sed -e 's@/man[0-9]@@'`; \
config/md2nroff.pl --source=$$source.md; \
done
@@ -279,7 +306,8 @@ dist-hook: libfabric.spec
cp libfabric.spec $(distdir)
"$(top_srcdir)/config/distscript.pl" "$(distdir)" "$(PACKAGE_VERSION)"
-TESTS = util/fi_info
+TESTS = \
+ util/fi_info
test:
./util/fi_info
@@ -298,6 +326,8 @@ include prov/psm/Makefile.include
include prov/psm2/Makefile.include
include prov/mxm/Makefile.include
include prov/gni/Makefile.include
+include prov/rxm/Makefile.include
+include prov/rxd/Makefile.include
man_MANS = $(real_man_pages) $(prov_install_man_pages) $(dummy_man_pages)
diff --git a/Makefile.in b/Makefile.in
new file mode 100644
index 0000000..51e0c11
--- /dev/null
+++ b/Makefile.in
@@ -0,0 +1,10942 @@
+# Makefile.in generated by automake 1.11.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
+# 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation,
+# Inc.
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+ at SET_MAKE@
+
+#
+# Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
+#
+# Makefile.am for libfabric
+
+# Makefile.include for sockets provider
+
+#
+# Copyright (c) 2014-2016, Cisco Systems, Inc. All rights reserved.
+#
+# This software is available to you under a choice of one of two
+# licenses. You may choose to be licensed under the terms of the GNU
+# General Public License (GPL) Version 2, available from the file
+# COPYING in the main directory of this source tree, or the
+# BSD license below:
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+# - Redistributions of source code must retain the above
+# copyright notice, this list of conditions and the following
+# disclaimer.
+#
+# - Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials
+# provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+
+# Makefile.am for gni provider
+
+
+
+
+
+VPATH = @srcdir@
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+ at EMBEDDED_TRUE@am__append_1 = src/libfabric.la
+ at EMBEDDED_FALSE@am__append_2 = src/libfabric.la
+ at MACOS_TRUE@am__append_3 = src/osx/osd.c src/unix/osd.c \
+ at MACOS_TRUE@ include/osx/osd.h include/unix/osd.h
+ at FREEBSD_TRUE@am__append_4 = src/unix/osd.c include/freebsd/osd.h \
+ at FREEBSD_TRUE@ include/unix/osd.h
+ at LINUX_TRUE@am__append_5 = src/unix/osd.c include/linux/osd.h \
+ at LINUX_TRUE@ include/unix/osd.h
+bin_PROGRAMS = util/fi_info$(EXEEXT) util/fi_strerror$(EXEEXT) \
+ util/fi_pingpong$(EXEEXT) $(am__EXEEXT_1)
+TESTS = util/fi_info$(EXEEXT)
+DIST_COMMON = README $(am__configure_deps) \
+ $(am__rdmainclude_HEADERS_DIST) $(srcdir)/Makefile.am \
+ $(srcdir)/Makefile.in $(srcdir)/config.h.in \
+ $(srcdir)/libfabric.map.in $(srcdir)/libfabric.pc.in \
+ $(srcdir)/libfabric.spec.in \
+ $(srcdir)/prov/gni/Makefile.include \
+ $(srcdir)/prov/mxm/Makefile.include \
+ $(srcdir)/prov/psm/Makefile.include \
+ $(srcdir)/prov/psm2/Makefile.include \
+ $(srcdir)/prov/rxd/Makefile.include \
+ $(srcdir)/prov/rxm/Makefile.include \
+ $(srcdir)/prov/sockets/Makefile.include \
+ $(srcdir)/prov/udp/Makefile.include \
+ $(srcdir)/prov/usnic/Makefile.include \
+ $(srcdir)/prov/verbs/Makefile.include $(top_srcdir)/configure \
+ AUTHORS COPYING config/compile config/config.guess \
+ config/config.sub config/depcomp config/install-sh \
+ config/ltmain.sh config/missing
+ at HAVE_SOCKETS_TRUE@am__append_6 = -I$(top_srcdir)/prov/sockets/include -I$(top_srcdir)/prov/sockets
+ at HAVE_SOCKETS_DL_TRUE@@HAVE_SOCKETS_TRUE at am__append_7 = libsockets-fi.la
+ at HAVE_SOCKETS_DL_FALSE@@HAVE_SOCKETS_TRUE at am__append_8 = $(_sockets_files) $(_sockets_headers)
+ at HAVE_SOCKETS_DL_FALSE@@HAVE_SOCKETS_TRUE at am__append_9 = $(sockets_LIBS)
+ at HAVE_SOCKETS_TRUE@am__append_10 = man/man7/fi_sockets.7
+ at HAVE_UDP_DL_TRUE@@HAVE_UDP_TRUE at am__append_11 = libudp-fi.la
+ at HAVE_UDP_DL_FALSE@@HAVE_UDP_TRUE at am__append_12 = $(_udp_files)
+ at HAVE_UDP_DL_FALSE@@HAVE_UDP_TRUE at am__append_13 = $(udp_shm_LIBS)
+ at HAVE_UDP_TRUE@am__append_14 = man/man7/fi_udp.7
+ at HAVE_VERBS_DL_TRUE@@HAVE_VERBS_TRUE at am__append_15 = libverbs-fi.la
+ at HAVE_VERBS_DL_FALSE@@HAVE_VERBS_TRUE at am__append_16 = $(_verbs_files)
+ at HAVE_VERBS_DL_FALSE@@HAVE_VERBS_TRUE at am__append_17 = $(verbs_CPPFLAGS)
+ at HAVE_VERBS_DL_FALSE@@HAVE_VERBS_TRUE at am__append_18 = $(verbs_LDFLAGS)
+ at HAVE_VERBS_DL_FALSE@@HAVE_VERBS_TRUE at am__append_19 = $(verbs_LIBS)
+ at HAVE_VERBS_TRUE@am__append_20 = man/man7/fi_verbs.7
+ at HAVE_USNIC_TRUE@@HAVE_VERBS_TRUE at am__append_21 = prov/usnic/src/usdf_fake_ibv.c
+ at HAVE_USNIC_TRUE@am__append_22 = \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/fi_ext_usnic.h
+
+ at HAVE_USNIC_DL_TRUE@@HAVE_USNIC_TRUE at am__append_23 = libusnic-fi.la
+ at HAVE_USNIC_DL_FALSE@@HAVE_USNIC_TRUE at am__append_24 = $(_usnic_files)
+ at HAVE_USNIC_DL_FALSE@@HAVE_USNIC_TRUE at am__append_25 = $(_usnic_cppflags)
+ at HAVE_USNIC_DL_FALSE@@HAVE_USNIC_TRUE at am__append_26 = $(usnic_LDFLAGS)
+ at HAVE_USNIC_DL_FALSE@@HAVE_USNIC_TRUE at am__append_27 = $(usnic_LIBS)
+ at HAVE_USNIC_TRUE@am__append_28 = man/man7/fi_usnic.7
+ at HAVE_PSM_DL_TRUE@@HAVE_PSM_TRUE at am__append_29 = libpsmx-fi.la
+ at HAVE_PSM_DL_FALSE@@HAVE_PSM_TRUE at am__append_30 = libpsmx.la
+ at HAVE_PSM_DL_FALSE@@HAVE_PSM_TRUE at am__append_31 = libpsmx.la
+ at HAVE_PSM_DL_FALSE@@HAVE_PSM_TRUE at am__append_32 = libpsmx.la
+ at HAVE_PSM_TRUE@am__append_33 = man/man7/fi_psm.7
+ at HAVE_PSM2_DL_TRUE@@HAVE_PSM2_TRUE at am__append_34 = libpsmx2-fi.la
+ at HAVE_PSM2_DL_FALSE@@HAVE_PSM2_TRUE at am__append_35 = libpsmx2.la
+ at HAVE_PSM2_DL_FALSE@@HAVE_PSM2_TRUE at am__append_36 = libpsmx2.la
+ at HAVE_PSM2_DL_FALSE@@HAVE_PSM2_TRUE at am__append_37 = libpsmx2.la
+ at HAVE_PSM2_TRUE@am__append_38 = man/man7/fi_psm2.7
+ at HAVE_MXM_DL_TRUE@@HAVE_MXM_TRUE at am__append_39 = libmlxm-fi.la
+ at HAVE_MXM_DL_FALSE@@HAVE_MXM_TRUE at am__append_40 = $(_mxm_files)
+ at HAVE_MXM_DL_FALSE@@HAVE_MXM_TRUE at am__append_41 = $(mxm_CPPFLAGS)
+ at HAVE_MXM_DL_FALSE@@HAVE_MXM_TRUE at am__append_42 = $(mxm_LDFLAGS)
+ at HAVE_MXM_DL_FALSE@@HAVE_MXM_TRUE at am__append_43 = $(mxm_LIBS)
+ at HAVE_MXM_TRUE@am__append_44 = man/man7/fi_mxm.7
+
+#
+# want to keep ccan in the include path seen in the
+# source code, so just add prov/gni to the include path
+# rather than prov/gni/ccan
+#
+ at HAVE_GNI_TRUE@am__append_45 = -I$(top_srcdir)/prov/gni/include -I$(top_srcdir)/prov/gni
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE at am__append_46 = prov/gni/test/gnitest
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE at am__append_47 = prov/gni/test/run_gnitest
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE at am__append_48 = libgnix-fi.la
+ at HAVE_GNI_DL_FALSE@@HAVE_GNI_TRUE at am__append_49 = $(_gni_files) $(_gni_headers)
+ at HAVE_GNI_DL_FALSE@@HAVE_GNI_TRUE at am__append_50 = $(gni_CPPFLAGS)
+ at HAVE_GNI_DL_FALSE@@HAVE_GNI_TRUE at am__append_51 = $(gni_LDFLAGS)
+ at HAVE_GNI_DL_FALSE@@HAVE_GNI_TRUE at am__append_52 = $(gni_LIBS)
+ at HAVE_GNI_TRUE@am__append_53 = \
+ at HAVE_GNI_TRUE@ prov/gni/include/fi_ext_gni.h
+
+ at HAVE_GNI_TRUE@am__append_54 = man/man7/fi_gni.7
+ at HAVE_RXM_DL_TRUE@@HAVE_RXM_TRUE at am__append_55 = librxm-fi.la
+ at HAVE_RXM_DL_FALSE@@HAVE_RXM_TRUE at am__append_56 = $(_rxm_files)
+ at HAVE_RXM_DL_FALSE@@HAVE_RXM_TRUE at am__append_57 = $(rxm_shm_LIBS)
+ at HAVE_RXD_DL_TRUE@@HAVE_RXD_TRUE at am__append_58 = librxd-fi.la
+ at HAVE_RXD_DL_FALSE@@HAVE_RXD_TRUE at am__append_59 = $(_rxd_files)
+ at HAVE_RXD_DL_FALSE@@HAVE_RXD_TRUE at am__append_60 = $(rxd_shm_LIBS)
+subdir = .
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/config/fi_provider.m4 \
+ $(top_srcdir)/config/libtool.m4 \
+ $(top_srcdir)/config/ltoptions.m4 \
+ $(top_srcdir)/config/ltsugar.m4 \
+ $(top_srcdir)/config/ltversion.m4 \
+ $(top_srcdir)/config/lt~obsolete.m4 \
+ $(top_srcdir)/config/fi_check_package.m4 \
+ $(top_srcdir)/prov/psm/configure.m4 \
+ $(top_srcdir)/prov/psm2/configure.m4 \
+ $(top_srcdir)/prov/sockets/configure.m4 \
+ $(top_srcdir)/prov/verbs/configure.m4 \
+ $(top_srcdir)/prov/usnic/configure.m4 \
+ $(top_srcdir)/prov/mxm/configure.m4 \
+ $(top_srcdir)/prov/gni/configure.m4 \
+ $(top_srcdir)/config/fi_pkg.m4 \
+ $(top_srcdir)/prov/udp/configure.m4 \
+ $(top_srcdir)/prov/rxm/configure.m4 \
+ $(top_srcdir)/prov/rxd/configure.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+ $(ACLOCAL_M4)
+am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
+ configure.lineno config.status.lineno
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = config.h
+CONFIG_CLEAN_FILES = libfabric.pc libfabric.spec libfabric.map
+CONFIG_CLEAN_VPATH_FILES =
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+ $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+ *) f=$$p;; \
+ esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+ srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+ for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+ for p in $$list; do echo "$$p $$p"; done | \
+ sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+ $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+ if (++n[$$2] == $(am__install_max)) \
+ { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+ END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+ sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+ sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__installdirs = "$(DESTDIR)$(libdir)" "$(DESTDIR)$(pkglibdir)" \
+ "$(DESTDIR)$(bindir)" "$(DESTDIR)$(bindir)" \
+ "$(DESTDIR)$(man1dir)" "$(DESTDIR)$(man3dir)" \
+ "$(DESTDIR)$(man7dir)" "$(DESTDIR)$(pkgconfigdir)" \
+ "$(DESTDIR)$(rdmaincludedir)" "$(DESTDIR)$(rdmaincludedir)"
+LTLIBRARIES = $(lib_LTLIBRARIES) $(noinst_LTLIBRARIES) \
+ $(pkglib_LTLIBRARIES)
+am__libgnix_fi_la_SOURCES_DIST = prov/gni/src/gnix_atomic.c \
+ prov/gni/src/gnix_av.c prov/gni/src/gnix_bitmap.c \
+ prov/gni/src/gnix_buddy_allocator.c prov/gni/src/gnix_cm.c \
+ prov/gni/src/gnix_cm_nic.c prov/gni/src/gnix_cntr.c \
+ prov/gni/src/gnix_cq.c prov/gni/src/gnix_datagram.c \
+ prov/gni/src/gnix_dom.c prov/gni/src/gnix_ep.c \
+ prov/gni/src/gnix_eq.c prov/gni/src/gnix_fabric.c \
+ prov/gni/src/gnix_freelist.c prov/gni/src/gnix_hashtable.c \
+ prov/gni/src/gnix_init.c prov/gni/src/gnix_mbox_allocator.c \
+ prov/gni/src/gnix_mr.c prov/gni/src/gnix_mr_cache.c \
+ prov/gni/src/gnix_mr_notifier.c prov/gni/src/gnix_msg.c \
+ prov/gni/src/gnix_nameserver.c prov/gni/src/gnix_nic.c \
+ prov/gni/src/gnix_poll.c prov/gni/src/gnix_queue.c \
+ prov/gni/src/gnix_rma.c prov/gni/src/gnix_tags.c \
+ prov/gni/src/gnix_trigger.c prov/gni/src/gnix_util.c \
+ prov/gni/src/gnix_vc.c prov/gni/src/gnix_vector.c \
+ prov/gni/src/gnix_xpmem.c prov/gni/src/gnix_wait.c \
+ prov/gni/include/fi_ext_gni.h prov/gni/include/gnix_atomic.h \
+ prov/gni/include/gnix_av.h prov/gni/include/gnix_bitmap.h \
+ prov/gni/include/gnix_buddy_allocator.h \
+ prov/gni/include/gnix_cm_nic.h prov/gni/include/gnix_cntr.h \
+ prov/gni/include/gnix_cq.h prov/gni/include/gnix_datagram.h \
+ prov/gni/include/gnix_ep.h prov/gni/include/gnix_eq.h \
+ prov/gni/include/gnix_freelist.h prov/gni/include/gnix.h \
+ prov/gni/include/gnix_hashtable.h \
+ prov/gni/include/gnix_mbox_allocator.h \
+ prov/gni/include/gnix_mr.h prov/gni/include/gnix_mr_cache.h \
+ prov/gni/include/gnix_mr_notifier.h \
+ prov/gni/include/gnix_msg.h prov/gni/include/gnix_nameserver.h \
+ prov/gni/include/gnix_nic.h prov/gni/include/gnix_poll.h \
+ prov/gni/include/gnix_priv.h prov/gni/include/gnix_queue.h \
+ prov/gni/include/gnix_rma.h prov/gni/include/gnix_tags.h \
+ prov/gni/include/gnix_trigger.h prov/gni/include/gnix_util.h \
+ prov/gni/include/gnix_vc.h prov/gni/include/gnix_vector.h \
+ prov/gni/include/gnix_xpmem.h prov/gni/include/gnix_wait.h \
+ src/common.c src/enosys.c src/rbtree.c src/fasthash.c \
+ src/indexer.c prov/util/src/util_attr.c \
+ prov/util/src/util_av.c prov/util/src/util_cq.c \
+ prov/util/src/util_domain.c prov/util/src/util_ep.c \
+ prov/util/src/util_eq.c prov/util/src/util_fabric.c \
+ prov/util/src/util_main.c prov/util/src/util_poll.c \
+ prov/util/src/util_wait.c prov/util/src/util_buf.c \
+ prov/util/src/util_mr.c src/osx/osd.c src/unix/osd.c \
+ include/osx/osd.h include/unix/osd.h include/freebsd/osd.h \
+ include/linux/osd.h
+am__dirstamp = $(am__leading_dot)dirstamp
+ at HAVE_GNI_TRUE@am__objects_1 = \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_atomic.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_av.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_bitmap.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_buddy_allocator.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_cm.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_cm_nic.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_cntr.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_cq.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_datagram.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_dom.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_ep.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_eq.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_fabric.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_freelist.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_hashtable.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_init.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_mbox_allocator.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_mr.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_mr_cache.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_mr_notifier.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_msg.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_nameserver.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_nic.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_poll.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_queue.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_rma.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_tags.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_trigger.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_util.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_vc.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_vector.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_xpmem.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/libgnix_fi_la-gnix_wait.lo
+am__objects_2 =
+ at MACOS_TRUE@am__objects_3 = src/osx/libgnix_fi_la-osd.lo \
+ at MACOS_TRUE@ src/unix/libgnix_fi_la-osd.lo
+ at FREEBSD_TRUE@am__objects_4 = src/unix/libgnix_fi_la-osd.lo
+ at LINUX_TRUE@am__objects_5 = src/unix/libgnix_fi_la-osd.lo
+am__objects_6 = src/libgnix_fi_la-common.lo \
+ src/libgnix_fi_la-enosys.lo src/libgnix_fi_la-rbtree.lo \
+ src/libgnix_fi_la-fasthash.lo src/libgnix_fi_la-indexer.lo \
+ prov/util/src/libgnix_fi_la-util_attr.lo \
+ prov/util/src/libgnix_fi_la-util_av.lo \
+ prov/util/src/libgnix_fi_la-util_cq.lo \
+ prov/util/src/libgnix_fi_la-util_domain.lo \
+ prov/util/src/libgnix_fi_la-util_ep.lo \
+ prov/util/src/libgnix_fi_la-util_eq.lo \
+ prov/util/src/libgnix_fi_la-util_fabric.lo \
+ prov/util/src/libgnix_fi_la-util_main.lo \
+ prov/util/src/libgnix_fi_la-util_poll.lo \
+ prov/util/src/libgnix_fi_la-util_wait.lo \
+ prov/util/src/libgnix_fi_la-util_buf.lo \
+ prov/util/src/libgnix_fi_la-util_mr.lo $(am__objects_3) \
+ $(am__objects_4) $(am__objects_5)
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE at am_libgnix_fi_la_OBJECTS = \
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE@ $(am__objects_1) \
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE@ $(am__objects_2) \
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE@ $(am__objects_6)
+libgnix_fi_la_OBJECTS = $(am_libgnix_fi_la_OBJECTS)
+AM_V_lt = $(am__v_lt_$(V))
+am__v_lt_ = $(am__v_lt_$(AM_DEFAULT_VERBOSITY))
+am__v_lt_0 = --silent
+libgnix_fi_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(libgnix_fi_la_LDFLAGS) $(LDFLAGS) -o $@
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE at am_libgnix_fi_la_rpath = -rpath \
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE@ $(pkglibdir)
+am__DEPENDENCIES_1 =
+am__libmlxm_fi_la_SOURCES_DIST = prov/mxm/src/mlxm.h \
+ prov/mxm/src/uthash.h prov/mxm/src/mpool.h \
+ prov/mxm/src/mlxm_mq_storage.h prov/mxm/src/mlxm_helpers.h \
+ prov/mxm/src/mlxm_init.c prov/mxm/src/mlxm_domain.c \
+ prov/mxm/src/mlxm_cq.c prov/mxm/src/mlxm_av.c \
+ prov/mxm/src/mlxm_ep.c prov/mxm/src/mlxm_cm.c \
+ prov/mxm/src/mlxm_tagged.c prov/mxm/src/mlxm_mr.c src/common.c \
+ src/enosys.c src/rbtree.c src/fasthash.c src/indexer.c \
+ prov/util/src/util_attr.c prov/util/src/util_av.c \
+ prov/util/src/util_cq.c prov/util/src/util_domain.c \
+ prov/util/src/util_ep.c prov/util/src/util_eq.c \
+ prov/util/src/util_fabric.c prov/util/src/util_main.c \
+ prov/util/src/util_poll.c prov/util/src/util_wait.c \
+ prov/util/src/util_buf.c prov/util/src/util_mr.c src/osx/osd.c \
+ src/unix/osd.c include/osx/osd.h include/unix/osd.h \
+ include/freebsd/osd.h include/linux/osd.h
+ at HAVE_MXM_TRUE@am__objects_7 = \
+ at HAVE_MXM_TRUE@ prov/mxm/src/libmlxm_fi_la-mlxm_init.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/libmlxm_fi_la-mlxm_domain.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/libmlxm_fi_la-mlxm_cq.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/libmlxm_fi_la-mlxm_av.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/libmlxm_fi_la-mlxm_ep.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/libmlxm_fi_la-mlxm_cm.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/libmlxm_fi_la-mlxm_tagged.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/libmlxm_fi_la-mlxm_mr.lo
+ at MACOS_TRUE@am__objects_8 = src/osx/libmlxm_fi_la-osd.lo \
+ at MACOS_TRUE@ src/unix/libmlxm_fi_la-osd.lo
+ at FREEBSD_TRUE@am__objects_9 = src/unix/libmlxm_fi_la-osd.lo
+ at LINUX_TRUE@am__objects_10 = src/unix/libmlxm_fi_la-osd.lo
+am__objects_11 = src/libmlxm_fi_la-common.lo \
+ src/libmlxm_fi_la-enosys.lo src/libmlxm_fi_la-rbtree.lo \
+ src/libmlxm_fi_la-fasthash.lo src/libmlxm_fi_la-indexer.lo \
+ prov/util/src/libmlxm_fi_la-util_attr.lo \
+ prov/util/src/libmlxm_fi_la-util_av.lo \
+ prov/util/src/libmlxm_fi_la-util_cq.lo \
+ prov/util/src/libmlxm_fi_la-util_domain.lo \
+ prov/util/src/libmlxm_fi_la-util_ep.lo \
+ prov/util/src/libmlxm_fi_la-util_eq.lo \
+ prov/util/src/libmlxm_fi_la-util_fabric.lo \
+ prov/util/src/libmlxm_fi_la-util_main.lo \
+ prov/util/src/libmlxm_fi_la-util_poll.lo \
+ prov/util/src/libmlxm_fi_la-util_wait.lo \
+ prov/util/src/libmlxm_fi_la-util_buf.lo \
+ prov/util/src/libmlxm_fi_la-util_mr.lo $(am__objects_8) \
+ $(am__objects_9) $(am__objects_10)
+ at HAVE_MXM_DL_TRUE@@HAVE_MXM_TRUE at am_libmlxm_fi_la_OBJECTS = \
+ at HAVE_MXM_DL_TRUE@@HAVE_MXM_TRUE@ $(am__objects_7) \
+ at HAVE_MXM_DL_TRUE@@HAVE_MXM_TRUE@ $(am__objects_11)
+libmlxm_fi_la_OBJECTS = $(am_libmlxm_fi_la_OBJECTS)
+libmlxm_fi_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(libmlxm_fi_la_LDFLAGS) $(LDFLAGS) -o $@
+ at HAVE_MXM_DL_TRUE@@HAVE_MXM_TRUE at am_libmlxm_fi_la_rpath = -rpath \
+ at HAVE_MXM_DL_TRUE@@HAVE_MXM_TRUE@ $(pkglibdir)
+am__libpsmx_fi_la_SOURCES_DIST = prov/psm/src/version.h \
+ prov/psm/src/psm_am.h prov/psm/src/psmx.h \
+ prov/psm/src/psmx_init.c prov/psm/src/psmx_fabric.c \
+ prov/psm/src/psmx_domain.c prov/psm/src/psmx_cq.c \
+ prov/psm/src/psmx_cntr.c prov/psm/src/psmx_av.c \
+ prov/psm/src/psmx_ep.c prov/psm/src/psmx_cm.c \
+ prov/psm/src/psmx_tagged.c prov/psm/src/psmx_msg.c \
+ prov/psm/src/psmx_msg2.c prov/psm/src/psmx_rma.c \
+ prov/psm/src/psmx_atomic.c prov/psm/src/psmx_am.c \
+ prov/psm/src/psmx_mr.c prov/psm/src/psmx_wait.c \
+ prov/psm/src/psmx_util.c src/common.c src/enosys.c \
+ src/rbtree.c src/fasthash.c src/indexer.c \
+ prov/util/src/util_attr.c prov/util/src/util_av.c \
+ prov/util/src/util_cq.c prov/util/src/util_domain.c \
+ prov/util/src/util_ep.c prov/util/src/util_eq.c \
+ prov/util/src/util_fabric.c prov/util/src/util_main.c \
+ prov/util/src/util_poll.c prov/util/src/util_wait.c \
+ prov/util/src/util_buf.c prov/util/src/util_mr.c src/osx/osd.c \
+ src/unix/osd.c include/osx/osd.h include/unix/osd.h \
+ include/freebsd/osd.h include/linux/osd.h
+ at HAVE_PSM_TRUE@am__objects_12 = \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_init.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_fabric.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_domain.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_cq.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_cntr.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_av.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_ep.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_cm.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_tagged.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_msg.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_msg2.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_rma.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_atomic.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_am.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_mr.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_wait.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_fi_la-psmx_util.lo
+ at MACOS_TRUE@am__objects_13 = src/osx/libpsmx_fi_la-osd.lo \
+ at MACOS_TRUE@ src/unix/libpsmx_fi_la-osd.lo
+ at FREEBSD_TRUE@am__objects_14 = src/unix/libpsmx_fi_la-osd.lo
+ at LINUX_TRUE@am__objects_15 = src/unix/libpsmx_fi_la-osd.lo
+am__objects_16 = src/libpsmx_fi_la-common.lo \
+ src/libpsmx_fi_la-enosys.lo src/libpsmx_fi_la-rbtree.lo \
+ src/libpsmx_fi_la-fasthash.lo src/libpsmx_fi_la-indexer.lo \
+ prov/util/src/libpsmx_fi_la-util_attr.lo \
+ prov/util/src/libpsmx_fi_la-util_av.lo \
+ prov/util/src/libpsmx_fi_la-util_cq.lo \
+ prov/util/src/libpsmx_fi_la-util_domain.lo \
+ prov/util/src/libpsmx_fi_la-util_ep.lo \
+ prov/util/src/libpsmx_fi_la-util_eq.lo \
+ prov/util/src/libpsmx_fi_la-util_fabric.lo \
+ prov/util/src/libpsmx_fi_la-util_main.lo \
+ prov/util/src/libpsmx_fi_la-util_poll.lo \
+ prov/util/src/libpsmx_fi_la-util_wait.lo \
+ prov/util/src/libpsmx_fi_la-util_buf.lo \
+ prov/util/src/libpsmx_fi_la-util_mr.lo $(am__objects_13) \
+ $(am__objects_14) $(am__objects_15)
+ at HAVE_PSM_DL_TRUE@@HAVE_PSM_TRUE at am_libpsmx_fi_la_OBJECTS = \
+ at HAVE_PSM_DL_TRUE@@HAVE_PSM_TRUE@ $(am__objects_12) \
+ at HAVE_PSM_DL_TRUE@@HAVE_PSM_TRUE@ $(am__objects_16)
+libpsmx_fi_la_OBJECTS = $(am_libpsmx_fi_la_OBJECTS)
+libpsmx_fi_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(libpsmx_fi_la_LDFLAGS) $(LDFLAGS) -o $@
+ at HAVE_PSM_DL_TRUE@@HAVE_PSM_TRUE at am_libpsmx_fi_la_rpath = -rpath \
+ at HAVE_PSM_DL_TRUE@@HAVE_PSM_TRUE@ $(pkglibdir)
+ at HAVE_PSM_DL_FALSE@@HAVE_PSM_TRUE at libpsmx_la_DEPENDENCIES = \
+ at HAVE_PSM_DL_FALSE@@HAVE_PSM_TRUE@ $(am__DEPENDENCIES_1)
+am__libpsmx_la_SOURCES_DIST = prov/psm/src/version.h \
+ prov/psm/src/psm_am.h prov/psm/src/psmx.h \
+ prov/psm/src/psmx_init.c prov/psm/src/psmx_fabric.c \
+ prov/psm/src/psmx_domain.c prov/psm/src/psmx_cq.c \
+ prov/psm/src/psmx_cntr.c prov/psm/src/psmx_av.c \
+ prov/psm/src/psmx_ep.c prov/psm/src/psmx_cm.c \
+ prov/psm/src/psmx_tagged.c prov/psm/src/psmx_msg.c \
+ prov/psm/src/psmx_msg2.c prov/psm/src/psmx_rma.c \
+ prov/psm/src/psmx_atomic.c prov/psm/src/psmx_am.c \
+ prov/psm/src/psmx_mr.c prov/psm/src/psmx_wait.c \
+ prov/psm/src/psmx_util.c
+ at HAVE_PSM_TRUE@am__objects_17 = prov/psm/src/libpsmx_la-psmx_init.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_fabric.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_domain.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_cq.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_cntr.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_av.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_ep.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_cm.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_tagged.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_msg.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_msg2.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_rma.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_atomic.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_am.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_mr.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_wait.lo \
+ at HAVE_PSM_TRUE@ prov/psm/src/libpsmx_la-psmx_util.lo
+ at HAVE_PSM_DL_FALSE@@HAVE_PSM_TRUE at am_libpsmx_la_OBJECTS = \
+ at HAVE_PSM_DL_FALSE@@HAVE_PSM_TRUE@ $(am__objects_17)
+libpsmx_la_OBJECTS = $(am_libpsmx_la_OBJECTS)
+libpsmx_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(libpsmx_la_LDFLAGS) $(LDFLAGS) -o $@
+ at HAVE_PSM_DL_FALSE@@HAVE_PSM_TRUE at am_libpsmx_la_rpath =
+am__libpsmx2_fi_la_SOURCES_DIST = prov/psm2/src/version.h \
+ prov/psm2/src/psmx2.h prov/psm2/src/psmx2_init.c \
+ prov/psm2/src/psmx2_fabric.c prov/psm2/src/psmx2_domain.c \
+ prov/psm2/src/psmx2_cq.c prov/psm2/src/psmx2_cntr.c \
+ prov/psm2/src/psmx2_av.c prov/psm2/src/psmx2_ep.c \
+ prov/psm2/src/psmx2_cm.c prov/psm2/src/psmx2_tagged.c \
+ prov/psm2/src/psmx2_msg.c prov/psm2/src/psmx2_rma.c \
+ prov/psm2/src/psmx2_atomic.c prov/psm2/src/psmx2_am.c \
+ prov/psm2/src/psmx2_mr.c prov/psm2/src/psmx2_wait.c \
+ prov/psm2/src/psmx2_util.c src/common.c src/enosys.c \
+ src/rbtree.c src/fasthash.c src/indexer.c \
+ prov/util/src/util_attr.c prov/util/src/util_av.c \
+ prov/util/src/util_cq.c prov/util/src/util_domain.c \
+ prov/util/src/util_ep.c prov/util/src/util_eq.c \
+ prov/util/src/util_fabric.c prov/util/src/util_main.c \
+ prov/util/src/util_poll.c prov/util/src/util_wait.c \
+ prov/util/src/util_buf.c prov/util/src/util_mr.c src/osx/osd.c \
+ src/unix/osd.c include/osx/osd.h include/unix/osd.h \
+ include/freebsd/osd.h include/linux/osd.h
+ at HAVE_PSM2_TRUE@am__objects_18 = \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_init.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_fabric.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_domain.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_cq.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_cntr.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_av.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_ep.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_cm.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_tagged.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_msg.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_rma.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_atomic.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_am.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_mr.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_wait.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_fi_la-psmx2_util.lo
+ at MACOS_TRUE@am__objects_19 = src/osx/libpsmx2_fi_la-osd.lo \
+ at MACOS_TRUE@ src/unix/libpsmx2_fi_la-osd.lo
+ at FREEBSD_TRUE@am__objects_20 = src/unix/libpsmx2_fi_la-osd.lo
+ at LINUX_TRUE@am__objects_21 = src/unix/libpsmx2_fi_la-osd.lo
+am__objects_22 = src/libpsmx2_fi_la-common.lo \
+ src/libpsmx2_fi_la-enosys.lo src/libpsmx2_fi_la-rbtree.lo \
+ src/libpsmx2_fi_la-fasthash.lo src/libpsmx2_fi_la-indexer.lo \
+ prov/util/src/libpsmx2_fi_la-util_attr.lo \
+ prov/util/src/libpsmx2_fi_la-util_av.lo \
+ prov/util/src/libpsmx2_fi_la-util_cq.lo \
+ prov/util/src/libpsmx2_fi_la-util_domain.lo \
+ prov/util/src/libpsmx2_fi_la-util_ep.lo \
+ prov/util/src/libpsmx2_fi_la-util_eq.lo \
+ prov/util/src/libpsmx2_fi_la-util_fabric.lo \
+ prov/util/src/libpsmx2_fi_la-util_main.lo \
+ prov/util/src/libpsmx2_fi_la-util_poll.lo \
+ prov/util/src/libpsmx2_fi_la-util_wait.lo \
+ prov/util/src/libpsmx2_fi_la-util_buf.lo \
+ prov/util/src/libpsmx2_fi_la-util_mr.lo $(am__objects_19) \
+ $(am__objects_20) $(am__objects_21)
+ at HAVE_PSM2_DL_TRUE@@HAVE_PSM2_TRUE at am_libpsmx2_fi_la_OBJECTS = \
+ at HAVE_PSM2_DL_TRUE@@HAVE_PSM2_TRUE@ $(am__objects_18) \
+ at HAVE_PSM2_DL_TRUE@@HAVE_PSM2_TRUE@ $(am__objects_22)
+libpsmx2_fi_la_OBJECTS = $(am_libpsmx2_fi_la_OBJECTS)
+libpsmx2_fi_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+ $(AM_CFLAGS) $(CFLAGS) $(libpsmx2_fi_la_LDFLAGS) $(LDFLAGS) -o \
+ $@
+ at HAVE_PSM2_DL_TRUE@@HAVE_PSM2_TRUE at am_libpsmx2_fi_la_rpath = -rpath \
+ at HAVE_PSM2_DL_TRUE@@HAVE_PSM2_TRUE@ $(pkglibdir)
+ at HAVE_PSM2_DL_FALSE@@HAVE_PSM2_TRUE at libpsmx2_la_DEPENDENCIES = \
+ at HAVE_PSM2_DL_FALSE@@HAVE_PSM2_TRUE@ $(am__DEPENDENCIES_1)
+am__libpsmx2_la_SOURCES_DIST = prov/psm2/src/version.h \
+ prov/psm2/src/psmx2.h prov/psm2/src/psmx2_init.c \
+ prov/psm2/src/psmx2_fabric.c prov/psm2/src/psmx2_domain.c \
+ prov/psm2/src/psmx2_cq.c prov/psm2/src/psmx2_cntr.c \
+ prov/psm2/src/psmx2_av.c prov/psm2/src/psmx2_ep.c \
+ prov/psm2/src/psmx2_cm.c prov/psm2/src/psmx2_tagged.c \
+ prov/psm2/src/psmx2_msg.c prov/psm2/src/psmx2_rma.c \
+ prov/psm2/src/psmx2_atomic.c prov/psm2/src/psmx2_am.c \
+ prov/psm2/src/psmx2_mr.c prov/psm2/src/psmx2_wait.c \
+ prov/psm2/src/psmx2_util.c
+ at HAVE_PSM2_TRUE@am__objects_23 = \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_init.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_fabric.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_domain.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_cq.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_cntr.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_av.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_ep.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_cm.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_tagged.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_msg.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_rma.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_atomic.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_am.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_mr.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_wait.lo \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/libpsmx2_la-psmx2_util.lo
+ at HAVE_PSM2_DL_FALSE@@HAVE_PSM2_TRUE at am_libpsmx2_la_OBJECTS = \
+ at HAVE_PSM2_DL_FALSE@@HAVE_PSM2_TRUE@ $(am__objects_23)
+libpsmx2_la_OBJECTS = $(am_libpsmx2_la_OBJECTS)
+libpsmx2_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(libpsmx2_la_LDFLAGS) $(LDFLAGS) -o $@
+ at HAVE_PSM2_DL_FALSE@@HAVE_PSM2_TRUE at am_libpsmx2_la_rpath =
+am__librxd_fi_la_SOURCES_DIST = prov/rxd/src/rxd_attr.c \
+ prov/rxd/src/rxd_init.c prov/rxd/src/rxd_fabric.c \
+ prov/rxd/src/rxd_domain.c prov/rxd/src/rxd_av.c \
+ prov/rxd/src/rxd_cq.c prov/rxd/src/rxd_ep.c \
+ prov/rxd/src/rxd_rma.c prov/rxd/src/rxd.h src/common.c \
+ src/enosys.c src/rbtree.c src/fasthash.c src/indexer.c \
+ prov/util/src/util_attr.c prov/util/src/util_av.c \
+ prov/util/src/util_cq.c prov/util/src/util_domain.c \
+ prov/util/src/util_ep.c prov/util/src/util_eq.c \
+ prov/util/src/util_fabric.c prov/util/src/util_main.c \
+ prov/util/src/util_poll.c prov/util/src/util_wait.c \
+ prov/util/src/util_buf.c prov/util/src/util_mr.c src/osx/osd.c \
+ src/unix/osd.c include/osx/osd.h include/unix/osd.h \
+ include/freebsd/osd.h include/linux/osd.h
+ at HAVE_RXD_TRUE@am__objects_24 = prov/rxd/src/rxd_attr.lo \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd_init.lo \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd_fabric.lo \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd_domain.lo \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd_av.lo prov/rxd/src/rxd_cq.lo \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd_ep.lo prov/rxd/src/rxd_rma.lo
+ at MACOS_TRUE@am__objects_25 = src/osx/osd.lo src/unix/osd.lo
+ at FREEBSD_TRUE@am__objects_26 = src/unix/osd.lo
+ at LINUX_TRUE@am__objects_27 = src/unix/osd.lo
+am__objects_28 = src/common.lo src/enosys.lo src/rbtree.lo \
+ src/fasthash.lo src/indexer.lo prov/util/src/util_attr.lo \
+ prov/util/src/util_av.lo prov/util/src/util_cq.lo \
+ prov/util/src/util_domain.lo prov/util/src/util_ep.lo \
+ prov/util/src/util_eq.lo prov/util/src/util_fabric.lo \
+ prov/util/src/util_main.lo prov/util/src/util_poll.lo \
+ prov/util/src/util_wait.lo prov/util/src/util_buf.lo \
+ prov/util/src/util_mr.lo $(am__objects_25) $(am__objects_26) \
+ $(am__objects_27)
+ at HAVE_RXD_DL_TRUE@@HAVE_RXD_TRUE at am_librxd_fi_la_OBJECTS = \
+ at HAVE_RXD_DL_TRUE@@HAVE_RXD_TRUE@ $(am__objects_24) \
+ at HAVE_RXD_DL_TRUE@@HAVE_RXD_TRUE@ $(am__objects_28)
+librxd_fi_la_OBJECTS = $(am_librxd_fi_la_OBJECTS)
+librxd_fi_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(librxd_fi_la_LDFLAGS) $(LDFLAGS) -o $@
+ at HAVE_RXD_DL_TRUE@@HAVE_RXD_TRUE at am_librxd_fi_la_rpath = -rpath \
+ at HAVE_RXD_DL_TRUE@@HAVE_RXD_TRUE@ $(pkglibdir)
+am__librxm_fi_la_SOURCES_DIST = prov/rxm/src/rxm_attr.c \
+ prov/rxm/src/rxm_init.c prov/rxm/src/rxm_fabric.c \
+ prov/rxm/src/rxm_domain.c prov/rxm/src/rxm_conn.c \
+ prov/rxm/src/rxm_ep.c prov/rxm/src/rxm_cq.c prov/rxm/src/rxm.h \
+ src/common.c src/enosys.c src/rbtree.c src/fasthash.c \
+ src/indexer.c prov/util/src/util_attr.c \
+ prov/util/src/util_av.c prov/util/src/util_cq.c \
+ prov/util/src/util_domain.c prov/util/src/util_ep.c \
+ prov/util/src/util_eq.c prov/util/src/util_fabric.c \
+ prov/util/src/util_main.c prov/util/src/util_poll.c \
+ prov/util/src/util_wait.c prov/util/src/util_buf.c \
+ prov/util/src/util_mr.c src/osx/osd.c src/unix/osd.c \
+ include/osx/osd.h include/unix/osd.h include/freebsd/osd.h \
+ include/linux/osd.h
+ at HAVE_RXM_TRUE@am__objects_29 = prov/rxm/src/rxm_attr.lo \
+ at HAVE_RXM_TRUE@ prov/rxm/src/rxm_init.lo \
+ at HAVE_RXM_TRUE@ prov/rxm/src/rxm_fabric.lo \
+ at HAVE_RXM_TRUE@ prov/rxm/src/rxm_domain.lo \
+ at HAVE_RXM_TRUE@ prov/rxm/src/rxm_conn.lo prov/rxm/src/rxm_ep.lo \
+ at HAVE_RXM_TRUE@ prov/rxm/src/rxm_cq.lo
+ at HAVE_RXM_DL_TRUE@@HAVE_RXM_TRUE at am_librxm_fi_la_OBJECTS = \
+ at HAVE_RXM_DL_TRUE@@HAVE_RXM_TRUE@ $(am__objects_29) \
+ at HAVE_RXM_DL_TRUE@@HAVE_RXM_TRUE@ $(am__objects_28)
+librxm_fi_la_OBJECTS = $(am_librxm_fi_la_OBJECTS)
+librxm_fi_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(librxm_fi_la_LDFLAGS) $(LDFLAGS) -o $@
+ at HAVE_RXM_DL_TRUE@@HAVE_RXM_TRUE at am_librxm_fi_la_rpath = -rpath \
+ at HAVE_RXM_DL_TRUE@@HAVE_RXM_TRUE@ $(pkglibdir)
+am__libsockets_fi_la_SOURCES_DIST = prov/sockets/src/sock_av.c \
+ prov/sockets/src/sock_dom.c prov/sockets/src/sock_eq.c \
+ prov/sockets/src/sock_cq.c prov/sockets/src/sock_cntr.c \
+ prov/sockets/src/sock_poll.c prov/sockets/src/sock_wait.c \
+ prov/sockets/src/sock_ep_rdm.c \
+ prov/sockets/src/sock_ep_dgram.c \
+ prov/sockets/src/sock_ep_msg.c prov/sockets/src/sock_fabric.c \
+ prov/sockets/src/sock_ep.c prov/sockets/src/sock_ctx.c \
+ prov/sockets/src/sock_rx_entry.c \
+ prov/sockets/src/sock_progress.c prov/sockets/src/sock_comm.c \
+ prov/sockets/src/sock_conn.c prov/sockets/src/sock_msg.c \
+ prov/sockets/src/sock_rma.c prov/sockets/src/sock_atomic.c \
+ prov/sockets/src/sock_trigger.c prov/sockets/src/sock_epoll.c \
+ prov/sockets/include/sock.h prov/sockets/include/sock_util.h \
+ src/common.c src/enosys.c src/rbtree.c src/fasthash.c \
+ src/indexer.c prov/util/src/util_attr.c \
+ prov/util/src/util_av.c prov/util/src/util_cq.c \
+ prov/util/src/util_domain.c prov/util/src/util_ep.c \
+ prov/util/src/util_eq.c prov/util/src/util_fabric.c \
+ prov/util/src/util_main.c prov/util/src/util_poll.c \
+ prov/util/src/util_wait.c prov/util/src/util_buf.c \
+ prov/util/src/util_mr.c src/osx/osd.c src/unix/osd.c \
+ include/osx/osd.h include/unix/osd.h include/freebsd/osd.h \
+ include/linux/osd.h
+ at HAVE_SOCKETS_TRUE@am__objects_30 = prov/sockets/src/sock_av.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_dom.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_eq.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_cq.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_cntr.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_poll.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_wait.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_ep_rdm.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_ep_dgram.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_ep_msg.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_fabric.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_ep.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_ctx.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_rx_entry.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_progress.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_comm.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_conn.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_msg.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_rma.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_atomic.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_trigger.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_epoll.lo
+ at HAVE_SOCKETS_DL_TRUE@@HAVE_SOCKETS_TRUE at am_libsockets_fi_la_OBJECTS = \
+ at HAVE_SOCKETS_DL_TRUE@@HAVE_SOCKETS_TRUE@ $(am__objects_30) \
+ at HAVE_SOCKETS_DL_TRUE@@HAVE_SOCKETS_TRUE@ $(am__objects_2) \
+ at HAVE_SOCKETS_DL_TRUE@@HAVE_SOCKETS_TRUE@ $(am__objects_28)
+libsockets_fi_la_OBJECTS = $(am_libsockets_fi_la_OBJECTS)
+libsockets_fi_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+ $(AM_CFLAGS) $(CFLAGS) $(libsockets_fi_la_LDFLAGS) $(LDFLAGS) \
+ -o $@
+ at HAVE_SOCKETS_DL_TRUE@@HAVE_SOCKETS_TRUE at am_libsockets_fi_la_rpath = \
+ at HAVE_SOCKETS_DL_TRUE@@HAVE_SOCKETS_TRUE@ -rpath $(pkglibdir)
+am__libudp_fi_la_SOURCES_DIST = prov/udp/src/udpx_attr.c \
+ prov/udp/src/udpx_cq.c prov/udp/src/udpx_domain.c \
+ prov/udp/src/udpx_ep.c prov/udp/src/udpx_fabric.c \
+ prov/udp/src/udpx_init.c prov/udp/src/udpx.h src/common.c \
+ src/enosys.c src/rbtree.c src/fasthash.c src/indexer.c \
+ prov/util/src/util_attr.c prov/util/src/util_av.c \
+ prov/util/src/util_cq.c prov/util/src/util_domain.c \
+ prov/util/src/util_ep.c prov/util/src/util_eq.c \
+ prov/util/src/util_fabric.c prov/util/src/util_main.c \
+ prov/util/src/util_poll.c prov/util/src/util_wait.c \
+ prov/util/src/util_buf.c prov/util/src/util_mr.c src/osx/osd.c \
+ src/unix/osd.c include/osx/osd.h include/unix/osd.h \
+ include/freebsd/osd.h include/linux/osd.h
+ at HAVE_UDP_TRUE@am__objects_31 = prov/udp/src/udpx_attr.lo \
+ at HAVE_UDP_TRUE@ prov/udp/src/udpx_cq.lo \
+ at HAVE_UDP_TRUE@ prov/udp/src/udpx_domain.lo \
+ at HAVE_UDP_TRUE@ prov/udp/src/udpx_ep.lo \
+ at HAVE_UDP_TRUE@ prov/udp/src/udpx_fabric.lo \
+ at HAVE_UDP_TRUE@ prov/udp/src/udpx_init.lo
+ at HAVE_UDP_DL_TRUE@@HAVE_UDP_TRUE at am_libudp_fi_la_OBJECTS = \
+ at HAVE_UDP_DL_TRUE@@HAVE_UDP_TRUE@ $(am__objects_31) \
+ at HAVE_UDP_DL_TRUE@@HAVE_UDP_TRUE@ $(am__objects_28)
+libudp_fi_la_OBJECTS = $(am_libudp_fi_la_OBJECTS)
+libudp_fi_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(libudp_fi_la_LDFLAGS) $(LDFLAGS) -o $@
+ at HAVE_UDP_DL_TRUE@@HAVE_UDP_TRUE at am_libudp_fi_la_rpath = -rpath \
+ at HAVE_UDP_DL_TRUE@@HAVE_UDP_TRUE@ $(pkglibdir)
+am__libusnic_fi_la_SOURCES_DIST = \
+ prov/usnic/src/usnic_direct/cq_desc.h \
+ prov/usnic/src/usnic_direct/cq_enet_desc.h \
+ prov/usnic/src/usnic_direct/kcompat.h \
+ prov/usnic/src/usnic_direct/kcompat_priv.h \
+ prov/usnic/src/usnic_direct/libnl1_utils.h \
+ prov/usnic/src/usnic_direct/libnl3_utils.h \
+ prov/usnic/src/usnic_direct/libnl_utils_common.c \
+ prov/usnic/src/usnic_direct/libnl_utils.h \
+ prov/usnic/src/usnic_direct/linux/delay.h \
+ prov/usnic/src/usnic_direct/linux/slab.h \
+ prov/usnic/src/usnic_direct/linux_types.h \
+ prov/usnic/src/usnic_direct/rq_enet_desc.h \
+ prov/usnic/src/usnic_direct/usd_caps.c \
+ prov/usnic/src/usnic_direct/usd_caps.h \
+ prov/usnic/src/usnic_direct/usd_dest.c \
+ prov/usnic/src/usnic_direct/usd_dest.h \
+ prov/usnic/src/usnic_direct/usd_device.c \
+ prov/usnic/src/usnic_direct/usd_device.h \
+ prov/usnic/src/usnic_direct/usd_event.c \
+ prov/usnic/src/usnic_direct/usd_enum.c \
+ prov/usnic/src/usnic_direct/usd.h \
+ prov/usnic/src/usnic_direct/usd_ib_cmd.c \
+ prov/usnic/src/usnic_direct/usd_ib_cmd.h \
+ prov/usnic/src/usnic_direct/usd_ib_sysfs.c \
+ prov/usnic/src/usnic_direct/usd_ib_sysfs.h \
+ prov/usnic/src/usnic_direct/usd_mem.c \
+ prov/usnic/src/usnic_direct/usd_poll.c \
+ prov/usnic/src/usnic_direct/usd_post.c \
+ prov/usnic/src/usnic_direct/usd_post.h \
+ prov/usnic/src/usnic_direct/usd_post_ud_raw.c \
+ prov/usnic/src/usnic_direct/usd_post_ud_udp.c \
+ prov/usnic/src/usnic_direct/usd_post_ud_pio_udp.c \
+ prov/usnic/src/usnic_direct/usd_queue.h \
+ prov/usnic/src/usnic_direct/usd_queues.c \
+ prov/usnic/src/usnic_direct/usd_socket.c \
+ prov/usnic/src/usnic_direct/usd_socket.h \
+ prov/usnic/src/usnic_direct/usd_time.h \
+ prov/usnic/src/usnic_direct/usd_util.h \
+ prov/usnic/src/usnic_direct/usd_vnic.c \
+ prov/usnic/src/usnic_direct/usd_vnic.h \
+ prov/usnic/src/usnic_direct/usnic_abi.h \
+ prov/usnic/src/usnic_direct/usnic_direct.h \
+ prov/usnic/src/usnic_direct/usnic_ib_abi.h \
+ prov/usnic/src/usnic_direct/usnic_ip_utils.c \
+ prov/usnic/src/usnic_direct/usnic_ip_utils.h \
+ prov/usnic/src/usnic_direct/usnic_user_utils.h \
+ prov/usnic/src/usnic_direct/vnic_cq.c \
+ prov/usnic/src/usnic_direct/vnic_cq.h \
+ prov/usnic/src/usnic_direct/vnic_dev.c \
+ prov/usnic/src/usnic_direct/vnic_devcmd.h \
+ prov/usnic/src/usnic_direct/vnic_dev.h \
+ prov/usnic/src/usnic_direct/vnic_enet.h \
+ prov/usnic/src/usnic_direct/vnic_resource.h \
+ prov/usnic/src/usnic_direct/vnic_rq.c \
+ prov/usnic/src/usnic_direct/vnic_rq.h \
+ prov/usnic/src/usnic_direct/vnic_stats.h \
+ prov/usnic/src/usnic_direct/vnic_wq.c \
+ prov/usnic/src/usnic_direct/vnic_wq.h \
+ prov/usnic/src/usnic_direct/vnic_intr.c \
+ prov/usnic/src/usnic_direct/vnic_intr.h \
+ prov/usnic/src/usnic_direct/wq_enet_desc.h \
+ prov/usnic/src/fi_ext_usnic.h prov/usnic/src/usdf.h \
+ prov/usnic/src/usdf_av.c prov/usnic/src/usdf_av.h \
+ prov/usnic/src/usdf_cm.c prov/usnic/src/usdf_cm.h \
+ prov/usnic/src/usdf_cq.c prov/usnic/src/usdf_cq.h \
+ prov/usnic/src/usdf_dgram.c prov/usnic/src/usdf_dgram.h \
+ prov/usnic/src/usdf_domain.c prov/usnic/src/usdf_endpoint.c \
+ prov/usnic/src/usdf_endpoint.h prov/usnic/src/usdf_ep_dgram.c \
+ prov/usnic/src/usdf_ep_msg.c prov/usnic/src/usdf_ep_rdm.c \
+ prov/usnic/src/usdf_eq.c prov/usnic/src/usdf_fabric.c \
+ prov/usnic/src/usdf_mem.c prov/usnic/src/usdf_msg.c \
+ prov/usnic/src/usdf_msg.h prov/usnic/src/usdf_pep.c \
+ prov/usnic/src/usdf_progress.c prov/usnic/src/usdf_progress.h \
+ prov/usnic/src/usdf_rdm.c prov/usnic/src/usdf_rdm.h \
+ prov/usnic/src/usdf_rudp.h prov/usnic/src/usdf_timer.c \
+ prov/usnic/src/usdf_timer.h prov/usnic/src/usdf_poll.c \
+ prov/usnic/src/usdf_poll.h prov/usnic/src/usdf_ext.c \
+ prov/usnic/src/usdf_wait.h prov/usnic/src/usdf_wait.c \
+ prov/usnic/src/usdf_fake_ibv.c src/common.c src/enosys.c \
+ src/rbtree.c src/fasthash.c src/indexer.c \
+ prov/util/src/util_attr.c prov/util/src/util_av.c \
+ prov/util/src/util_cq.c prov/util/src/util_domain.c \
+ prov/util/src/util_ep.c prov/util/src/util_eq.c \
+ prov/util/src/util_fabric.c prov/util/src/util_main.c \
+ prov/util/src/util_poll.c prov/util/src/util_wait.c \
+ prov/util/src/util_buf.c prov/util/src/util_mr.c src/osx/osd.c \
+ src/unix/osd.c include/osx/osd.h include/unix/osd.h \
+ include/freebsd/osd.h include/linux/osd.h
+ at HAVE_USNIC_TRUE@am__objects_32 = prov/usnic/src/usnic_direct/libusnic_fi_la-libnl_utils_common.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_caps.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_dest.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_device.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_event.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_enum.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_cmd.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_sysfs.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_mem.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_poll.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_raw.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_udp.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_pio_udp.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_queues.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_socket.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usd_vnic.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-usnic_ip_utils.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_cq.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_dev.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_rq.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_wq.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_intr.lo
+ at HAVE_USNIC_TRUE@@HAVE_VERBS_TRUE at am__objects_33 = prov/usnic/src/libusnic_fi_la-usdf_fake_ibv.lo
+ at HAVE_USNIC_TRUE@am__objects_34 = $(am__objects_32) \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_av.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_cm.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_cq.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_dgram.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_domain.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_endpoint.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_ep_dgram.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_ep_msg.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_ep_rdm.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_eq.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_fabric.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_mem.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_msg.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_pep.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_progress.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_rdm.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_timer.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_poll.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_ext.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/libusnic_fi_la-usdf_wait.lo \
+ at HAVE_USNIC_TRUE@ $(am__objects_33)
+ at MACOS_TRUE@am__objects_35 = src/osx/libusnic_fi_la-osd.lo \
+ at MACOS_TRUE@ src/unix/libusnic_fi_la-osd.lo
+ at FREEBSD_TRUE@am__objects_36 = src/unix/libusnic_fi_la-osd.lo
+ at LINUX_TRUE@am__objects_37 = src/unix/libusnic_fi_la-osd.lo
+am__objects_38 = src/libusnic_fi_la-common.lo \
+ src/libusnic_fi_la-enosys.lo src/libusnic_fi_la-rbtree.lo \
+ src/libusnic_fi_la-fasthash.lo src/libusnic_fi_la-indexer.lo \
+ prov/util/src/libusnic_fi_la-util_attr.lo \
+ prov/util/src/libusnic_fi_la-util_av.lo \
+ prov/util/src/libusnic_fi_la-util_cq.lo \
+ prov/util/src/libusnic_fi_la-util_domain.lo \
+ prov/util/src/libusnic_fi_la-util_ep.lo \
+ prov/util/src/libusnic_fi_la-util_eq.lo \
+ prov/util/src/libusnic_fi_la-util_fabric.lo \
+ prov/util/src/libusnic_fi_la-util_main.lo \
+ prov/util/src/libusnic_fi_la-util_poll.lo \
+ prov/util/src/libusnic_fi_la-util_wait.lo \
+ prov/util/src/libusnic_fi_la-util_buf.lo \
+ prov/util/src/libusnic_fi_la-util_mr.lo $(am__objects_35) \
+ $(am__objects_36) $(am__objects_37)
+ at HAVE_USNIC_DL_TRUE@@HAVE_USNIC_TRUE at am_libusnic_fi_la_OBJECTS = \
+ at HAVE_USNIC_DL_TRUE@@HAVE_USNIC_TRUE@ $(am__objects_34) \
+ at HAVE_USNIC_DL_TRUE@@HAVE_USNIC_TRUE@ $(am__objects_38)
+libusnic_fi_la_OBJECTS = $(am_libusnic_fi_la_OBJECTS)
+libusnic_fi_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+ $(AM_CFLAGS) $(CFLAGS) $(libusnic_fi_la_LDFLAGS) $(LDFLAGS) -o \
+ $@
+ at HAVE_USNIC_DL_TRUE@@HAVE_USNIC_TRUE at am_libusnic_fi_la_rpath = -rpath \
+ at HAVE_USNIC_DL_TRUE@@HAVE_USNIC_TRUE@ $(pkglibdir)
+am__libverbs_fi_la_SOURCES_DIST = prov/verbs/src/fi_verbs.h \
+ prov/verbs/src/fi_verbs.c prov/verbs/src/uthash.h \
+ prov/verbs/src/verbs_atomic.c prov/verbs/src/verbs_av.c \
+ prov/verbs/src/verbs_cm.c prov/verbs/src/verbs_cq.c \
+ prov/verbs/src/verbs_srq.c prov/verbs/src/verbs_domain.c \
+ prov/verbs/src/verbs_eq.c prov/verbs/src/verbs_info.c \
+ prov/verbs/src/verbs_msg.c prov/verbs/src/verbs_msg_ep.c \
+ prov/verbs/src/verbs_rma.c \
+ prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c \
+ prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c \
+ prov/verbs/src/ep_rdm/verbs_ep_rdm.c \
+ prov/verbs/src/ep_rdm/verbs_queuing.h \
+ prov/verbs/src/ep_rdm/verbs_rdm_cm.c \
+ prov/verbs/src/ep_rdm/verbs_rdm_cntr.c \
+ prov/verbs/src/ep_rdm/verbs_rdm_msg.c \
+ prov/verbs/src/ep_rdm/verbs_rdm.h \
+ prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.h \
+ prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c \
+ prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c \
+ prov/verbs/src/ep_rdm/verbs_utils.c \
+ prov/verbs/src/ep_rdm/verbs_utils.h src/common.c src/enosys.c \
+ src/rbtree.c src/fasthash.c src/indexer.c \
+ prov/util/src/util_attr.c prov/util/src/util_av.c \
+ prov/util/src/util_cq.c prov/util/src/util_domain.c \
+ prov/util/src/util_ep.c prov/util/src/util_eq.c \
+ prov/util/src/util_fabric.c prov/util/src/util_main.c \
+ prov/util/src/util_poll.c prov/util/src/util_wait.c \
+ prov/util/src/util_buf.c prov/util/src/util_mr.c src/osx/osd.c \
+ src/unix/osd.c include/osx/osd.h include/unix/osd.h \
+ include/freebsd/osd.h include/linux/osd.h
+ at HAVE_VERBS_TRUE@am__objects_39 = \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/libverbs_fi_la-fi_verbs.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/libverbs_fi_la-verbs_atomic.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/libverbs_fi_la-verbs_av.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/libverbs_fi_la-verbs_cm.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/libverbs_fi_la-verbs_cq.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/libverbs_fi_la-verbs_srq.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/libverbs_fi_la-verbs_domain.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/libverbs_fi_la-verbs_eq.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/libverbs_fi_la-verbs_info.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/libverbs_fi_la-verbs_msg.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/libverbs_fi_la-verbs_msg_ep.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/libverbs_fi_la-verbs_rma.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_av_ep_rdm.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_cq_ep_rdm.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_ep_rdm.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cm.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cntr.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_msg.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm_states.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_utils.lo
+ at MACOS_TRUE@am__objects_40 = src/osx/libverbs_fi_la-osd.lo \
+ at MACOS_TRUE@ src/unix/libverbs_fi_la-osd.lo
+ at FREEBSD_TRUE@am__objects_41 = src/unix/libverbs_fi_la-osd.lo
+ at LINUX_TRUE@am__objects_42 = src/unix/libverbs_fi_la-osd.lo
+am__objects_43 = src/libverbs_fi_la-common.lo \
+ src/libverbs_fi_la-enosys.lo src/libverbs_fi_la-rbtree.lo \
+ src/libverbs_fi_la-fasthash.lo src/libverbs_fi_la-indexer.lo \
+ prov/util/src/libverbs_fi_la-util_attr.lo \
+ prov/util/src/libverbs_fi_la-util_av.lo \
+ prov/util/src/libverbs_fi_la-util_cq.lo \
+ prov/util/src/libverbs_fi_la-util_domain.lo \
+ prov/util/src/libverbs_fi_la-util_ep.lo \
+ prov/util/src/libverbs_fi_la-util_eq.lo \
+ prov/util/src/libverbs_fi_la-util_fabric.lo \
+ prov/util/src/libverbs_fi_la-util_main.lo \
+ prov/util/src/libverbs_fi_la-util_poll.lo \
+ prov/util/src/libverbs_fi_la-util_wait.lo \
+ prov/util/src/libverbs_fi_la-util_buf.lo \
+ prov/util/src/libverbs_fi_la-util_mr.lo $(am__objects_40) \
+ $(am__objects_41) $(am__objects_42)
+ at HAVE_VERBS_DL_TRUE@@HAVE_VERBS_TRUE at am_libverbs_fi_la_OBJECTS = \
+ at HAVE_VERBS_DL_TRUE@@HAVE_VERBS_TRUE@ $(am__objects_39) \
+ at HAVE_VERBS_DL_TRUE@@HAVE_VERBS_TRUE@ $(am__objects_43)
+libverbs_fi_la_OBJECTS = $(am_libverbs_fi_la_OBJECTS)
+libverbs_fi_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+ $(AM_CFLAGS) $(CFLAGS) $(libverbs_fi_la_LDFLAGS) $(LDFLAGS) -o \
+ $@
+ at HAVE_VERBS_DL_TRUE@@HAVE_VERBS_TRUE at am_libverbs_fi_la_rpath = -rpath \
+ at HAVE_VERBS_DL_TRUE@@HAVE_VERBS_TRUE@ $(pkglibdir)
+ at HAVE_SOCKETS_DL_FALSE@@HAVE_SOCKETS_TRUE at am__DEPENDENCIES_2 = $(am__DEPENDENCIES_1)
+ at HAVE_UDP_DL_FALSE@@HAVE_UDP_TRUE at am__DEPENDENCIES_3 = \
+ at HAVE_UDP_DL_FALSE@@HAVE_UDP_TRUE@ $(am__DEPENDENCIES_1)
+ at HAVE_VERBS_DL_FALSE@@HAVE_VERBS_TRUE at am__DEPENDENCIES_4 = \
+ at HAVE_VERBS_DL_FALSE@@HAVE_VERBS_TRUE@ $(am__DEPENDENCIES_1)
+ at HAVE_USNIC_DL_FALSE@@HAVE_USNIC_TRUE at am__DEPENDENCIES_5 = \
+ at HAVE_USNIC_DL_FALSE@@HAVE_USNIC_TRUE@ $(am__DEPENDENCIES_1)
+ at HAVE_MXM_DL_FALSE@@HAVE_MXM_TRUE at am__DEPENDENCIES_6 = \
+ at HAVE_MXM_DL_FALSE@@HAVE_MXM_TRUE@ $(am__DEPENDENCIES_1)
+am__src_libfabric_la_SOURCES_DIST = include/fi.h include/fi_abi.h \
+ include/fi_atom.h include/fi_enosys.h include/fi_file.h \
+ include/fi_indexer.h include/fi_list.h include/fi_lock.h \
+ include/fi_mem.h include/fi_osd.h include/fi_proto.h \
+ include/fi_rbuf.h include/fi_signal.h include/fi_util.h \
+ include/fasthash.h include/rbtree.h include/prov.h \
+ include/rdma/providers/fi_log.h \
+ include/rdma/providers/fi_prov.h src/fabric.c src/fi_tostr.c \
+ src/log.c src/var.c src/common.c src/enosys.c src/rbtree.c \
+ src/fasthash.c src/indexer.c prov/util/src/util_attr.c \
+ prov/util/src/util_av.c prov/util/src/util_cq.c \
+ prov/util/src/util_domain.c prov/util/src/util_ep.c \
+ prov/util/src/util_eq.c prov/util/src/util_fabric.c \
+ prov/util/src/util_main.c prov/util/src/util_poll.c \
+ prov/util/src/util_wait.c prov/util/src/util_buf.c \
+ prov/util/src/util_mr.c src/osx/osd.c src/unix/osd.c \
+ include/osx/osd.h include/unix/osd.h include/freebsd/osd.h \
+ include/linux/osd.h prov/sockets/src/sock_av.c \
+ prov/sockets/src/sock_dom.c prov/sockets/src/sock_eq.c \
+ prov/sockets/src/sock_cq.c prov/sockets/src/sock_cntr.c \
+ prov/sockets/src/sock_poll.c prov/sockets/src/sock_wait.c \
+ prov/sockets/src/sock_ep_rdm.c \
+ prov/sockets/src/sock_ep_dgram.c \
+ prov/sockets/src/sock_ep_msg.c prov/sockets/src/sock_fabric.c \
+ prov/sockets/src/sock_ep.c prov/sockets/src/sock_ctx.c \
+ prov/sockets/src/sock_rx_entry.c \
+ prov/sockets/src/sock_progress.c prov/sockets/src/sock_comm.c \
+ prov/sockets/src/sock_conn.c prov/sockets/src/sock_msg.c \
+ prov/sockets/src/sock_rma.c prov/sockets/src/sock_atomic.c \
+ prov/sockets/src/sock_trigger.c prov/sockets/src/sock_epoll.c \
+ prov/sockets/include/sock.h prov/sockets/include/sock_util.h \
+ prov/udp/src/udpx_attr.c prov/udp/src/udpx_cq.c \
+ prov/udp/src/udpx_domain.c prov/udp/src/udpx_ep.c \
+ prov/udp/src/udpx_fabric.c prov/udp/src/udpx_init.c \
+ prov/udp/src/udpx.h prov/verbs/src/fi_verbs.h \
+ prov/verbs/src/fi_verbs.c prov/verbs/src/uthash.h \
+ prov/verbs/src/verbs_atomic.c prov/verbs/src/verbs_av.c \
+ prov/verbs/src/verbs_cm.c prov/verbs/src/verbs_cq.c \
+ prov/verbs/src/verbs_srq.c prov/verbs/src/verbs_domain.c \
+ prov/verbs/src/verbs_eq.c prov/verbs/src/verbs_info.c \
+ prov/verbs/src/verbs_msg.c prov/verbs/src/verbs_msg_ep.c \
+ prov/verbs/src/verbs_rma.c \
+ prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c \
+ prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c \
+ prov/verbs/src/ep_rdm/verbs_ep_rdm.c \
+ prov/verbs/src/ep_rdm/verbs_queuing.h \
+ prov/verbs/src/ep_rdm/verbs_rdm_cm.c \
+ prov/verbs/src/ep_rdm/verbs_rdm_cntr.c \
+ prov/verbs/src/ep_rdm/verbs_rdm_msg.c \
+ prov/verbs/src/ep_rdm/verbs_rdm.h \
+ prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.h \
+ prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c \
+ prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c \
+ prov/verbs/src/ep_rdm/verbs_utils.c \
+ prov/verbs/src/ep_rdm/verbs_utils.h \
+ prov/usnic/src/usnic_direct/cq_desc.h \
+ prov/usnic/src/usnic_direct/cq_enet_desc.h \
+ prov/usnic/src/usnic_direct/kcompat.h \
+ prov/usnic/src/usnic_direct/kcompat_priv.h \
+ prov/usnic/src/usnic_direct/libnl1_utils.h \
+ prov/usnic/src/usnic_direct/libnl3_utils.h \
+ prov/usnic/src/usnic_direct/libnl_utils_common.c \
+ prov/usnic/src/usnic_direct/libnl_utils.h \
+ prov/usnic/src/usnic_direct/linux/delay.h \
+ prov/usnic/src/usnic_direct/linux/slab.h \
+ prov/usnic/src/usnic_direct/linux_types.h \
+ prov/usnic/src/usnic_direct/rq_enet_desc.h \
+ prov/usnic/src/usnic_direct/usd_caps.c \
+ prov/usnic/src/usnic_direct/usd_caps.h \
+ prov/usnic/src/usnic_direct/usd_dest.c \
+ prov/usnic/src/usnic_direct/usd_dest.h \
+ prov/usnic/src/usnic_direct/usd_device.c \
+ prov/usnic/src/usnic_direct/usd_device.h \
+ prov/usnic/src/usnic_direct/usd_event.c \
+ prov/usnic/src/usnic_direct/usd_enum.c \
+ prov/usnic/src/usnic_direct/usd.h \
+ prov/usnic/src/usnic_direct/usd_ib_cmd.c \
+ prov/usnic/src/usnic_direct/usd_ib_cmd.h \
+ prov/usnic/src/usnic_direct/usd_ib_sysfs.c \
+ prov/usnic/src/usnic_direct/usd_ib_sysfs.h \
+ prov/usnic/src/usnic_direct/usd_mem.c \
+ prov/usnic/src/usnic_direct/usd_poll.c \
+ prov/usnic/src/usnic_direct/usd_post.c \
+ prov/usnic/src/usnic_direct/usd_post.h \
+ prov/usnic/src/usnic_direct/usd_post_ud_raw.c \
+ prov/usnic/src/usnic_direct/usd_post_ud_udp.c \
+ prov/usnic/src/usnic_direct/usd_post_ud_pio_udp.c \
+ prov/usnic/src/usnic_direct/usd_queue.h \
+ prov/usnic/src/usnic_direct/usd_queues.c \
+ prov/usnic/src/usnic_direct/usd_socket.c \
+ prov/usnic/src/usnic_direct/usd_socket.h \
+ prov/usnic/src/usnic_direct/usd_time.h \
+ prov/usnic/src/usnic_direct/usd_util.h \
+ prov/usnic/src/usnic_direct/usd_vnic.c \
+ prov/usnic/src/usnic_direct/usd_vnic.h \
+ prov/usnic/src/usnic_direct/usnic_abi.h \
+ prov/usnic/src/usnic_direct/usnic_direct.h \
+ prov/usnic/src/usnic_direct/usnic_ib_abi.h \
+ prov/usnic/src/usnic_direct/usnic_ip_utils.c \
+ prov/usnic/src/usnic_direct/usnic_ip_utils.h \
+ prov/usnic/src/usnic_direct/usnic_user_utils.h \
+ prov/usnic/src/usnic_direct/vnic_cq.c \
+ prov/usnic/src/usnic_direct/vnic_cq.h \
+ prov/usnic/src/usnic_direct/vnic_dev.c \
+ prov/usnic/src/usnic_direct/vnic_devcmd.h \
+ prov/usnic/src/usnic_direct/vnic_dev.h \
+ prov/usnic/src/usnic_direct/vnic_enet.h \
+ prov/usnic/src/usnic_direct/vnic_resource.h \
+ prov/usnic/src/usnic_direct/vnic_rq.c \
+ prov/usnic/src/usnic_direct/vnic_rq.h \
+ prov/usnic/src/usnic_direct/vnic_stats.h \
+ prov/usnic/src/usnic_direct/vnic_wq.c \
+ prov/usnic/src/usnic_direct/vnic_wq.h \
+ prov/usnic/src/usnic_direct/vnic_intr.c \
+ prov/usnic/src/usnic_direct/vnic_intr.h \
+ prov/usnic/src/usnic_direct/wq_enet_desc.h \
+ prov/usnic/src/fi_ext_usnic.h prov/usnic/src/usdf.h \
+ prov/usnic/src/usdf_av.c prov/usnic/src/usdf_av.h \
+ prov/usnic/src/usdf_cm.c prov/usnic/src/usdf_cm.h \
+ prov/usnic/src/usdf_cq.c prov/usnic/src/usdf_cq.h \
+ prov/usnic/src/usdf_dgram.c prov/usnic/src/usdf_dgram.h \
+ prov/usnic/src/usdf_domain.c prov/usnic/src/usdf_endpoint.c \
+ prov/usnic/src/usdf_endpoint.h prov/usnic/src/usdf_ep_dgram.c \
+ prov/usnic/src/usdf_ep_msg.c prov/usnic/src/usdf_ep_rdm.c \
+ prov/usnic/src/usdf_eq.c prov/usnic/src/usdf_fabric.c \
+ prov/usnic/src/usdf_mem.c prov/usnic/src/usdf_msg.c \
+ prov/usnic/src/usdf_msg.h prov/usnic/src/usdf_pep.c \
+ prov/usnic/src/usdf_progress.c prov/usnic/src/usdf_progress.h \
+ prov/usnic/src/usdf_rdm.c prov/usnic/src/usdf_rdm.h \
+ prov/usnic/src/usdf_rudp.h prov/usnic/src/usdf_timer.c \
+ prov/usnic/src/usdf_timer.h prov/usnic/src/usdf_poll.c \
+ prov/usnic/src/usdf_poll.h prov/usnic/src/usdf_ext.c \
+ prov/usnic/src/usdf_wait.h prov/usnic/src/usdf_wait.c \
+ prov/usnic/src/usdf_fake_ibv.c prov/mxm/src/mlxm.h \
+ prov/mxm/src/uthash.h prov/mxm/src/mpool.h \
+ prov/mxm/src/mlxm_mq_storage.h prov/mxm/src/mlxm_helpers.h \
+ prov/mxm/src/mlxm_init.c prov/mxm/src/mlxm_domain.c \
+ prov/mxm/src/mlxm_cq.c prov/mxm/src/mlxm_av.c \
+ prov/mxm/src/mlxm_ep.c prov/mxm/src/mlxm_cm.c \
+ prov/mxm/src/mlxm_tagged.c prov/mxm/src/mlxm_mr.c \
+ prov/gni/src/gnix_atomic.c prov/gni/src/gnix_av.c \
+ prov/gni/src/gnix_bitmap.c prov/gni/src/gnix_buddy_allocator.c \
+ prov/gni/src/gnix_cm.c prov/gni/src/gnix_cm_nic.c \
+ prov/gni/src/gnix_cntr.c prov/gni/src/gnix_cq.c \
+ prov/gni/src/gnix_datagram.c prov/gni/src/gnix_dom.c \
+ prov/gni/src/gnix_ep.c prov/gni/src/gnix_eq.c \
+ prov/gni/src/gnix_fabric.c prov/gni/src/gnix_freelist.c \
+ prov/gni/src/gnix_hashtable.c prov/gni/src/gnix_init.c \
+ prov/gni/src/gnix_mbox_allocator.c prov/gni/src/gnix_mr.c \
+ prov/gni/src/gnix_mr_cache.c prov/gni/src/gnix_mr_notifier.c \
+ prov/gni/src/gnix_msg.c prov/gni/src/gnix_nameserver.c \
+ prov/gni/src/gnix_nic.c prov/gni/src/gnix_poll.c \
+ prov/gni/src/gnix_queue.c prov/gni/src/gnix_rma.c \
+ prov/gni/src/gnix_tags.c prov/gni/src/gnix_trigger.c \
+ prov/gni/src/gnix_util.c prov/gni/src/gnix_vc.c \
+ prov/gni/src/gnix_vector.c prov/gni/src/gnix_xpmem.c \
+ prov/gni/src/gnix_wait.c prov/gni/include/fi_ext_gni.h \
+ prov/gni/include/gnix_atomic.h prov/gni/include/gnix_av.h \
+ prov/gni/include/gnix_bitmap.h \
+ prov/gni/include/gnix_buddy_allocator.h \
+ prov/gni/include/gnix_cm_nic.h prov/gni/include/gnix_cntr.h \
+ prov/gni/include/gnix_cq.h prov/gni/include/gnix_datagram.h \
+ prov/gni/include/gnix_ep.h prov/gni/include/gnix_eq.h \
+ prov/gni/include/gnix_freelist.h prov/gni/include/gnix.h \
+ prov/gni/include/gnix_hashtable.h \
+ prov/gni/include/gnix_mbox_allocator.h \
+ prov/gni/include/gnix_mr.h prov/gni/include/gnix_mr_cache.h \
+ prov/gni/include/gnix_mr_notifier.h \
+ prov/gni/include/gnix_msg.h prov/gni/include/gnix_nameserver.h \
+ prov/gni/include/gnix_nic.h prov/gni/include/gnix_poll.h \
+ prov/gni/include/gnix_priv.h prov/gni/include/gnix_queue.h \
+ prov/gni/include/gnix_rma.h prov/gni/include/gnix_tags.h \
+ prov/gni/include/gnix_trigger.h prov/gni/include/gnix_util.h \
+ prov/gni/include/gnix_vc.h prov/gni/include/gnix_vector.h \
+ prov/gni/include/gnix_xpmem.h prov/gni/include/gnix_wait.h \
+ prov/rxm/src/rxm_attr.c prov/rxm/src/rxm_init.c \
+ prov/rxm/src/rxm_fabric.c prov/rxm/src/rxm_domain.c \
+ prov/rxm/src/rxm_conn.c prov/rxm/src/rxm_ep.c \
+ prov/rxm/src/rxm_cq.c prov/rxm/src/rxm.h \
+ prov/rxd/src/rxd_attr.c prov/rxd/src/rxd_init.c \
+ prov/rxd/src/rxd_fabric.c prov/rxd/src/rxd_domain.c \
+ prov/rxd/src/rxd_av.c prov/rxd/src/rxd_cq.c \
+ prov/rxd/src/rxd_ep.c prov/rxd/src/rxd_rma.c \
+ prov/rxd/src/rxd.h
+ at MACOS_TRUE@am__objects_44 = src/osx/src_libfabric_la-osd.lo \
+ at MACOS_TRUE@ src/unix/src_libfabric_la-osd.lo
+ at FREEBSD_TRUE@am__objects_45 = src/unix/src_libfabric_la-osd.lo
+ at LINUX_TRUE@am__objects_46 = src/unix/src_libfabric_la-osd.lo
+am__objects_47 = src/src_libfabric_la-common.lo \
+ src/src_libfabric_la-enosys.lo src/src_libfabric_la-rbtree.lo \
+ src/src_libfabric_la-fasthash.lo \
+ src/src_libfabric_la-indexer.lo \
+ prov/util/src/src_libfabric_la-util_attr.lo \
+ prov/util/src/src_libfabric_la-util_av.lo \
+ prov/util/src/src_libfabric_la-util_cq.lo \
+ prov/util/src/src_libfabric_la-util_domain.lo \
+ prov/util/src/src_libfabric_la-util_ep.lo \
+ prov/util/src/src_libfabric_la-util_eq.lo \
+ prov/util/src/src_libfabric_la-util_fabric.lo \
+ prov/util/src/src_libfabric_la-util_main.lo \
+ prov/util/src/src_libfabric_la-util_poll.lo \
+ prov/util/src/src_libfabric_la-util_wait.lo \
+ prov/util/src/src_libfabric_la-util_buf.lo \
+ prov/util/src/src_libfabric_la-util_mr.lo $(am__objects_44) \
+ $(am__objects_45) $(am__objects_46)
+ at HAVE_SOCKETS_TRUE@am__objects_48 = prov/sockets/src/src_libfabric_la-sock_av.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_dom.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_eq.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_cq.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_cntr.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_poll.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_wait.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_ep_rdm.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_ep_dgram.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_ep_msg.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_fabric.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_ep.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_ctx.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_rx_entry.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_progress.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_comm.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_conn.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_msg.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_rma.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_atomic.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_trigger.lo \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/src_libfabric_la-sock_epoll.lo
+ at HAVE_SOCKETS_DL_FALSE@@HAVE_SOCKETS_TRUE at am__objects_49 = \
+ at HAVE_SOCKETS_DL_FALSE@@HAVE_SOCKETS_TRUE@ $(am__objects_48) \
+ at HAVE_SOCKETS_DL_FALSE@@HAVE_SOCKETS_TRUE@ $(am__objects_2)
+ at HAVE_UDP_TRUE@am__objects_50 = \
+ at HAVE_UDP_TRUE@ prov/udp/src/src_libfabric_la-udpx_attr.lo \
+ at HAVE_UDP_TRUE@ prov/udp/src/src_libfabric_la-udpx_cq.lo \
+ at HAVE_UDP_TRUE@ prov/udp/src/src_libfabric_la-udpx_domain.lo \
+ at HAVE_UDP_TRUE@ prov/udp/src/src_libfabric_la-udpx_ep.lo \
+ at HAVE_UDP_TRUE@ prov/udp/src/src_libfabric_la-udpx_fabric.lo \
+ at HAVE_UDP_TRUE@ prov/udp/src/src_libfabric_la-udpx_init.lo
+ at HAVE_UDP_DL_FALSE@@HAVE_UDP_TRUE at am__objects_51 = $(am__objects_50)
+ at HAVE_VERBS_TRUE@am__objects_52 = \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/src_libfabric_la-fi_verbs.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/src_libfabric_la-verbs_atomic.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/src_libfabric_la-verbs_av.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/src_libfabric_la-verbs_cm.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/src_libfabric_la-verbs_cq.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/src_libfabric_la-verbs_srq.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/src_libfabric_la-verbs_domain.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/src_libfabric_la-verbs_eq.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/src_libfabric_la-verbs_info.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/src_libfabric_la-verbs_msg.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/src_libfabric_la-verbs_msg_ep.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/src_libfabric_la-verbs_rma.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/src_libfabric_la-verbs_av_ep_rdm.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/src_libfabric_la-verbs_cq_ep_rdm.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/src_libfabric_la-verbs_ep_rdm.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cm.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cntr.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_msg.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm_states.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm.lo \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/src_libfabric_la-verbs_utils.lo
+ at HAVE_VERBS_DL_FALSE@@HAVE_VERBS_TRUE at am__objects_53 = \
+ at HAVE_VERBS_DL_FALSE@@HAVE_VERBS_TRUE@ $(am__objects_52)
+ at HAVE_USNIC_TRUE@am__objects_54 = prov/usnic/src/usnic_direct/src_libfabric_la-libnl_utils_common.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_caps.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_dest.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_device.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_event.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_enum.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_cmd.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_sysfs.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_mem.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_poll.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_post.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_raw.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_udp.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_pio_udp.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_queues.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_socket.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usd_vnic.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-usnic_ip_utils.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-vnic_cq.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-vnic_dev.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-vnic_rq.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-vnic_wq.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/src_libfabric_la-vnic_intr.lo
+ at HAVE_USNIC_TRUE@@HAVE_VERBS_TRUE at am__objects_55 = prov/usnic/src/src_libfabric_la-usdf_fake_ibv.lo
+ at HAVE_USNIC_TRUE@am__objects_56 = $(am__objects_54) \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_av.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_cm.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_cq.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_dgram.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_domain.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_endpoint.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_ep_dgram.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_ep_msg.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_ep_rdm.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_eq.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_fabric.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_mem.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_msg.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_pep.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_progress.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_rdm.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_timer.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_poll.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_ext.lo \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/src_libfabric_la-usdf_wait.lo \
+ at HAVE_USNIC_TRUE@ $(am__objects_55)
+ at HAVE_USNIC_DL_FALSE@@HAVE_USNIC_TRUE at am__objects_57 = \
+ at HAVE_USNIC_DL_FALSE@@HAVE_USNIC_TRUE@ $(am__objects_56)
+ at HAVE_MXM_TRUE@am__objects_58 = \
+ at HAVE_MXM_TRUE@ prov/mxm/src/src_libfabric_la-mlxm_init.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/src_libfabric_la-mlxm_domain.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/src_libfabric_la-mlxm_cq.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/src_libfabric_la-mlxm_av.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/src_libfabric_la-mlxm_ep.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/src_libfabric_la-mlxm_cm.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/src_libfabric_la-mlxm_tagged.lo \
+ at HAVE_MXM_TRUE@ prov/mxm/src/src_libfabric_la-mlxm_mr.lo
+ at HAVE_MXM_DL_FALSE@@HAVE_MXM_TRUE at am__objects_59 = $(am__objects_58)
+ at HAVE_GNI_TRUE@am__objects_60 = \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_atomic.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_av.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_bitmap.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_buddy_allocator.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_cm.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_cm_nic.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_cntr.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_cq.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_datagram.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_dom.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_ep.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_eq.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_fabric.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_freelist.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_hashtable.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_init.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_mbox_allocator.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_mr.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_mr_cache.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_mr_notifier.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_msg.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_nameserver.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_nic.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_poll.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_queue.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_rma.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_tags.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_trigger.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_util.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_vc.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_vector.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_xpmem.lo \
+ at HAVE_GNI_TRUE@ prov/gni/src/src_libfabric_la-gnix_wait.lo
+ at HAVE_GNI_DL_FALSE@@HAVE_GNI_TRUE at am__objects_61 = $(am__objects_60) \
+ at HAVE_GNI_DL_FALSE@@HAVE_GNI_TRUE@ $(am__objects_2)
+ at HAVE_RXM_TRUE@am__objects_62 = \
+ at HAVE_RXM_TRUE@ prov/rxm/src/src_libfabric_la-rxm_attr.lo \
+ at HAVE_RXM_TRUE@ prov/rxm/src/src_libfabric_la-rxm_init.lo \
+ at HAVE_RXM_TRUE@ prov/rxm/src/src_libfabric_la-rxm_fabric.lo \
+ at HAVE_RXM_TRUE@ prov/rxm/src/src_libfabric_la-rxm_domain.lo \
+ at HAVE_RXM_TRUE@ prov/rxm/src/src_libfabric_la-rxm_conn.lo \
+ at HAVE_RXM_TRUE@ prov/rxm/src/src_libfabric_la-rxm_ep.lo \
+ at HAVE_RXM_TRUE@ prov/rxm/src/src_libfabric_la-rxm_cq.lo
+ at HAVE_RXM_DL_FALSE@@HAVE_RXM_TRUE at am__objects_63 = $(am__objects_62)
+ at HAVE_RXD_TRUE@am__objects_64 = \
+ at HAVE_RXD_TRUE@ prov/rxd/src/src_libfabric_la-rxd_attr.lo \
+ at HAVE_RXD_TRUE@ prov/rxd/src/src_libfabric_la-rxd_init.lo \
+ at HAVE_RXD_TRUE@ prov/rxd/src/src_libfabric_la-rxd_fabric.lo \
+ at HAVE_RXD_TRUE@ prov/rxd/src/src_libfabric_la-rxd_domain.lo \
+ at HAVE_RXD_TRUE@ prov/rxd/src/src_libfabric_la-rxd_av.lo \
+ at HAVE_RXD_TRUE@ prov/rxd/src/src_libfabric_la-rxd_cq.lo \
+ at HAVE_RXD_TRUE@ prov/rxd/src/src_libfabric_la-rxd_ep.lo \
+ at HAVE_RXD_TRUE@ prov/rxd/src/src_libfabric_la-rxd_rma.lo
+ at HAVE_RXD_DL_FALSE@@HAVE_RXD_TRUE at am__objects_65 = $(am__objects_64)
+am_src_libfabric_la_OBJECTS = src/src_libfabric_la-fabric.lo \
+ src/src_libfabric_la-fi_tostr.lo src/src_libfabric_la-log.lo \
+ src/src_libfabric_la-var.lo $(am__objects_47) \
+ $(am__objects_49) $(am__objects_51) $(am__objects_53) \
+ $(am__objects_57) $(am__objects_59) $(am__objects_61) \
+ $(am__objects_63) $(am__objects_65)
+src_libfabric_la_OBJECTS = $(am_src_libfabric_la_OBJECTS)
+src_libfabric_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+ $(AM_CFLAGS) $(CFLAGS) $(src_libfabric_la_LDFLAGS) $(LDFLAGS) \
+ -o $@
+ at EMBEDDED_FALSE@am_src_libfabric_la_rpath = -rpath $(libdir)
+ at EMBEDDED_TRUE@am_src_libfabric_la_rpath =
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE at am__EXEEXT_1 = prov/gni/test/gnitest$(EXEEXT)
+PROGRAMS = $(bin_PROGRAMS)
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE at nodist_prov_gni_test_gnitest_OBJECTS = prov/gni/test/prov_gni_test_gnitest-allocator.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-av.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-bitmap.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-buddy_allocator.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-cancel.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-cntr.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-cq.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-datagram.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-dlist-utils.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-dom.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-ep.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-eq.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-freelist.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-hashtable.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-mr.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-mr_notifier.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-nic.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-pmi_utils.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-queue.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-rdm_atomic.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-rdm_dgram_rma.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-rdm_rx_overrun.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-rdm_sr.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-rdm_tagged_sr.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-tags.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-api.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-api_cq.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-api_cntr.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-utils.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-vc.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-vc_lookup.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-vector.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-wait.$(OBJEXT) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/prov_gni_test_gnitest-common.$(OBJEXT)
+prov_gni_test_gnitest_OBJECTS = \
+ $(nodist_prov_gni_test_gnitest_OBJECTS)
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE at prov_gni_test_gnitest_DEPENDENCIES = \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ $(am__DEPENDENCIES_1) \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ $(linkback)
+prov_gni_test_gnitest_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+ $(AM_CFLAGS) $(CFLAGS) $(prov_gni_test_gnitest_LDFLAGS) \
+ $(LDFLAGS) -o $@
+am_util_fi_info_OBJECTS = util/info.$(OBJEXT)
+util_fi_info_OBJECTS = $(am_util_fi_info_OBJECTS)
+util_fi_info_DEPENDENCIES = $(linkback)
+am_util_fi_pingpong_OBJECTS = util/pingpong.$(OBJEXT)
+util_fi_pingpong_OBJECTS = $(am_util_fi_pingpong_OBJECTS)
+util_fi_pingpong_DEPENDENCIES = $(linkback)
+am_util_fi_strerror_OBJECTS = util/strerror.$(OBJEXT)
+util_fi_strerror_OBJECTS = $(am_util_fi_strerror_OBJECTS)
+util_fi_strerror_DEPENDENCIES = $(linkback)
+SCRIPTS = $(bin_SCRIPTS)
+DEFAULT_INCLUDES = -I. at am__isrc@
+depcomp = $(SHELL) $(top_srcdir)/config/depcomp
+am__depfiles_maybe = depfiles
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+ $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+ $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+ $(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_$(V))
+am__v_CC_ = $(am__v_CC_$(AM_DEFAULT_VERBOSITY))
+am__v_CC_0 = @echo " CC " $@;
+AM_V_at = $(am__v_at_$(V))
+am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY))
+am__v_at_0 = @
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_$(V))
+am__v_CCLD_ = $(am__v_CCLD_$(AM_DEFAULT_VERBOSITY))
+am__v_CCLD_0 = @echo " CCLD " $@;
+AM_V_GEN = $(am__v_GEN_$(V))
+am__v_GEN_ = $(am__v_GEN_$(AM_DEFAULT_VERBOSITY))
+am__v_GEN_0 = @echo " GEN " $@;
+SOURCES = $(libgnix_fi_la_SOURCES) $(libmlxm_fi_la_SOURCES) \
+ $(libpsmx_fi_la_SOURCES) $(libpsmx_la_SOURCES) \
+ $(libpsmx2_fi_la_SOURCES) $(libpsmx2_la_SOURCES) \
+ $(librxd_fi_la_SOURCES) $(librxm_fi_la_SOURCES) \
+ $(libsockets_fi_la_SOURCES) $(libudp_fi_la_SOURCES) \
+ $(libusnic_fi_la_SOURCES) $(libverbs_fi_la_SOURCES) \
+ $(src_libfabric_la_SOURCES) \
+ $(nodist_prov_gni_test_gnitest_SOURCES) \
+ $(util_fi_info_SOURCES) $(util_fi_pingpong_SOURCES) \
+ $(util_fi_strerror_SOURCES)
+DIST_SOURCES = $(am__libgnix_fi_la_SOURCES_DIST) \
+ $(am__libmlxm_fi_la_SOURCES_DIST) \
+ $(am__libpsmx_fi_la_SOURCES_DIST) \
+ $(am__libpsmx_la_SOURCES_DIST) \
+ $(am__libpsmx2_fi_la_SOURCES_DIST) \
+ $(am__libpsmx2_la_SOURCES_DIST) \
+ $(am__librxd_fi_la_SOURCES_DIST) \
+ $(am__librxm_fi_la_SOURCES_DIST) \
+ $(am__libsockets_fi_la_SOURCES_DIST) \
+ $(am__libudp_fi_la_SOURCES_DIST) \
+ $(am__libusnic_fi_la_SOURCES_DIST) \
+ $(am__libverbs_fi_la_SOURCES_DIST) \
+ $(am__src_libfabric_la_SOURCES_DIST) $(util_fi_info_SOURCES) \
+ $(util_fi_pingpong_SOURCES) $(util_fi_strerror_SOURCES)
+man1dir = $(mandir)/man1
+man3dir = $(mandir)/man3
+man7dir = $(mandir)/man7
+NROFF = nroff
+MANS = $(man_MANS)
+DATA = $(pkgconfig_DATA)
+am__rdmainclude_HEADERS_DIST = $(top_srcdir)/include/rdma/fabric.h \
+ $(top_srcdir)/include/rdma/fi_atomic.h \
+ $(top_srcdir)/include/rdma/fi_cm.h \
+ $(top_srcdir)/include/rdma/fi_domain.h \
+ $(top_srcdir)/include/rdma/fi_eq.h \
+ $(top_srcdir)/include/rdma/fi_rma.h \
+ $(top_srcdir)/include/rdma/fi_endpoint.h \
+ $(top_srcdir)/include/rdma/fi_errno.h \
+ $(top_srcdir)/include/rdma/fi_tagged.h \
+ $(top_srcdir)/include/rdma/fi_trigger.h \
+ prov/usnic/src/fi_ext_usnic.h prov/gni/include/fi_ext_gni.h
+HEADERS = $(nodist_rdmainclude_HEADERS) $(rdmainclude_HEADERS)
+ETAGS = etags
+CTAGS = ctags
+am__tty_colors = \
+red=; grn=; lgn=; blu=; std=
+# Restructured Text title and section.
+am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
+am__rst_section = sed 'p;s/./=/g;p;g'
+# Put stdin (possibly several lines separated by ". ") in a box.
+am__text_box = $(AWK) '{ \
+ n = split($$0, lines, "\\. "); max = 0; \
+ for (i = 1; i <= n; ++i) \
+ if (max < length(lines[i])) \
+ max = length(lines[i]); \
+ for (i = 0; i < max; ++i) line = line "="; \
+ print line; \
+ for (i = 1; i <= n; ++i) if (lines[i]) print lines[i];\
+ print line; \
+}'
+# Solaris 10 'make', and several other traditional 'make' implementations,
+# pass "-e" to $(SHELL). This contradicts POSIX. Work around the problem
+# by disabling -e (using the XSI extension "set +e") if it's set.
+am__sh_e_setup = case $$- in *e*) set +e;; esac
+# To be inserted before the command running the test. Creates the
+# directory for the log if needed. Stores in $dir the directory
+# containing $f, in $tst the test, in $log the log, and passes
+# TESTS_ENVIRONMENT. Save and restore TERM around use of
+# TESTS_ENVIRONMENT, in case that unsets it.
+am__check_pre = \
+$(am__sh_e_setup); \
+$(am__vpath_adj_setup) $(am__vpath_adj) \
+srcdir=$(srcdir); export srcdir; \
+rm -f $@-t; \
+trap 'st=$$?; rm -f '\''$(abs_builddir)/$@-t'\''; (exit $$st); exit $$st' \
+ 1 2 13 15; \
+am__odir=`echo "./$@" | sed 's|/[^/]*$$||'`; \
+test "x$$am__odir" = x. || $(MKDIR_P) "$$am__odir" || exit $$?; \
+if test -f "./$$f"; then dir=./; \
+elif test -f "$$f"; then dir=; \
+else dir="$(srcdir)/"; fi; \
+tst=$$dir$$f; log='$@'; __SAVED_TERM=$$TERM; \
+$(TESTS_ENVIRONMENT)
+RECHECK_LOGS = $(TEST_LOGS)
+AM_RECURSIVE_TARGETS = check check-html recheck recheck-html
+TEST_SUITE_LOG = test-suite.log
+TEST_SUITE_HTML = $(TEST_SUITE_LOG:.log=.html)
+TEST_EXTENSIONS = @EXEEXT@ .test
+LOG_COMPILE = $(LOG_COMPILER) $(AM_LOG_FLAGS) $(LOG_FLAGS)
+am__test_logs1 = $(TESTS:=.log)
+am__test_logs2 = $(am__test_logs1:@EXEEXT at .log=.log)
+TEST_LOGS = $(am__test_logs2:.test.log=.log)
+TEST_LOG_COMPILE = $(TEST_LOG_COMPILER) $(AM_TEST_LOG_FLAGS) \
+ $(TEST_LOG_FLAGS)
+TEST_LOGS_TMP = $(TEST_LOGS:.log=.log-t)
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+distdir = $(PACKAGE)-$(VERSION)
+top_distdir = $(distdir)
+am__remove_distdir = \
+ { test ! -d "$(distdir)" \
+ || { find "$(distdir)" -type d ! -perm -200 -exec chmod u+w {} ';' \
+ && rm -fr "$(distdir)"; }; }
+DIST_ARCHIVES = $(distdir).tar.gz $(distdir).tar.bz2
+GZIP_ENV = --best
+distuninstallcheck_listfiles = find . -type f -print
+distcleancheck_listfiles = find . -type f -print
+ACLOCAL = @ACLOCAL@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+AR = @AR@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CFLAGS = @CFLAGS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRAY_ALPS_LLI_CFLAGS = @CRAY_ALPS_LLI_CFLAGS@
+CRAY_ALPS_LLI_LIBS = @CRAY_ALPS_LLI_LIBS@
+CRAY_ALPS_LLI_STATIC_LIBS = @CRAY_ALPS_LLI_STATIC_LIBS@
+CRAY_ALPS_UTIL_CFLAGS = @CRAY_ALPS_UTIL_CFLAGS@
+CRAY_ALPS_UTIL_LIBS = @CRAY_ALPS_UTIL_LIBS@
+CRAY_GNI_HEADERS_CFLAGS = @CRAY_GNI_HEADERS_CFLAGS@
+CRAY_GNI_HEADERS_LIBS = @CRAY_GNI_HEADERS_LIBS@
+CRAY_PMI_CFLAGS = @CRAY_PMI_CFLAGS@
+CRAY_PMI_LIBS = @CRAY_PMI_LIBS@
+CRAY_UDREG_CFLAGS = @CRAY_UDREG_CFLAGS@
+CRAY_UDREG_LIBS = @CRAY_UDREG_LIBS@
+CRAY_UGNI_CFLAGS = @CRAY_UGNI_CFLAGS@
+CRAY_UGNI_LIBS = @CRAY_UGNI_LIBS@
+CRAY_XPMEM_CFLAGS = @CRAY_XPMEM_CFLAGS@
+CRAY_XPMEM_LIBS = @CRAY_XPMEM_LIBS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FI_PC_CFLAGS = @FI_PC_CFLAGS@
+FI_PC_PRIVATE_LIBS = @FI_PC_PRIVATE_LIBS@
+FI_PC_PUBLIC_LIBS = @FI_PC_PUBLIC_LIBS@
+FI_PKG_CONFIG = @FI_PKG_CONFIG@
+FI_PKG_CONFIG_LIBDIR = @FI_PKG_CONFIG_LIBDIR@
+FI_PKG_CONFIG_PATH = @FI_PKG_CONFIG_PATH@
+GREP = @GREP@
+HAVE_LIBNL3 = @HAVE_LIBNL3@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+LD = @LD@
+LDFLAGS = @LDFLAGS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBTOOL = @LIBTOOL@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBOBJS = @LTLIBOBJS@
+MAKEINFO = @MAKEINFO@
+MKDIR_P = @MKDIR_P@
+NM = @NM@
+NMEDIT = @NMEDIT@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PROVIDERS_DL = @PROVIDERS_DL@
+PROVIDERS_STATIC = @PROVIDERS_STATIC@
+PROVIDERS_TO_BUILD = @PROVIDERS_TO_BUILD@
+PROVIDER_DIRECT = @PROVIDER_DIRECT@
+RANLIB = @RANLIB@
+SED = @SED@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+STRIP = @STRIP@
+VERSION = @VERSION@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+gni_CPPFLAGS = @gni_CPPFLAGS@
+gni_LDFLAGS = @gni_LDFLAGS@
+gnitest_CPPFLAGS = @gnitest_CPPFLAGS@
+gnitest_LDFLAGS = @gnitest_LDFLAGS@
+gnitest_LIBS = @gnitest_LIBS@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+lt_ECHO = @lt_ECHO@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+mxm_CPPFLAGS = @mxm_CPPFLAGS@
+mxm_LDFLAGS = @mxm_LDFLAGS@
+mxm_LIBS = @mxm_LIBS@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+psm2_CPPFLAGS = @psm2_CPPFLAGS@
+psm2_LDFLAGS = @psm2_LDFLAGS@
+psm2_LIBS = @psm2_LIBS@
+psm_CPPFLAGS = @psm_CPPFLAGS@
+psm_LDFLAGS = @psm_LDFLAGS@
+psm_LIBS = @psm_LIBS@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sockets_CPPFLAGS = @sockets_CPPFLAGS@
+sockets_LDFLAGS = @sockets_LDFLAGS@
+sockets_LIBS = @sockets_LIBS@
+srcdir = @srcdir@
+sysconfdir = @sysconfdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+udp_shm_CPPFLAGS = @udp_shm_CPPFLAGS@
+udp_shm_LDFLAGS = @udp_shm_LDFLAGS@
+udp_shm_LIBS = @udp_shm_LIBS@
+usnic_CPPFLAGS = @usnic_CPPFLAGS@
+usnic_LDFLAGS = @usnic_LDFLAGS@
+usnic_LIBS = @usnic_LIBS@
+usnic_nl_CPPFLAGS = @usnic_nl_CPPFLAGS@
+usnic_nl_LDFLAGS = @usnic_nl_LDFLAGS@
+usnic_nl_LIBS = @usnic_nl_LIBS@
+verbs_CPPFLAGS = @verbs_CPPFLAGS@
+verbs_LDFLAGS = @verbs_LDFLAGS@
+verbs_LIBS = @verbs_LIBS@
+verbs_ibverbs_CPPFLAGS = @verbs_ibverbs_CPPFLAGS@
+verbs_ibverbs_LDFLAGS = @verbs_ibverbs_LDFLAGS@
+verbs_ibverbs_LIBS = @verbs_ibverbs_LIBS@
+verbs_rdmacm_CPPFLAGS = @verbs_rdmacm_CPPFLAGS@
+verbs_rdmacm_LDFLAGS = @verbs_rdmacm_LDFLAGS@
+verbs_rdmacm_LIBS = @verbs_rdmacm_LIBS@
+AM_CPPFLAGS = -I$(srcdir)/include -D_GNU_SOURCE \
+ -DSYSCONFDIR=\"$(sysconfdir)\" -DRDMADIR=\"@rdmadir@\" \
+ -DPROVDLDIR=\"$(pkglibdir)\" $(am__append_6) $(am__append_45)
+noinst_LTLIBRARIES = $(am__append_1) $(am__append_30) $(am__append_35)
+lib_LTLIBRARIES = $(am__append_2)
+pkglib_LTLIBRARIES = $(DL_PROVIDERS) $(am__append_7) $(am__append_11) \
+ $(am__append_15) $(am__append_23) $(am__append_29) \
+ $(am__append_34) $(am__append_39) $(am__append_48) \
+ $(am__append_55) $(am__append_58)
+ACLOCAL_AMFLAGS = -I config
+AM_CFLAGS = -g -Wall
+ at HAVE_LD_VERSION_SCRIPT_FALSE@libfabric_version_script =
+ at HAVE_LD_VERSION_SCRIPT_TRUE@libfabric_version_script = -Wl,--version-script=$(builddir)/libfabric.map
+rdmaincludedir = $(includedir)/rdma
+rdmainclude_HEADERS = $(top_srcdir)/include/rdma/fabric.h \
+ $(top_srcdir)/include/rdma/fi_atomic.h \
+ $(top_srcdir)/include/rdma/fi_cm.h \
+ $(top_srcdir)/include/rdma/fi_domain.h \
+ $(top_srcdir)/include/rdma/fi_eq.h \
+ $(top_srcdir)/include/rdma/fi_rma.h \
+ $(top_srcdir)/include/rdma/fi_endpoint.h \
+ $(top_srcdir)/include/rdma/fi_errno.h \
+ $(top_srcdir)/include/rdma/fi_tagged.h \
+ $(top_srcdir)/include/rdma/fi_trigger.h $(am__append_22) \
+ $(am__append_53)
+
+# internal utility functions shared by in-tree providers:
+common_srcs = src/common.c src/enosys.c src/rbtree.c src/fasthash.c \
+ src/indexer.c prov/util/src/util_attr.c \
+ prov/util/src/util_av.c prov/util/src/util_cq.c \
+ prov/util/src/util_domain.c prov/util/src/util_ep.c \
+ prov/util/src/util_eq.c prov/util/src/util_fabric.c \
+ prov/util/src/util_main.c prov/util/src/util_poll.c \
+ prov/util/src/util_wait.c prov/util/src/util_buf.c \
+ prov/util/src/util_mr.c $(am__append_3) $(am__append_4) \
+ $(am__append_5)
+
+# ensure dl-built providers link back to libfabric
+linkback = src/libfabric.la
+bin_SCRIPTS = $(am__append_47)
+util_fi_info_SOURCES = \
+ util/info.c
+
+util_fi_info_LDADD = $(linkback)
+util_fi_strerror_SOURCES = \
+ util/strerror.c
+
+util_fi_strerror_LDADD = $(linkback)
+util_fi_pingpong_SOURCES = \
+ util/pingpong.c
+
+util_fi_pingpong_LDADD = $(linkback)
+src_libfabric_la_SOURCES = include/fi.h include/fi_abi.h \
+ include/fi_atom.h include/fi_enosys.h include/fi_file.h \
+ include/fi_indexer.h include/fi_list.h include/fi_lock.h \
+ include/fi_mem.h include/fi_osd.h include/fi_proto.h \
+ include/fi_rbuf.h include/fi_signal.h include/fi_util.h \
+ include/fasthash.h include/rbtree.h include/prov.h \
+ include/rdma/providers/fi_log.h \
+ include/rdma/providers/fi_prov.h src/fabric.c src/fi_tostr.c \
+ src/log.c src/var.c $(common_srcs) $(am__append_8) \
+ $(am__append_12) $(am__append_16) $(am__append_24) \
+ $(am__append_40) $(am__append_49) $(am__append_56) \
+ $(am__append_59)
+src_libfabric_la_CPPFLAGS = $(AM_CPPFLAGS) $(am__append_17) \
+ $(am__append_25) $(am__append_41) $(am__append_50)
+src_libfabric_la_LDFLAGS = -version-info 3:1:2 -export-dynamic \
+ $(libfabric_version_script) $(am__append_18) $(am__append_26) \
+ $(am__append_42) $(am__append_51)
+src_libfabric_la_LIBADD = $(am__append_9) $(am__append_13) \
+ $(am__append_19) $(am__append_27) $(am__append_31) \
+ $(am__append_36) $(am__append_43) $(am__append_52) \
+ $(am__append_57) $(am__append_60)
+src_libfabric_la_DEPENDENCIES = libfabric.map $(am__append_32) \
+ $(am__append_37)
+ at HAVE_DIRECT_TRUE@nodist_rdmainclude_HEADERS = \
+ at HAVE_DIRECT_TRUE@ $(top_srcdir)/prov/$(PROVIDER_DIRECT)/include/rdma/fi_direct.h \
+ at HAVE_DIRECT_TRUE@ $(top_srcdir)/prov/$(PROVIDER_DIRECT)/include/rdma/fi_direct_domain.h \
+ at HAVE_DIRECT_TRUE@ $(top_srcdir)/prov/$(PROVIDER_DIRECT)/include/rdma/fi_direct_endpoint.h \
+ at HAVE_DIRECT_TRUE@ $(top_srcdir)/prov/$(PROVIDER_DIRECT)/include/rdma/fi_direct_tagged.h \
+ at HAVE_DIRECT_TRUE@ $(top_srcdir)/prov/$(PROVIDER_DIRECT)/include/rdma/fi_direct_rma.h \
+ at HAVE_DIRECT_TRUE@ $(top_srcdir)/prov/$(PROVIDER_DIRECT)/include/rdma/fi_direct_atomic_def.h \
+ at HAVE_DIRECT_TRUE@ $(top_srcdir)/prov/$(PROVIDER_DIRECT)/include/rdma/fi_direct_atomic.h \
+ at HAVE_DIRECT_TRUE@ $(top_srcdir)/prov/$(PROVIDER_DIRECT)/include/rdma/fi_direct_cm.h \
+ at HAVE_DIRECT_TRUE@ $(top_srcdir)/prov/$(PROVIDER_DIRECT)/include/rdma/fi_direct_eq.h \
+ at HAVE_DIRECT_TRUE@ $(top_srcdir)/prov/$(PROVIDER_DIRECT)/include/rdma/fi_direct_trigger.h
+
+real_man_pages = \
+ man/man1/fi_info.1 \
+ man/man1/fi_pingpong.1 \
+ man/man1/fi_strerror.1 \
+ man/man3/fi_av.3 \
+ man/man3/fi_cm.3 \
+ man/man3/fi_cntr.3 \
+ man/man3/fi_control.3 \
+ man/man3/fi_cq.3 \
+ man/man3/fi_domain.3 \
+ man/man3/fi_endpoint.3 \
+ man/man3/fi_errno.3 \
+ man/man3/fi_eq.3 \
+ man/man3/fi_fabric.3 \
+ man/man3/fi_getinfo.3 \
+ man/man3/fi_mr.3 \
+ man/man3/fi_msg.3 \
+ man/man3/fi_poll.3 \
+ man/man3/fi_rma.3 \
+ man/man3/fi_tagged.3 \
+ man/man3/fi_trigger.3 \
+ man/man3/fi_version.3 \
+ man/man7/fabric.7 \
+ man/man7/fi_provider.7 \
+ man/man7/fi_direct.7
+
+dummy_man_pages = \
+ man/man3/fi_accept.3 \
+ man/man3/fi_alias.3 \
+ man/man3/fi_atomic.3 \
+ man/man3/fi_atomic_valid.3 \
+ man/man3/fi_atomicmsg.3 \
+ man/man3/fi_atomicv.3 \
+ man/man3/fi_av_bind.3 \
+ man/man3/fi_av_insert.3 \
+ man/man3/fi_av_insertsvc.3 \
+ man/man3/fi_av_lookup.3 \
+ man/man3/fi_av_open.3 \
+ man/man3/fi_av_remove.3 \
+ man/man3/fi_av_straddr.3 \
+ man/man3/fi_cancel.3 \
+ man/man3/fi_close.3 \
+ man/man3/fi_cntr_add.3 \
+ man/man3/fi_cntr_open.3 \
+ man/man3/fi_cntr_read.3 \
+ man/man3/fi_cntr_set.3 \
+ man/man3/fi_cntr_wait.3 \
+ man/man3/fi_compare_atomic.3 \
+ man/man3/fi_compare_atomic_valid.3 \
+ man/man3/fi_compare_atomicmsg.3 \
+ man/man3/fi_compare_atomicv.3 \
+ man/man3/fi_connect.3 \
+ man/man3/fi_cq_open.3 \
+ man/man3/fi_cq_read.3 \
+ man/man3/fi_cq_readerr.3 \
+ man/man3/fi_cq_readfrom.3 \
+ man/man3/fi_cq_sread.3 \
+ man/man3/fi_cq_sreadfrom.3 \
+ man/man3/fi_cq_strerror.3 \
+ man/man3/fi_cq_signal.3 \
+ man/man3/fi_domain_bind.3 \
+ man/man3/fi_domain_query.3 \
+ man/man3/fi_dupinfo.3 \
+ man/man3/fi_enable.3 \
+ man/man3/fi_ep.3 \
+ man/man3/fi_ep_bind.3 \
+ man/man3/fi_eq_open.3 \
+ man/man3/fi_eq_read.3 \
+ man/man3/fi_eq_readerr.3 \
+ man/man3/fi_eq_sread.3 \
+ man/man3/fi_eq_strerror.3 \
+ man/man3/fi_eq_write.3 \
+ man/man3/fi_fetch_atomic.3 \
+ man/man3/fi_fetch_atomic_valid.3 \
+ man/man3/fi_fetch_atomicmsg.3 \
+ man/man3/fi_fetch_atomicv.3 \
+ man/man3/fi_freeinfo.3 \
+ man/man3/fi_getname.3 \
+ man/man3/fi_getopt.3 \
+ man/man3/fi_getpeer.3 \
+ man/man3/fi_inject.3 \
+ man/man3/fi_injectdata.3 \
+ man/man3/fi_inject_atomic.3 \
+ man/man3/fi_inject_write.3 \
+ man/man3/fi_inject_writedata.3 \
+ man/man3/fi_join.3 \
+ man/man3/fi_leave.3 \
+ man/man3/fi_listen.3 \
+ man/man3/fi_mr_bind.3 \
+ man/man3/fi_mr_desc.3 \
+ man/man3/fi_mr_key.3 \
+ man/man3/fi_mr_reg.3 \
+ man/man3/fi_mr_regattr.3 \
+ man/man3/fi_mr_regv.3 \
+ man/man3/fi_open.3 \
+ man/man3/fi_open_ops.3 \
+ man/man3/fi_passive_ep.3 \
+ man/man3/fi_pep_bind.3 \
+ man/man3/fi_poll_add.3 \
+ man/man3/fi_poll_del.3 \
+ man/man3/fi_poll_open.3 \
+ man/man3/fi_read.3 \
+ man/man3/fi_readmsg.3 \
+ man/man3/fi_readv.3 \
+ man/man3/fi_recv.3 \
+ man/man3/fi_recvmsg.3 \
+ man/man3/fi_recvv.3 \
+ man/man3/fi_reject.3 \
+ man/man3/fi_rx_addr.3 \
+ man/man3/fi_rx_size_left.3 \
+ man/man3/fi_scalable_ep_bind.3 \
+ man/man3/fi_send.3 \
+ man/man3/fi_senddata.3 \
+ man/man3/fi_sendmsg.3 \
+ man/man3/fi_sendv.3 \
+ man/man3/fi_setname.3 \
+ man/man3/fi_setopt.3 \
+ man/man3/fi_shutdown.3 \
+ man/man3/fi_strerror.3 \
+ man/man3/fi_tinject.3 \
+ man/man3/fi_tinjectdata.3 \
+ man/man3/fi_tostr.3 \
+ man/man3/fi_trecv.3 \
+ man/man3/fi_trecvmsg.3 \
+ man/man3/fi_trecvv.3 \
+ man/man3/fi_trywait.3 \
+ man/man3/fi_tsend.3 \
+ man/man3/fi_tsenddata.3 \
+ man/man3/fi_tsendmsg.3 \
+ man/man3/fi_tsendv.3 \
+ man/man3/fi_tx_size_left.3 \
+ man/man3/fi_wait.3 \
+ man/man3/fi_wait_open.3 \
+ man/man3/fi_write.3 \
+ man/man3/fi_writedata.3 \
+ man/man3/fi_writemsg.3 \
+ man/man3/fi_writev.3
+
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = libfabric.pc
+prov_install_man_pages = $(am__append_10) $(am__append_14) \
+ $(am__append_20) $(am__append_28) $(am__append_33) \
+ $(am__append_38) $(am__append_44) $(am__append_54)
+prov_dist_man_pages = man/man7/fi_sockets.7 man/man7/fi_udp.7 \
+ man/man7/fi_verbs.7 man/man7/fi_usnic.7 man/man7/fi_psm.7 \
+ man/man7/fi_psm2.7 man/man7/fi_mxm.7 man/man7/fi_gni.7
+ at HAVE_SOCKETS_TRUE@_sockets_files = \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_av.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_dom.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_eq.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_cq.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_cntr.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_poll.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_wait.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_ep_rdm.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_ep_dgram.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_ep_msg.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_fabric.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_ep.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_ctx.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_rx_entry.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_progress.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_comm.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_conn.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_msg.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_rma.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_atomic.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_trigger.c \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/src/sock_epoll.c
+
+ at HAVE_SOCKETS_TRUE@_sockets_headers = \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/include/sock.h \
+ at HAVE_SOCKETS_TRUE@ prov/sockets/include/sock_util.h
+
+ at HAVE_SOCKETS_DL_TRUE@@HAVE_SOCKETS_TRUE at libsockets_fi_la_SOURCES = $(_sockets_files) $(_sockets_headers) $(common_srcs)
+ at HAVE_SOCKETS_DL_TRUE@@HAVE_SOCKETS_TRUE at libsockets_fi_la_LIBADD = $(linkback) $(sockets_LIBS)
+ at HAVE_SOCKETS_DL_TRUE@@HAVE_SOCKETS_TRUE at libsockets_fi_la_LDFLAGS = -module -avoid-version -shared -export-dynamic
+ at HAVE_SOCKETS_DL_TRUE@@HAVE_SOCKETS_TRUE at libsockets_fi_la_DEPENDENCIES = $(linkback)
+ at HAVE_UDP_TRUE@_udp_files = \
+ at HAVE_UDP_TRUE@ prov/udp/src/udpx_attr.c \
+ at HAVE_UDP_TRUE@ prov/udp/src/udpx_cq.c \
+ at HAVE_UDP_TRUE@ prov/udp/src/udpx_domain.c \
+ at HAVE_UDP_TRUE@ prov/udp/src/udpx_ep.c \
+ at HAVE_UDP_TRUE@ prov/udp/src/udpx_fabric.c \
+ at HAVE_UDP_TRUE@ prov/udp/src/udpx_init.c \
+ at HAVE_UDP_TRUE@ prov/udp/src/udpx.h
+
+ at HAVE_UDP_DL_TRUE@@HAVE_UDP_TRUE at libudp_fi_la_SOURCES = $(_udp_files) $(common_srcs)
+ at HAVE_UDP_DL_TRUE@@HAVE_UDP_TRUE at libudp_fi_la_LIBADD = $(linkback) $(udp_shm_LIBS)
+ at HAVE_UDP_DL_TRUE@@HAVE_UDP_TRUE at libudp_fi_la_LDFLAGS = -module -avoid-version -shared -export-dynamic
+ at HAVE_UDP_DL_TRUE@@HAVE_UDP_TRUE at libudp_fi_la_DEPENDENCIES = $(linkback)
+ at HAVE_VERBS_TRUE@_verbs_files = \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/fi_verbs.h \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/fi_verbs.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/uthash.h \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/verbs_atomic.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/verbs_av.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/verbs_cm.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/verbs_cq.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/verbs_srq.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/verbs_domain.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/verbs_eq.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/verbs_info.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/verbs_msg.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/verbs_msg_ep.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/verbs_rma.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/verbs_ep_rdm.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/verbs_queuing.h \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/verbs_rdm_cm.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/verbs_rdm_cntr.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/verbs_rdm_msg.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/verbs_rdm.h \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.h \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/verbs_utils.c \
+ at HAVE_VERBS_TRUE@ prov/verbs/src/ep_rdm/verbs_utils.h
+
+ at HAVE_VERBS_DL_TRUE@@HAVE_VERBS_TRUE at libverbs_fi_la_SOURCES = $(_verbs_files) $(common_srcs)
+ at HAVE_VERBS_DL_TRUE@@HAVE_VERBS_TRUE at libverbs_fi_la_CPPFLAGS = $(AM_CPPFLAGS) $(verbs_CPPFLAGS)
+ at HAVE_VERBS_DL_TRUE@@HAVE_VERBS_TRUE at libverbs_fi_la_LDFLAGS = \
+ at HAVE_VERBS_DL_TRUE@@HAVE_VERBS_TRUE@ -module -avoid-version -shared -export-dynamic $(verbs_LDFLAGS)
+
+ at HAVE_VERBS_DL_TRUE@@HAVE_VERBS_TRUE at libverbs_fi_la_LIBADD = $(linkback) $(verbs_LIBS)
+ at HAVE_VERBS_DL_TRUE@@HAVE_VERBS_TRUE at libverbs_fi_la_DEPENDENCIES = $(linkback)
+ at HAVE_USNIC_TRUE@libusnic_direct_sources = \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/cq_desc.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/cq_enet_desc.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/kcompat.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/kcompat_priv.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libnl1_utils.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libnl3_utils.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libnl_utils_common.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/libnl_utils.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/linux/delay.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/linux/slab.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/linux_types.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/rq_enet_desc.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_caps.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_caps.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_dest.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_dest.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_device.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_device.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_event.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_enum.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_ib_cmd.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_ib_cmd.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_ib_sysfs.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_ib_sysfs.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_mem.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_poll.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_post.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_post.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_post_ud_raw.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_post_ud_udp.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_post_ud_pio_udp.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_queue.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_queues.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_socket.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_socket.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_time.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_util.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_vnic.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usd_vnic.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usnic_abi.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usnic_direct.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usnic_ib_abi.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usnic_ip_utils.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usnic_ip_utils.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/usnic_user_utils.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_cq.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_cq.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_dev.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_devcmd.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_dev.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_enet.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_resource.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_rq.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_rq.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_stats.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_wq.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_wq.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_intr.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/vnic_intr.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usnic_direct/wq_enet_desc.h
+
+ at HAVE_USNIC_TRUE@_usnic_files = $(libusnic_direct_sources) \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/fi_ext_usnic.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_av.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_av.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_cm.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_cm.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_cq.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_cq.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_dgram.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_dgram.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_domain.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_endpoint.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_endpoint.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_ep_dgram.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_ep_msg.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_ep_rdm.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_eq.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_fabric.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_mem.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_msg.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_msg.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_pep.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_progress.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_progress.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_rdm.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_rdm.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_rudp.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_timer.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_timer.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_poll.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_poll.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_ext.c \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_wait.h \
+ at HAVE_USNIC_TRUE@ prov/usnic/src/usdf_wait.c $(am__append_21)
+ at HAVE_USNIC_TRUE@_usnic_cppflags = \
+ at HAVE_USNIC_TRUE@ -D__LIBUSNIC__ -DWANT_DEBUG_MSGS=0 \
+ at HAVE_USNIC_TRUE@ -DHAVE_LIBNL3=$(HAVE_LIBNL3) $(usnic_CPPFLAGS) \
+ at HAVE_USNIC_TRUE@ -I$(top_srcdir)/prov/usnic/src/usnic_direct
+
+ at HAVE_USNIC_DL_TRUE@@HAVE_USNIC_TRUE at libusnic_fi_la_CPPFLAGS = $(AM_CPPFLAGS) $(_usnic_cppflags)
+ at HAVE_USNIC_DL_TRUE@@HAVE_USNIC_TRUE at libusnic_fi_la_SOURCES = $(_usnic_files) $(common_srcs)
+ at HAVE_USNIC_DL_TRUE@@HAVE_USNIC_TRUE at libusnic_fi_la_LDFLAGS = \
+ at HAVE_USNIC_DL_TRUE@@HAVE_USNIC_TRUE@ $(usnic_ln_LDFLAGS) \
+ at HAVE_USNIC_DL_TRUE@@HAVE_USNIC_TRUE@ -module -avoid-version -shared -export-dynamic
+
+ at HAVE_USNIC_DL_TRUE@@HAVE_USNIC_TRUE at libusnic_fi_la_LIBADD = $(linkback) $(usnic_LIBS)
+ at HAVE_USNIC_DL_TRUE@@HAVE_USNIC_TRUE at libusnic_fi_la_DEPENDENCIES = $(linkback)
+ at HAVE_PSM_TRUE@_psm_files = \
+ at HAVE_PSM_TRUE@ prov/psm/src/version.h \
+ at HAVE_PSM_TRUE@ prov/psm/src/psm_am.h \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx.h \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_init.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_fabric.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_domain.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_cq.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_cntr.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_av.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_ep.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_cm.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_tagged.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_msg.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_msg2.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_rma.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_atomic.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_am.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_mr.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_wait.c \
+ at HAVE_PSM_TRUE@ prov/psm/src/psmx_util.c
+
+ at HAVE_PSM_DL_TRUE@@HAVE_PSM_TRUE at libpsmx_fi_la_SOURCES = $(_psm_files) $(common_srcs)
+ at HAVE_PSM_DL_TRUE@@HAVE_PSM_TRUE at libpsmx_fi_la_CPPFLAGS = $(AM_CPPFLAGS) $(psm_CPPFLAGS)
+ at HAVE_PSM_DL_TRUE@@HAVE_PSM_TRUE at libpsmx_fi_la_LDFLAGS = \
+ at HAVE_PSM_DL_TRUE@@HAVE_PSM_TRUE@ -module -avoid-version -shared -export-dynamic $(psm_LDFLAGS)
+
+ at HAVE_PSM_DL_TRUE@@HAVE_PSM_TRUE at libpsmx_fi_la_LIBADD = $(linkback) $(psm_LIBS)
+ at HAVE_PSM_DL_TRUE@@HAVE_PSM_TRUE at libpsmx_fi_la_DEPENDENCIES = $(linkback)
+ at HAVE_PSM_DL_FALSE@@HAVE_PSM_TRUE at libpsmx_la_SOURCES = $(_psm_files)
+ at HAVE_PSM_DL_FALSE@@HAVE_PSM_TRUE at libpsmx_la_CPPFLAGS = $(src_libfabric_la_CPPFLAGS) $(psm_CPPFLAGS)
+ at HAVE_PSM_DL_FALSE@@HAVE_PSM_TRUE at libpsmx_la_LDFLAGS = $(psm_LDFLAGS)
+ at HAVE_PSM_DL_FALSE@@HAVE_PSM_TRUE at libpsmx_la_LIBADD = $(psm_LIBS)
+ at HAVE_PSM2_TRUE@_psm2_files = \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/version.h \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2.h \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_init.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_fabric.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_domain.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_cq.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_cntr.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_av.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_ep.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_cm.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_tagged.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_msg.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_rma.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_atomic.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_am.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_mr.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_wait.c \
+ at HAVE_PSM2_TRUE@ prov/psm2/src/psmx2_util.c
+
+ at HAVE_PSM2_DL_TRUE@@HAVE_PSM2_TRUE at libpsmx2_fi_la_SOURCES = $(_psm2_files) $(common_srcs)
+ at HAVE_PSM2_DL_TRUE@@HAVE_PSM2_TRUE at libpsmx2_fi_la_CPPFLAGS = $(AM_CPPFLAGS) $(psm2_CPPFLAGS)
+ at HAVE_PSM2_DL_TRUE@@HAVE_PSM2_TRUE at libpsmx2_fi_la_LDFLAGS = \
+ at HAVE_PSM2_DL_TRUE@@HAVE_PSM2_TRUE@ -module -avoid-version -shared -export-dynamic $(psm2_LDFLAGS)
+
+ at HAVE_PSM2_DL_TRUE@@HAVE_PSM2_TRUE at libpsmx2_fi_la_LIBADD = $(linkback) $(psm2_LIBS)
+ at HAVE_PSM2_DL_TRUE@@HAVE_PSM2_TRUE at libpsmx2_fi_la_DEPENDENCIES = $(linkback)
+ at HAVE_PSM2_DL_FALSE@@HAVE_PSM2_TRUE at libpsmx2_la_SOURCES = $(_psm2_files)
+ at HAVE_PSM2_DL_FALSE@@HAVE_PSM2_TRUE at libpsmx2_la_CPPFLAGS = $(src_libfabric_la_CPPFLAGS) $(psm2_CPPFLAGS)
+ at HAVE_PSM2_DL_FALSE@@HAVE_PSM2_TRUE at libpsmx2_la_LDFLAGS = $(psm2_LDFLAGS)
+ at HAVE_PSM2_DL_FALSE@@HAVE_PSM2_TRUE at libpsmx2_la_LIBADD = $(psm2_LIBS)
+ at HAVE_MXM_TRUE@_mxm_files = prov/mxm/src/mlxm.h \
+ at HAVE_MXM_TRUE@ prov/mxm/src/uthash.h \
+ at HAVE_MXM_TRUE@ prov/mxm/src/mpool.h \
+ at HAVE_MXM_TRUE@ prov/mxm/src/mlxm_mq_storage.h \
+ at HAVE_MXM_TRUE@ prov/mxm/src/mlxm_helpers.h \
+ at HAVE_MXM_TRUE@ prov/mxm/src/mlxm_init.c \
+ at HAVE_MXM_TRUE@ prov/mxm/src/mlxm_domain.c \
+ at HAVE_MXM_TRUE@ prov/mxm/src/mlxm_cq.c \
+ at HAVE_MXM_TRUE@ prov/mxm/src/mlxm_av.c \
+ at HAVE_MXM_TRUE@ prov/mxm/src/mlxm_ep.c \
+ at HAVE_MXM_TRUE@ prov/mxm/src/mlxm_cm.c \
+ at HAVE_MXM_TRUE@ prov/mxm/src/mlxm_tagged.c \
+ at HAVE_MXM_TRUE@ prov/mxm/src/mlxm_mr.c
+
+ at HAVE_MXM_DL_TRUE@@HAVE_MXM_TRUE at libmlxm_fi_la_CPPFLAGS = $(AM_CPPFLAGS) $(mxm_CPPFLAGS)
+ at HAVE_MXM_DL_TRUE@@HAVE_MXM_TRUE at libmlxm_fi_la_SOURCES = $(_mxm_files) $(common_srcs)
+ at HAVE_MXM_DL_TRUE@@HAVE_MXM_TRUE at libmlxm_fi_la_LDFLAGS = \
+ at HAVE_MXM_DL_TRUE@@HAVE_MXM_TRUE@ $(mxm_LDFLAGS) \
+ at HAVE_MXM_DL_TRUE@@HAVE_MXM_TRUE@ -module -avoid-version -shared -export-dynamic
+
+ at HAVE_MXM_DL_TRUE@@HAVE_MXM_TRUE at libmlxm_fi_la_LIBADD = $(linkback) $(mxm_LIBS)
+ at HAVE_MXM_DL_TRUE@@HAVE_MXM_TRUE at libmlxm_fi_la_DEPENDENCIES = $(linkback)
+ at HAVE_GNI_TRUE@_gni_files = \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_atomic.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_av.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_bitmap.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_buddy_allocator.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_cm.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_cm_nic.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_cntr.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_cq.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_datagram.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_dom.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_ep.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_eq.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_fabric.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_freelist.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_hashtable.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_init.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_mbox_allocator.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_mr.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_mr_cache.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_mr_notifier.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_msg.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_nameserver.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_nic.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_poll.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_queue.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_rma.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_tags.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_trigger.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_util.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_vc.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_vector.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_xpmem.c \
+ at HAVE_GNI_TRUE@ prov/gni/src/gnix_wait.c
+
+ at HAVE_GNI_TRUE@_gni_headers = \
+ at HAVE_GNI_TRUE@ prov/gni/include/fi_ext_gni.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_atomic.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_av.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_bitmap.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_buddy_allocator.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_cm_nic.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_cntr.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_cq.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_datagram.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_ep.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_eq.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_freelist.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_hashtable.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_mbox_allocator.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_mr.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_mr_cache.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_mr_notifier.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_msg.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_nameserver.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_nic.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_poll.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_priv.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_queue.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_rma.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_tags.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_trigger.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_util.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_vc.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_vector.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_xpmem.h \
+ at HAVE_GNI_TRUE@ prov/gni/include/gnix_wait.h
+
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE at nodist_prov_gni_test_gnitest_SOURCES = \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/allocator.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/av.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/bitmap.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/buddy_allocator.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/cancel.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/cntr.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/cq.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/datagram.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/dlist-utils.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/dom.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/ep.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/eq.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/freelist.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/hashtable.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/mr.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/mr_notifier.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/nic.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/pmi_utils.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/queue.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/rdm_atomic.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/rdm_fi_pcd_trecv_msg.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/rdm_dgram_rma.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/rdm_rx_overrun.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/rdm_sr.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/rdm_tagged_sr.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/tags.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/api.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/api_cq.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/api_cntr.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/utils.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/vc.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/vc_lookup.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/vector.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/wait.c \
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE@ prov/gni/test/common.c
+
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE at prov_gni_test_gnitest_LDFLAGS = $(CRAY_PMI_LIBS) $(gnitest_LDFLAGS) -static
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE at prov_gni_test_gnitest_CPPFLAGS = $(AM_CPPFLAGS) $(CRAY_PMI_CFLAGS) $(CRAY_XPMEM_CFLAGS) $(gnitest_CPPFLAGS)
+ at HAVE_CRITERION_TRUE@@HAVE_GNI_TRUE at prov_gni_test_gnitest_LDADD = $(gnitest_LIBS) $(linkback)
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE at libgnix_fi_la_CPPFLAGS = $(AM_CPPFLAGS) $(gni_CPPFLAGS)
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE at libgnix_fi_la_SOURCES = $(_gni_files) $(_gni_headers) $(common_srcs)
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE at libgnix_fi_la_LDFLAGS = \
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE@ $(gni_LDFLAGS) \
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE@ -module -avoid-version -shared -export-dynamic
+
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE at libgnix_fi_la_LIBADD = $(linkback)
+ at HAVE_GNI_DL_TRUE@@HAVE_GNI_TRUE at libgnix_fi_la_DEPENDENCIES = $(linkback)
+ at HAVE_RXM_TRUE@_rxm_files = \
+ at HAVE_RXM_TRUE@ prov/rxm/src/rxm_attr.c \
+ at HAVE_RXM_TRUE@ prov/rxm/src/rxm_init.c \
+ at HAVE_RXM_TRUE@ prov/rxm/src/rxm_fabric.c \
+ at HAVE_RXM_TRUE@ prov/rxm/src/rxm_domain.c \
+ at HAVE_RXM_TRUE@ prov/rxm/src/rxm_conn.c \
+ at HAVE_RXM_TRUE@ prov/rxm/src/rxm_ep.c \
+ at HAVE_RXM_TRUE@ prov/rxm/src/rxm_cq.c \
+ at HAVE_RXM_TRUE@ prov/rxm/src/rxm.h
+
+ at HAVE_RXM_DL_TRUE@@HAVE_RXM_TRUE at librxm_fi_la_SOURCES = $(_rxm_files) $(common_srcs)
+ at HAVE_RXM_DL_TRUE@@HAVE_RXM_TRUE at librxm_fi_la_LIBADD = $(linkback) $(rxm_shm_LIBS)
+ at HAVE_RXM_DL_TRUE@@HAVE_RXM_TRUE at librxm_fi_la_LDFLAGS = -module -avoid-version -shared -export-dynamic
+ at HAVE_RXM_DL_TRUE@@HAVE_RXM_TRUE at librxm_fi_la_DEPENDENCIES = $(linkback)
+ at HAVE_RXD_TRUE@_rxd_files = \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd_attr.c \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd_init.c \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd_fabric.c \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd_domain.c \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd_av.c \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd_cq.c \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd_ep.c \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd_rma.c \
+ at HAVE_RXD_TRUE@ prov/rxd/src/rxd.h
+
+ at HAVE_RXD_DL_TRUE@@HAVE_RXD_TRUE at librxd_fi_la_SOURCES = $(_rxd_files) $(common_srcs)
+ at HAVE_RXD_DL_TRUE@@HAVE_RXD_TRUE at librxd_fi_la_LIBADD = $(linkback) $(rxd_shm_LIBS)
+ at HAVE_RXD_DL_TRUE@@HAVE_RXD_TRUE at librxd_fi_la_LDFLAGS = -module -avoid-version -shared -export-dynamic
+ at HAVE_RXD_DL_TRUE@@HAVE_RXD_TRUE at librxd_fi_la_DEPENDENCIES = $(linkback)
+man_MANS = $(real_man_pages) $(prov_install_man_pages) $(dummy_man_pages)
+EXTRA_DIST = \
+ NEWS.md \
+ libfabric.spec.in \
+ config/distscript.pl \
+ $(real_man_pages) $(prov_dist_man_pages) $(dummy_man_pages)
+
+all: config.h
+ $(MAKE) $(AM_MAKEFLAGS) all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .html .lo .log .o .obj .test .test$(EXEEXT)
+am--refresh:
+ @:
+$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(srcdir)/prov/sockets/Makefile.include $(srcdir)/prov/udp/Makefile.include $(srcdir)/prov/verbs/Makefile.include $(srcdir)/prov/usnic/Makefile.include $(srcdir)/prov/psm/Makefile.include $(srcdir)/prov/psm2/Makefile.include $(srcdir)/prov/mxm/Makefile.include $(srcdir)/prov/gni/Makefile.include $(srcdir)/prov/rxm/Makefile.include $(srcdir)/prov/rxd/Makefile.include $(am__configure_deps)
+ @for dep in $?; do \
+ case '$(am__configure_deps)' in \
+ *$$dep*) \
+ echo ' cd $(srcdir) && $(AUTOMAKE) --foreign'; \
+ $(am__cd) $(srcdir) && $(AUTOMAKE) --foreign \
+ && exit 0; \
+ exit 1;; \
+ esac; \
+ done; \
+ echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign Makefile'; \
+ $(am__cd) $(top_srcdir) && \
+ $(AUTOMAKE) --foreign Makefile
+.PRECIOUS: Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+ @case '$?' in \
+ *config.status*) \
+ echo ' $(SHELL) ./config.status'; \
+ $(SHELL) ./config.status;; \
+ *) \
+ echo ' cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe)'; \
+ cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe);; \
+ esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+ $(SHELL) ./config.status --recheck
+
+$(top_srcdir)/configure: $(am__configure_deps)
+ $(am__cd) $(srcdir) && $(AUTOCONF)
+$(ACLOCAL_M4): $(am__aclocal_m4_deps)
+ $(am__cd) $(srcdir) && $(ACLOCAL) $(ACLOCAL_AMFLAGS)
+$(am__aclocal_m4_deps):
+
+config.h: stamp-h1
+ @if test ! -f $@; then \
+ rm -f stamp-h1; \
+ $(MAKE) $(AM_MAKEFLAGS) stamp-h1; \
+ else :; fi
+
+stamp-h1: $(srcdir)/config.h.in $(top_builddir)/config.status
+ @rm -f stamp-h1
+ cd $(top_builddir) && $(SHELL) ./config.status config.h
+$(srcdir)/config.h.in: $(am__configure_deps)
+ ($(am__cd) $(top_srcdir) && $(AUTOHEADER))
+ rm -f stamp-h1
+ touch $@
+
+distclean-hdr:
+ -rm -f config.h stamp-h1
+libfabric.pc: $(top_builddir)/config.status $(srcdir)/libfabric.pc.in
+ cd $(top_builddir) && $(SHELL) ./config.status $@
+libfabric.spec: $(top_builddir)/config.status $(srcdir)/libfabric.spec.in
+ cd $(top_builddir) && $(SHELL) ./config.status $@
+libfabric.map: $(top_builddir)/config.status $(srcdir)/libfabric.map.in
+ cd $(top_builddir) && $(SHELL) ./config.status $@
+install-libLTLIBRARIES: $(lib_LTLIBRARIES)
+ @$(NORMAL_INSTALL)
+ test -z "$(libdir)" || $(MKDIR_P) "$(DESTDIR)$(libdir)"
+ @list='$(lib_LTLIBRARIES)'; test -n "$(libdir)" || list=; \
+ list2=; for p in $$list; do \
+ if test -f $$p; then \
+ list2="$$list2 $$p"; \
+ else :; fi; \
+ done; \
+ test -z "$$list2" || { \
+ echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(libdir)'"; \
+ $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(libdir)"; \
+ }
+
+uninstall-libLTLIBRARIES:
+ @$(NORMAL_UNINSTALL)
+ @list='$(lib_LTLIBRARIES)'; test -n "$(libdir)" || list=; \
+ for p in $$list; do \
+ $(am__strip_dir) \
+ echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(libdir)/$$f'"; \
+ $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(libdir)/$$f"; \
+ done
+
+clean-libLTLIBRARIES:
+ -test -z "$(lib_LTLIBRARIES)" || rm -f $(lib_LTLIBRARIES)
+ @list='$(lib_LTLIBRARIES)'; for p in $$list; do \
+ dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \
+ test "$$dir" != "$$p" || dir=.; \
+ echo "rm -f \"$${dir}/so_locations\""; \
+ rm -f "$${dir}/so_locations"; \
+ done
+
+clean-noinstLTLIBRARIES:
+ -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+ @list='$(noinst_LTLIBRARIES)'; for p in $$list; do \
+ dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \
+ test "$$dir" != "$$p" || dir=.; \
+ echo "rm -f \"$${dir}/so_locations\""; \
+ rm -f "$${dir}/so_locations"; \
+ done
+install-pkglibLTLIBRARIES: $(pkglib_LTLIBRARIES)
+ @$(NORMAL_INSTALL)
+ test -z "$(pkglibdir)" || $(MKDIR_P) "$(DESTDIR)$(pkglibdir)"
+ @list='$(pkglib_LTLIBRARIES)'; test -n "$(pkglibdir)" || list=; \
+ list2=; for p in $$list; do \
+ if test -f $$p; then \
+ list2="$$list2 $$p"; \
+ else :; fi; \
+ done; \
+ test -z "$$list2" || { \
+ echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(pkglibdir)'"; \
+ $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(pkglibdir)"; \
+ }
+
+uninstall-pkglibLTLIBRARIES:
+ @$(NORMAL_UNINSTALL)
+ @list='$(pkglib_LTLIBRARIES)'; test -n "$(pkglibdir)" || list=; \
+ for p in $$list; do \
+ $(am__strip_dir) \
+ echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(pkglibdir)/$$f'"; \
+ $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(pkglibdir)/$$f"; \
+ done
+
+clean-pkglibLTLIBRARIES:
+ -test -z "$(pkglib_LTLIBRARIES)" || rm -f $(pkglib_LTLIBRARIES)
+ @list='$(pkglib_LTLIBRARIES)'; for p in $$list; do \
+ dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \
+ test "$$dir" != "$$p" || dir=.; \
+ echo "rm -f \"$${dir}/so_locations\""; \
+ rm -f "$${dir}/so_locations"; \
+ done
+prov/gni/src/$(am__dirstamp):
+ @$(MKDIR_P) prov/gni/src
+ @: > prov/gni/src/$(am__dirstamp)
+prov/gni/src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/gni/src/$(DEPDIR)
+ @: > prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_atomic.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_av.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_bitmap.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_buddy_allocator.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_cm.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_cm_nic.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_cntr.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_cq.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_datagram.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_dom.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_ep.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_eq.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_fabric.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_freelist.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_hashtable.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_init.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_mbox_allocator.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_mr.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_mr_cache.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_mr_notifier.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_msg.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_nameserver.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_nic.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_poll.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_queue.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_rma.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_tags.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_trigger.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_util.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_vc.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_vector.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_xpmem.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/libgnix_fi_la-gnix_wait.lo: prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+src/$(am__dirstamp):
+ @$(MKDIR_P) src
+ @: > src/$(am__dirstamp)
+src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) src/$(DEPDIR)
+ @: > src/$(DEPDIR)/$(am__dirstamp)
+src/libgnix_fi_la-common.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libgnix_fi_la-enosys.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libgnix_fi_la-rbtree.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libgnix_fi_la-fasthash.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libgnix_fi_la-indexer.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/$(am__dirstamp):
+ @$(MKDIR_P) prov/util/src
+ @: > prov/util/src/$(am__dirstamp)
+prov/util/src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/util/src/$(DEPDIR)
+ @: > prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libgnix_fi_la-util_attr.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libgnix_fi_la-util_av.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libgnix_fi_la-util_cq.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libgnix_fi_la-util_domain.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libgnix_fi_la-util_ep.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libgnix_fi_la-util_eq.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libgnix_fi_la-util_fabric.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libgnix_fi_la-util_main.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libgnix_fi_la-util_poll.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libgnix_fi_la-util_wait.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libgnix_fi_la-util_buf.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libgnix_fi_la-util_mr.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+src/osx/$(am__dirstamp):
+ @$(MKDIR_P) src/osx
+ @: > src/osx/$(am__dirstamp)
+src/osx/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) src/osx/$(DEPDIR)
+ @: > src/osx/$(DEPDIR)/$(am__dirstamp)
+src/osx/libgnix_fi_la-osd.lo: src/osx/$(am__dirstamp) \
+ src/osx/$(DEPDIR)/$(am__dirstamp)
+src/unix/$(am__dirstamp):
+ @$(MKDIR_P) src/unix
+ @: > src/unix/$(am__dirstamp)
+src/unix/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) src/unix/$(DEPDIR)
+ @: > src/unix/$(DEPDIR)/$(am__dirstamp)
+src/unix/libgnix_fi_la-osd.lo: src/unix/$(am__dirstamp) \
+ src/unix/$(DEPDIR)/$(am__dirstamp)
+libgnix-fi.la: $(libgnix_fi_la_OBJECTS) $(libgnix_fi_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(libgnix_fi_la_LINK) $(am_libgnix_fi_la_rpath) $(libgnix_fi_la_OBJECTS) $(libgnix_fi_la_LIBADD) $(LIBS)
+prov/mxm/src/$(am__dirstamp):
+ @$(MKDIR_P) prov/mxm/src
+ @: > prov/mxm/src/$(am__dirstamp)
+prov/mxm/src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/mxm/src/$(DEPDIR)
+ @: > prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/libmlxm_fi_la-mlxm_init.lo: prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/libmlxm_fi_la-mlxm_domain.lo: \
+ prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/libmlxm_fi_la-mlxm_cq.lo: prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/libmlxm_fi_la-mlxm_av.lo: prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/libmlxm_fi_la-mlxm_ep.lo: prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/libmlxm_fi_la-mlxm_cm.lo: prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/libmlxm_fi_la-mlxm_tagged.lo: \
+ prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/libmlxm_fi_la-mlxm_mr.lo: prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+src/libmlxm_fi_la-common.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libmlxm_fi_la-enosys.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libmlxm_fi_la-rbtree.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libmlxm_fi_la-fasthash.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libmlxm_fi_la-indexer.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libmlxm_fi_la-util_attr.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libmlxm_fi_la-util_av.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libmlxm_fi_la-util_cq.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libmlxm_fi_la-util_domain.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libmlxm_fi_la-util_ep.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libmlxm_fi_la-util_eq.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libmlxm_fi_la-util_fabric.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libmlxm_fi_la-util_main.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libmlxm_fi_la-util_poll.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libmlxm_fi_la-util_wait.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libmlxm_fi_la-util_buf.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libmlxm_fi_la-util_mr.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+src/osx/libmlxm_fi_la-osd.lo: src/osx/$(am__dirstamp) \
+ src/osx/$(DEPDIR)/$(am__dirstamp)
+src/unix/libmlxm_fi_la-osd.lo: src/unix/$(am__dirstamp) \
+ src/unix/$(DEPDIR)/$(am__dirstamp)
+libmlxm-fi.la: $(libmlxm_fi_la_OBJECTS) $(libmlxm_fi_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(libmlxm_fi_la_LINK) $(am_libmlxm_fi_la_rpath) $(libmlxm_fi_la_OBJECTS) $(libmlxm_fi_la_LIBADD) $(LIBS)
+prov/psm/src/$(am__dirstamp):
+ @$(MKDIR_P) prov/psm/src
+ @: > prov/psm/src/$(am__dirstamp)
+prov/psm/src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/psm/src/$(DEPDIR)
+ @: > prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_init.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_fabric.lo: \
+ prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_domain.lo: \
+ prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_cq.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_cntr.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_av.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_ep.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_cm.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_tagged.lo: \
+ prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_msg.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_msg2.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_rma.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_atomic.lo: \
+ prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_am.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_mr.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_wait.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_fi_la-psmx_util.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+src/libpsmx_fi_la-common.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libpsmx_fi_la-enosys.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libpsmx_fi_la-rbtree.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libpsmx_fi_la-fasthash.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libpsmx_fi_la-indexer.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx_fi_la-util_attr.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx_fi_la-util_av.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx_fi_la-util_cq.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx_fi_la-util_domain.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx_fi_la-util_ep.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx_fi_la-util_eq.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx_fi_la-util_fabric.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx_fi_la-util_main.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx_fi_la-util_poll.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx_fi_la-util_wait.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx_fi_la-util_buf.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx_fi_la-util_mr.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+src/osx/libpsmx_fi_la-osd.lo: src/osx/$(am__dirstamp) \
+ src/osx/$(DEPDIR)/$(am__dirstamp)
+src/unix/libpsmx_fi_la-osd.lo: src/unix/$(am__dirstamp) \
+ src/unix/$(DEPDIR)/$(am__dirstamp)
+libpsmx-fi.la: $(libpsmx_fi_la_OBJECTS) $(libpsmx_fi_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(libpsmx_fi_la_LINK) $(am_libpsmx_fi_la_rpath) $(libpsmx_fi_la_OBJECTS) $(libpsmx_fi_la_LIBADD) $(LIBS)
+prov/psm/src/libpsmx_la-psmx_init.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_fabric.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_domain.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_cq.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_cntr.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_av.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_ep.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_cm.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_tagged.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_msg.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_msg2.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_rma.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_atomic.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_am.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_mr.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_wait.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm/src/libpsmx_la-psmx_util.lo: prov/psm/src/$(am__dirstamp) \
+ prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+libpsmx.la: $(libpsmx_la_OBJECTS) $(libpsmx_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(libpsmx_la_LINK) $(am_libpsmx_la_rpath) $(libpsmx_la_OBJECTS) $(libpsmx_la_LIBADD) $(LIBS)
+prov/psm2/src/$(am__dirstamp):
+ @$(MKDIR_P) prov/psm2/src
+ @: > prov/psm2/src/$(am__dirstamp)
+prov/psm2/src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/psm2/src/$(DEPDIR)
+ @: > prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_init.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_fabric.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_domain.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_cq.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_cntr.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_av.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_ep.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_cm.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_tagged.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_msg.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_rma.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_atomic.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_am.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_mr.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_wait.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_fi_la-psmx2_util.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+src/libpsmx2_fi_la-common.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libpsmx2_fi_la-enosys.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libpsmx2_fi_la-rbtree.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libpsmx2_fi_la-fasthash.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libpsmx2_fi_la-indexer.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx2_fi_la-util_attr.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx2_fi_la-util_av.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx2_fi_la-util_cq.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx2_fi_la-util_domain.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx2_fi_la-util_ep.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx2_fi_la-util_eq.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx2_fi_la-util_fabric.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx2_fi_la-util_main.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx2_fi_la-util_poll.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx2_fi_la-util_wait.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx2_fi_la-util_buf.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libpsmx2_fi_la-util_mr.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+src/osx/libpsmx2_fi_la-osd.lo: src/osx/$(am__dirstamp) \
+ src/osx/$(DEPDIR)/$(am__dirstamp)
+src/unix/libpsmx2_fi_la-osd.lo: src/unix/$(am__dirstamp) \
+ src/unix/$(DEPDIR)/$(am__dirstamp)
+libpsmx2-fi.la: $(libpsmx2_fi_la_OBJECTS) $(libpsmx2_fi_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(libpsmx2_fi_la_LINK) $(am_libpsmx2_fi_la_rpath) $(libpsmx2_fi_la_OBJECTS) $(libpsmx2_fi_la_LIBADD) $(LIBS)
+prov/psm2/src/libpsmx2_la-psmx2_init.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_fabric.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_domain.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_cq.lo: prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_cntr.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_av.lo: prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_ep.lo: prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_cm.lo: prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_tagged.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_msg.lo: prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_rma.lo: prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_atomic.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_am.lo: prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_mr.lo: prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_wait.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+prov/psm2/src/libpsmx2_la-psmx2_util.lo: \
+ prov/psm2/src/$(am__dirstamp) \
+ prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+libpsmx2.la: $(libpsmx2_la_OBJECTS) $(libpsmx2_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(libpsmx2_la_LINK) $(am_libpsmx2_la_rpath) $(libpsmx2_la_OBJECTS) $(libpsmx2_la_LIBADD) $(LIBS)
+prov/rxd/src/$(am__dirstamp):
+ @$(MKDIR_P) prov/rxd/src
+ @: > prov/rxd/src/$(am__dirstamp)
+prov/rxd/src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/rxd/src/$(DEPDIR)
+ @: > prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/rxd_attr.lo: prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/rxd_init.lo: prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/rxd_fabric.lo: prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/rxd_domain.lo: prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/rxd_av.lo: prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/rxd_cq.lo: prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/rxd_ep.lo: prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/rxd_rma.lo: prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+src/common.lo: src/$(am__dirstamp) src/$(DEPDIR)/$(am__dirstamp)
+src/enosys.lo: src/$(am__dirstamp) src/$(DEPDIR)/$(am__dirstamp)
+src/rbtree.lo: src/$(am__dirstamp) src/$(DEPDIR)/$(am__dirstamp)
+src/fasthash.lo: src/$(am__dirstamp) src/$(DEPDIR)/$(am__dirstamp)
+src/indexer.lo: src/$(am__dirstamp) src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/util_attr.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/util_av.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/util_cq.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/util_domain.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/util_ep.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/util_eq.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/util_fabric.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/util_main.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/util_poll.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/util_wait.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/util_buf.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/util_mr.lo: prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+src/osx/osd.lo: src/osx/$(am__dirstamp) \
+ src/osx/$(DEPDIR)/$(am__dirstamp)
+src/unix/osd.lo: src/unix/$(am__dirstamp) \
+ src/unix/$(DEPDIR)/$(am__dirstamp)
+librxd-fi.la: $(librxd_fi_la_OBJECTS) $(librxd_fi_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(librxd_fi_la_LINK) $(am_librxd_fi_la_rpath) $(librxd_fi_la_OBJECTS) $(librxd_fi_la_LIBADD) $(LIBS)
+prov/rxm/src/$(am__dirstamp):
+ @$(MKDIR_P) prov/rxm/src
+ @: > prov/rxm/src/$(am__dirstamp)
+prov/rxm/src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/rxm/src/$(DEPDIR)
+ @: > prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/rxm_attr.lo: prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/rxm_init.lo: prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/rxm_fabric.lo: prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/rxm_domain.lo: prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/rxm_conn.lo: prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/rxm_ep.lo: prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/rxm_cq.lo: prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+librxm-fi.la: $(librxm_fi_la_OBJECTS) $(librxm_fi_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(librxm_fi_la_LINK) $(am_librxm_fi_la_rpath) $(librxm_fi_la_OBJECTS) $(librxm_fi_la_LIBADD) $(LIBS)
+prov/sockets/src/$(am__dirstamp):
+ @$(MKDIR_P) prov/sockets/src
+ @: > prov/sockets/src/$(am__dirstamp)
+prov/sockets/src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/sockets/src/$(DEPDIR)
+ @: > prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_av.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_dom.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_eq.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_cq.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_cntr.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_poll.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_wait.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_ep_rdm.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_ep_dgram.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_ep_msg.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_fabric.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_ep.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_ctx.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_rx_entry.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_progress.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_comm.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_conn.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_msg.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_rma.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_atomic.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_trigger.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/sock_epoll.lo: prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+libsockets-fi.la: $(libsockets_fi_la_OBJECTS) $(libsockets_fi_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(libsockets_fi_la_LINK) $(am_libsockets_fi_la_rpath) $(libsockets_fi_la_OBJECTS) $(libsockets_fi_la_LIBADD) $(LIBS)
+prov/udp/src/$(am__dirstamp):
+ @$(MKDIR_P) prov/udp/src
+ @: > prov/udp/src/$(am__dirstamp)
+prov/udp/src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/udp/src/$(DEPDIR)
+ @: > prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+prov/udp/src/udpx_attr.lo: prov/udp/src/$(am__dirstamp) \
+ prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+prov/udp/src/udpx_cq.lo: prov/udp/src/$(am__dirstamp) \
+ prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+prov/udp/src/udpx_domain.lo: prov/udp/src/$(am__dirstamp) \
+ prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+prov/udp/src/udpx_ep.lo: prov/udp/src/$(am__dirstamp) \
+ prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+prov/udp/src/udpx_fabric.lo: prov/udp/src/$(am__dirstamp) \
+ prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+prov/udp/src/udpx_init.lo: prov/udp/src/$(am__dirstamp) \
+ prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+libudp-fi.la: $(libudp_fi_la_OBJECTS) $(libudp_fi_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(libudp_fi_la_LINK) $(am_libudp_fi_la_rpath) $(libudp_fi_la_OBJECTS) $(libudp_fi_la_LIBADD) $(LIBS)
+prov/usnic/src/usnic_direct/$(am__dirstamp):
+ @$(MKDIR_P) prov/usnic/src/usnic_direct
+ @: > prov/usnic/src/usnic_direct/$(am__dirstamp)
+prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/usnic/src/usnic_direct/$(DEPDIR)
+ @: > prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-libnl_utils_common.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_caps.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_dest.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_device.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_event.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_enum.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_cmd.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_sysfs.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_mem.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_poll.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_raw.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_udp.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_pio_udp.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_queues.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_socket.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_vnic.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-usnic_ip_utils.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_cq.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_dev.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_rq.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_wq.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_intr.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/$(am__dirstamp):
+ @$(MKDIR_P) prov/usnic/src
+ @: > prov/usnic/src/$(am__dirstamp)
+prov/usnic/src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/usnic/src/$(DEPDIR)
+ @: > prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_av.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_cm.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_cq.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_dgram.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_domain.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_endpoint.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_ep_dgram.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_ep_msg.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_ep_rdm.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_eq.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_fabric.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_mem.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_msg.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_pep.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_progress.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_rdm.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_timer.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_poll.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_ext.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_wait.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/libusnic_fi_la-usdf_fake_ibv.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+src/libusnic_fi_la-common.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libusnic_fi_la-enosys.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libusnic_fi_la-rbtree.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libusnic_fi_la-fasthash.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libusnic_fi_la-indexer.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libusnic_fi_la-util_attr.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libusnic_fi_la-util_av.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libusnic_fi_la-util_cq.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libusnic_fi_la-util_domain.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libusnic_fi_la-util_ep.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libusnic_fi_la-util_eq.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libusnic_fi_la-util_fabric.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libusnic_fi_la-util_main.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libusnic_fi_la-util_poll.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libusnic_fi_la-util_wait.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libusnic_fi_la-util_buf.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libusnic_fi_la-util_mr.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+src/osx/libusnic_fi_la-osd.lo: src/osx/$(am__dirstamp) \
+ src/osx/$(DEPDIR)/$(am__dirstamp)
+src/unix/libusnic_fi_la-osd.lo: src/unix/$(am__dirstamp) \
+ src/unix/$(DEPDIR)/$(am__dirstamp)
+libusnic-fi.la: $(libusnic_fi_la_OBJECTS) $(libusnic_fi_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(libusnic_fi_la_LINK) $(am_libusnic_fi_la_rpath) $(libusnic_fi_la_OBJECTS) $(libusnic_fi_la_LIBADD) $(LIBS)
+prov/verbs/src/$(am__dirstamp):
+ @$(MKDIR_P) prov/verbs/src
+ @: > prov/verbs/src/$(am__dirstamp)
+prov/verbs/src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/verbs/src/$(DEPDIR)
+ @: > prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/libverbs_fi_la-fi_verbs.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/libverbs_fi_la-verbs_atomic.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/libverbs_fi_la-verbs_av.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/libverbs_fi_la-verbs_cm.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/libverbs_fi_la-verbs_cq.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/libverbs_fi_la-verbs_srq.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/libverbs_fi_la-verbs_domain.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/libverbs_fi_la-verbs_eq.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/libverbs_fi_la-verbs_info.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/libverbs_fi_la-verbs_msg.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/libverbs_fi_la-verbs_msg_ep.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/libverbs_fi_la-verbs_rma.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/$(am__dirstamp):
+ @$(MKDIR_P) prov/verbs/src/ep_rdm
+ @: > prov/verbs/src/ep_rdm/$(am__dirstamp)
+prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/verbs/src/ep_rdm/$(DEPDIR)
+ @: > prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_av_ep_rdm.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_cq_ep_rdm.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_ep_rdm.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cm.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cntr.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_msg.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm_states.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_utils.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+src/libverbs_fi_la-common.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libverbs_fi_la-enosys.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libverbs_fi_la-rbtree.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libverbs_fi_la-fasthash.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/libverbs_fi_la-indexer.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libverbs_fi_la-util_attr.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libverbs_fi_la-util_av.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libverbs_fi_la-util_cq.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libverbs_fi_la-util_domain.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libverbs_fi_la-util_ep.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libverbs_fi_la-util_eq.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libverbs_fi_la-util_fabric.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libverbs_fi_la-util_main.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libverbs_fi_la-util_poll.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libverbs_fi_la-util_wait.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libverbs_fi_la-util_buf.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/libverbs_fi_la-util_mr.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+src/osx/libverbs_fi_la-osd.lo: src/osx/$(am__dirstamp) \
+ src/osx/$(DEPDIR)/$(am__dirstamp)
+src/unix/libverbs_fi_la-osd.lo: src/unix/$(am__dirstamp) \
+ src/unix/$(DEPDIR)/$(am__dirstamp)
+libverbs-fi.la: $(libverbs_fi_la_OBJECTS) $(libverbs_fi_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(libverbs_fi_la_LINK) $(am_libverbs_fi_la_rpath) $(libverbs_fi_la_OBJECTS) $(libverbs_fi_la_LIBADD) $(LIBS)
+src/src_libfabric_la-fabric.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/src_libfabric_la-fi_tostr.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/src_libfabric_la-log.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/src_libfabric_la-var.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/src_libfabric_la-common.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/src_libfabric_la-enosys.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/src_libfabric_la-rbtree.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/src_libfabric_la-fasthash.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+src/src_libfabric_la-indexer.lo: src/$(am__dirstamp) \
+ src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/src_libfabric_la-util_attr.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/src_libfabric_la-util_av.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/src_libfabric_la-util_cq.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/src_libfabric_la-util_domain.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/src_libfabric_la-util_ep.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/src_libfabric_la-util_eq.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/src_libfabric_la-util_fabric.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/src_libfabric_la-util_main.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/src_libfabric_la-util_poll.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/src_libfabric_la-util_wait.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/src_libfabric_la-util_buf.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+prov/util/src/src_libfabric_la-util_mr.lo: \
+ prov/util/src/$(am__dirstamp) \
+ prov/util/src/$(DEPDIR)/$(am__dirstamp)
+src/osx/src_libfabric_la-osd.lo: src/osx/$(am__dirstamp) \
+ src/osx/$(DEPDIR)/$(am__dirstamp)
+src/unix/src_libfabric_la-osd.lo: src/unix/$(am__dirstamp) \
+ src/unix/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_av.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_dom.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_eq.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_cq.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_cntr.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_poll.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_wait.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_ep_rdm.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_ep_dgram.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_ep_msg.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_fabric.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_ep.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_ctx.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_rx_entry.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_progress.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_comm.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_conn.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_msg.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_rma.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_atomic.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_trigger.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/sockets/src/src_libfabric_la-sock_epoll.lo: \
+ prov/sockets/src/$(am__dirstamp) \
+ prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+prov/udp/src/src_libfabric_la-udpx_attr.lo: \
+ prov/udp/src/$(am__dirstamp) \
+ prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+prov/udp/src/src_libfabric_la-udpx_cq.lo: \
+ prov/udp/src/$(am__dirstamp) \
+ prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+prov/udp/src/src_libfabric_la-udpx_domain.lo: \
+ prov/udp/src/$(am__dirstamp) \
+ prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+prov/udp/src/src_libfabric_la-udpx_ep.lo: \
+ prov/udp/src/$(am__dirstamp) \
+ prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+prov/udp/src/src_libfabric_la-udpx_fabric.lo: \
+ prov/udp/src/$(am__dirstamp) \
+ prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+prov/udp/src/src_libfabric_la-udpx_init.lo: \
+ prov/udp/src/$(am__dirstamp) \
+ prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/src_libfabric_la-fi_verbs.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/src_libfabric_la-verbs_atomic.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/src_libfabric_la-verbs_av.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/src_libfabric_la-verbs_cm.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/src_libfabric_la-verbs_cq.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/src_libfabric_la-verbs_srq.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/src_libfabric_la-verbs_domain.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/src_libfabric_la-verbs_eq.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/src_libfabric_la-verbs_info.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/src_libfabric_la-verbs_msg.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/src_libfabric_la-verbs_msg_ep.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/src_libfabric_la-verbs_rma.lo: \
+ prov/verbs/src/$(am__dirstamp) \
+ prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_av_ep_rdm.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_cq_ep_rdm.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_ep_rdm.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cm.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cntr.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_msg.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm_states.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_utils.lo: \
+ prov/verbs/src/ep_rdm/$(am__dirstamp) \
+ prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-libnl_utils_common.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_caps.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_dest.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_device.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_event.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_enum.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_cmd.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_sysfs.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_mem.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_poll.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_post.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_raw.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_udp.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_pio_udp.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_queues.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_socket.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_vnic.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-usnic_ip_utils.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-vnic_cq.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-vnic_dev.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-vnic_rq.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-vnic_wq.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/usnic_direct/src_libfabric_la-vnic_intr.lo: \
+ prov/usnic/src/usnic_direct/$(am__dirstamp) \
+ prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_av.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_cm.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_cq.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_dgram.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_domain.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_endpoint.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_ep_dgram.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_ep_msg.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_ep_rdm.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_eq.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_fabric.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_mem.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_msg.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_pep.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_progress.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_rdm.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_timer.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_poll.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_ext.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_wait.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/usnic/src/src_libfabric_la-usdf_fake_ibv.lo: \
+ prov/usnic/src/$(am__dirstamp) \
+ prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/src_libfabric_la-mlxm_init.lo: \
+ prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/src_libfabric_la-mlxm_domain.lo: \
+ prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/src_libfabric_la-mlxm_cq.lo: \
+ prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/src_libfabric_la-mlxm_av.lo: \
+ prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/src_libfabric_la-mlxm_ep.lo: \
+ prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/src_libfabric_la-mlxm_cm.lo: \
+ prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/src_libfabric_la-mlxm_tagged.lo: \
+ prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/mxm/src/src_libfabric_la-mlxm_mr.lo: \
+ prov/mxm/src/$(am__dirstamp) \
+ prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_atomic.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_av.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_bitmap.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_buddy_allocator.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_cm.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_cm_nic.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_cntr.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_cq.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_datagram.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_dom.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_ep.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_eq.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_fabric.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_freelist.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_hashtable.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_init.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_mbox_allocator.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_mr.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_mr_cache.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_mr_notifier.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_msg.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_nameserver.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_nic.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_poll.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_queue.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_rma.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_tags.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_trigger.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_util.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_vc.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_vector.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_xpmem.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/gni/src/src_libfabric_la-gnix_wait.lo: \
+ prov/gni/src/$(am__dirstamp) \
+ prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/src_libfabric_la-rxm_attr.lo: \
+ prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/src_libfabric_la-rxm_init.lo: \
+ prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/src_libfabric_la-rxm_fabric.lo: \
+ prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/src_libfabric_la-rxm_domain.lo: \
+ prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/src_libfabric_la-rxm_conn.lo: \
+ prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/src_libfabric_la-rxm_ep.lo: prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxm/src/src_libfabric_la-rxm_cq.lo: prov/rxm/src/$(am__dirstamp) \
+ prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/src_libfabric_la-rxd_attr.lo: \
+ prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/src_libfabric_la-rxd_init.lo: \
+ prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/src_libfabric_la-rxd_fabric.lo: \
+ prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/src_libfabric_la-rxd_domain.lo: \
+ prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/src_libfabric_la-rxd_av.lo: prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/src_libfabric_la-rxd_cq.lo: prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/src_libfabric_la-rxd_ep.lo: prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+prov/rxd/src/src_libfabric_la-rxd_rma.lo: \
+ prov/rxd/src/$(am__dirstamp) \
+ prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+src/libfabric.la: $(src_libfabric_la_OBJECTS) $(src_libfabric_la_DEPENDENCIES) src/$(am__dirstamp)
+ $(AM_V_CCLD)$(src_libfabric_la_LINK) $(am_src_libfabric_la_rpath) $(src_libfabric_la_OBJECTS) $(src_libfabric_la_LIBADD) $(LIBS)
+install-binPROGRAMS: $(bin_PROGRAMS)
+ @$(NORMAL_INSTALL)
+ test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)"
+ @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \
+ for p in $$list; do echo "$$p $$p"; done | \
+ sed 's/$(EXEEXT)$$//' | \
+ while read p p1; do if test -f $$p || test -f $$p1; \
+ then echo "$$p"; echo "$$p"; else :; fi; \
+ done | \
+ sed -e 'p;s,.*/,,;n;h' -e 's|.*|.|' \
+ -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \
+ sed 'N;N;N;s,\n, ,g' | \
+ $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \
+ { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \
+ if ($$2 == $$4) files[d] = files[d] " " $$1; \
+ else { print "f", $$3 "/" $$4, $$1; } } \
+ END { for (d in files) print "f", d, files[d] }' | \
+ while read type dir files; do \
+ if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \
+ test -z "$$files" || { \
+ echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(bindir)$$dir'"; \
+ $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \
+ } \
+ ; done
+
+uninstall-binPROGRAMS:
+ @$(NORMAL_UNINSTALL)
+ @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \
+ files=`for p in $$list; do echo "$$p"; done | \
+ sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \
+ -e 's/$$/$(EXEEXT)/' `; \
+ test -n "$$list" || exit 0; \
+ echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \
+ cd "$(DESTDIR)$(bindir)" && rm -f $$files
+
+clean-binPROGRAMS:
+ @list='$(bin_PROGRAMS)'; test -n "$$list" || exit 0; \
+ echo " rm -f" $$list; \
+ rm -f $$list || exit $$?; \
+ test -n "$(EXEEXT)" || exit 0; \
+ list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
+ echo " rm -f" $$list; \
+ rm -f $$list
+prov/gni/test/$(am__dirstamp):
+ @$(MKDIR_P) prov/gni/test
+ @: > prov/gni/test/$(am__dirstamp)
+prov/gni/test/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/gni/test/$(DEPDIR)
+ @: > prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-allocator.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-av.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-bitmap.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-buddy_allocator.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-cancel.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-cntr.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-cq.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-datagram.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-dlist-utils.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-dom.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-ep.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-eq.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-freelist.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-hashtable.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-mr.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-mr_notifier.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-nic.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-pmi_utils.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-queue.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-rdm_atomic.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-rdm_dgram_rma.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-rdm_rx_overrun.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-rdm_sr.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-rdm_tagged_sr.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-tags.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-api.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-api_cq.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-api_cntr.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-utils.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-vc.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-vc_lookup.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-vector.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-wait.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/prov_gni_test_gnitest-common.$(OBJEXT): \
+ prov/gni/test/$(am__dirstamp) \
+ prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+prov/gni/test/gnitest$(EXEEXT): $(prov_gni_test_gnitest_OBJECTS) $(prov_gni_test_gnitest_DEPENDENCIES) prov/gni/test/$(am__dirstamp)
+ @rm -f prov/gni/test/gnitest$(EXEEXT)
+ $(AM_V_CCLD)$(prov_gni_test_gnitest_LINK) $(prov_gni_test_gnitest_OBJECTS) $(prov_gni_test_gnitest_LDADD) $(LIBS)
+util/$(am__dirstamp):
+ @$(MKDIR_P) util
+ @: > util/$(am__dirstamp)
+util/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) util/$(DEPDIR)
+ @: > util/$(DEPDIR)/$(am__dirstamp)
+util/info.$(OBJEXT): util/$(am__dirstamp) \
+ util/$(DEPDIR)/$(am__dirstamp)
+util/fi_info$(EXEEXT): $(util_fi_info_OBJECTS) $(util_fi_info_DEPENDENCIES) util/$(am__dirstamp)
+ @rm -f util/fi_info$(EXEEXT)
+ $(AM_V_CCLD)$(LINK) $(util_fi_info_OBJECTS) $(util_fi_info_LDADD) $(LIBS)
+util/pingpong.$(OBJEXT): util/$(am__dirstamp) \
+ util/$(DEPDIR)/$(am__dirstamp)
+util/fi_pingpong$(EXEEXT): $(util_fi_pingpong_OBJECTS) $(util_fi_pingpong_DEPENDENCIES) util/$(am__dirstamp)
+ @rm -f util/fi_pingpong$(EXEEXT)
+ $(AM_V_CCLD)$(LINK) $(util_fi_pingpong_OBJECTS) $(util_fi_pingpong_LDADD) $(LIBS)
+util/strerror.$(OBJEXT): util/$(am__dirstamp) \
+ util/$(DEPDIR)/$(am__dirstamp)
+util/fi_strerror$(EXEEXT): $(util_fi_strerror_OBJECTS) $(util_fi_strerror_DEPENDENCIES) util/$(am__dirstamp)
+ @rm -f util/fi_strerror$(EXEEXT)
+ $(AM_V_CCLD)$(LINK) $(util_fi_strerror_OBJECTS) $(util_fi_strerror_LDADD) $(LIBS)
+install-binSCRIPTS: $(bin_SCRIPTS)
+ @$(NORMAL_INSTALL)
+ test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)"
+ @list='$(bin_SCRIPTS)'; test -n "$(bindir)" || list=; \
+ for p in $$list; do \
+ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+ if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \
+ done | \
+ sed -e 'p;s,.*/,,;n' \
+ -e 'h;s|.*|.|' \
+ -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \
+ $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \
+ { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \
+ if ($$2 == $$4) { files[d] = files[d] " " $$1; \
+ if (++n[d] == $(am__install_max)) { \
+ print "f", d, files[d]; n[d] = 0; files[d] = "" } } \
+ else { print "f", d "/" $$4, $$1 } } \
+ END { for (d in files) print "f", d, files[d] }' | \
+ while read type dir files; do \
+ if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \
+ test -z "$$files" || { \
+ echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(bindir)$$dir'"; \
+ $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \
+ } \
+ ; done
+
+uninstall-binSCRIPTS:
+ @$(NORMAL_UNINSTALL)
+ @list='$(bin_SCRIPTS)'; test -n "$(bindir)" || exit 0; \
+ files=`for p in $$list; do echo "$$p"; done | \
+ sed -e 's,.*/,,;$(transform)'`; \
+ test -n "$$list" || exit 0; \
+ echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \
+ cd "$(DESTDIR)$(bindir)" && rm -f $$files
+
+mostlyclean-compile:
+ -rm -f *.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_atomic.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_atomic.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_av.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_av.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_bitmap.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_bitmap.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_buddy_allocator.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_buddy_allocator.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_cm.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_cm.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_cm_nic.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_cm_nic.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_cntr.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_cntr.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_cq.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_cq.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_datagram.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_datagram.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_dom.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_dom.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_ep.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_ep.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_eq.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_eq.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_fabric.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_fabric.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_freelist.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_freelist.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_hashtable.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_hashtable.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_init.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_init.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_mbox_allocator.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_mbox_allocator.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_mr.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_mr.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_mr_cache.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_mr_cache.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_mr_notifier.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_mr_notifier.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_msg.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_msg.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_nameserver.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_nameserver.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_nic.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_nic.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_poll.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_poll.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_queue.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_queue.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_rma.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_rma.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_tags.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_tags.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_trigger.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_trigger.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_util.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_util.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_vc.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_vc.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_vector.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_vector.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_wait.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_wait.lo
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_xpmem.$(OBJEXT)
+ -rm -f prov/gni/src/libgnix_fi_la-gnix_xpmem.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_atomic.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_atomic.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_av.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_av.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_bitmap.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_bitmap.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_buddy_allocator.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_buddy_allocator.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_cm.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_cm.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_cm_nic.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_cm_nic.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_cntr.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_cntr.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_cq.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_cq.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_datagram.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_datagram.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_dom.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_dom.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_ep.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_ep.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_eq.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_eq.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_fabric.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_fabric.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_freelist.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_freelist.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_hashtable.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_hashtable.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_init.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_init.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_mbox_allocator.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_mbox_allocator.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_mr.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_mr.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_mr_cache.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_mr_cache.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_mr_notifier.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_mr_notifier.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_msg.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_msg.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_nameserver.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_nameserver.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_nic.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_nic.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_poll.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_poll.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_queue.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_queue.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_rma.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_rma.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_tags.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_tags.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_trigger.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_trigger.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_util.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_util.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_vc.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_vc.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_vector.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_vector.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_wait.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_wait.lo
+ -rm -f prov/gni/src/src_libfabric_la-gnix_xpmem.$(OBJEXT)
+ -rm -f prov/gni/src/src_libfabric_la-gnix_xpmem.lo
+ -rm -f prov/gni/test/prov_gni_test_gnitest-allocator.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-api.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-api_cntr.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-api_cq.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-av.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-bitmap.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-buddy_allocator.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-cancel.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-cntr.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-common.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-cq.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-datagram.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-dlist-utils.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-dom.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-ep.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-eq.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-freelist.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-hashtable.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-mr.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-mr_notifier.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-nic.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-pmi_utils.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-queue.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-rdm_atomic.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-rdm_dgram_rma.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-rdm_rx_overrun.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-rdm_sr.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-rdm_tagged_sr.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-tags.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-utils.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-vc.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-vc_lookup.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-vector.$(OBJEXT)
+ -rm -f prov/gni/test/prov_gni_test_gnitest-wait.$(OBJEXT)
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_av.$(OBJEXT)
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_av.lo
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_cm.$(OBJEXT)
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_cm.lo
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_cq.$(OBJEXT)
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_cq.lo
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_domain.$(OBJEXT)
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_domain.lo
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_ep.$(OBJEXT)
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_ep.lo
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_init.$(OBJEXT)
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_init.lo
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_mr.$(OBJEXT)
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_mr.lo
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_tagged.$(OBJEXT)
+ -rm -f prov/mxm/src/libmlxm_fi_la-mlxm_tagged.lo
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_av.$(OBJEXT)
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_av.lo
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_cm.$(OBJEXT)
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_cm.lo
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_cq.$(OBJEXT)
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_cq.lo
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_domain.$(OBJEXT)
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_domain.lo
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_ep.$(OBJEXT)
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_ep.lo
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_init.$(OBJEXT)
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_init.lo
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_mr.$(OBJEXT)
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_mr.lo
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_tagged.$(OBJEXT)
+ -rm -f prov/mxm/src/src_libfabric_la-mlxm_tagged.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_am.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_am.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_atomic.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_atomic.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_av.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_av.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_cm.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_cm.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_cntr.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_cntr.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_cq.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_cq.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_domain.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_domain.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_ep.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_ep.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_fabric.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_fabric.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_init.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_init.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_mr.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_mr.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_msg.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_msg.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_msg2.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_msg2.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_rma.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_rma.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_tagged.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_tagged.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_util.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_util.lo
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_wait.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_fi_la-psmx_wait.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_am.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_am.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_atomic.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_atomic.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_av.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_av.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_cm.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_cm.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_cntr.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_cntr.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_cq.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_cq.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_domain.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_domain.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_ep.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_ep.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_fabric.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_fabric.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_init.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_init.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_mr.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_mr.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_msg.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_msg.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_msg2.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_msg2.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_rma.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_rma.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_tagged.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_tagged.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_util.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_util.lo
+ -rm -f prov/psm/src/libpsmx_la-psmx_wait.$(OBJEXT)
+ -rm -f prov/psm/src/libpsmx_la-psmx_wait.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_am.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_am.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_atomic.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_atomic.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_av.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_av.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_cm.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_cm.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_cntr.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_cntr.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_cq.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_cq.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_domain.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_domain.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_ep.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_ep.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_fabric.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_fabric.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_init.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_init.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_mr.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_mr.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_msg.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_msg.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_rma.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_rma.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_tagged.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_tagged.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_util.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_util.lo
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_wait.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_fi_la-psmx2_wait.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_am.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_am.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_atomic.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_atomic.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_av.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_av.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_cm.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_cm.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_cntr.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_cntr.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_cq.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_cq.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_domain.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_domain.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_ep.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_ep.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_fabric.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_fabric.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_init.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_init.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_mr.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_mr.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_msg.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_msg.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_rma.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_rma.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_tagged.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_tagged.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_util.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_util.lo
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_wait.$(OBJEXT)
+ -rm -f prov/psm2/src/libpsmx2_la-psmx2_wait.lo
+ -rm -f prov/rxd/src/rxd_attr.$(OBJEXT)
+ -rm -f prov/rxd/src/rxd_attr.lo
+ -rm -f prov/rxd/src/rxd_av.$(OBJEXT)
+ -rm -f prov/rxd/src/rxd_av.lo
+ -rm -f prov/rxd/src/rxd_cq.$(OBJEXT)
+ -rm -f prov/rxd/src/rxd_cq.lo
+ -rm -f prov/rxd/src/rxd_domain.$(OBJEXT)
+ -rm -f prov/rxd/src/rxd_domain.lo
+ -rm -f prov/rxd/src/rxd_ep.$(OBJEXT)
+ -rm -f prov/rxd/src/rxd_ep.lo
+ -rm -f prov/rxd/src/rxd_fabric.$(OBJEXT)
+ -rm -f prov/rxd/src/rxd_fabric.lo
+ -rm -f prov/rxd/src/rxd_init.$(OBJEXT)
+ -rm -f prov/rxd/src/rxd_init.lo
+ -rm -f prov/rxd/src/rxd_rma.$(OBJEXT)
+ -rm -f prov/rxd/src/rxd_rma.lo
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_attr.$(OBJEXT)
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_attr.lo
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_av.$(OBJEXT)
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_av.lo
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_cq.$(OBJEXT)
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_cq.lo
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_domain.$(OBJEXT)
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_domain.lo
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_ep.$(OBJEXT)
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_ep.lo
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_fabric.$(OBJEXT)
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_fabric.lo
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_init.$(OBJEXT)
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_init.lo
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_rma.$(OBJEXT)
+ -rm -f prov/rxd/src/src_libfabric_la-rxd_rma.lo
+ -rm -f prov/rxm/src/rxm_attr.$(OBJEXT)
+ -rm -f prov/rxm/src/rxm_attr.lo
+ -rm -f prov/rxm/src/rxm_conn.$(OBJEXT)
+ -rm -f prov/rxm/src/rxm_conn.lo
+ -rm -f prov/rxm/src/rxm_cq.$(OBJEXT)
+ -rm -f prov/rxm/src/rxm_cq.lo
+ -rm -f prov/rxm/src/rxm_domain.$(OBJEXT)
+ -rm -f prov/rxm/src/rxm_domain.lo
+ -rm -f prov/rxm/src/rxm_ep.$(OBJEXT)
+ -rm -f prov/rxm/src/rxm_ep.lo
+ -rm -f prov/rxm/src/rxm_fabric.$(OBJEXT)
+ -rm -f prov/rxm/src/rxm_fabric.lo
+ -rm -f prov/rxm/src/rxm_init.$(OBJEXT)
+ -rm -f prov/rxm/src/rxm_init.lo
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_attr.$(OBJEXT)
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_attr.lo
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_conn.$(OBJEXT)
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_conn.lo
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_cq.$(OBJEXT)
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_cq.lo
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_domain.$(OBJEXT)
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_domain.lo
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_ep.$(OBJEXT)
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_ep.lo
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_fabric.$(OBJEXT)
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_fabric.lo
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_init.$(OBJEXT)
+ -rm -f prov/rxm/src/src_libfabric_la-rxm_init.lo
+ -rm -f prov/sockets/src/sock_atomic.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_atomic.lo
+ -rm -f prov/sockets/src/sock_av.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_av.lo
+ -rm -f prov/sockets/src/sock_cntr.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_cntr.lo
+ -rm -f prov/sockets/src/sock_comm.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_comm.lo
+ -rm -f prov/sockets/src/sock_conn.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_conn.lo
+ -rm -f prov/sockets/src/sock_cq.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_cq.lo
+ -rm -f prov/sockets/src/sock_ctx.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_ctx.lo
+ -rm -f prov/sockets/src/sock_dom.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_dom.lo
+ -rm -f prov/sockets/src/sock_ep.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_ep.lo
+ -rm -f prov/sockets/src/sock_ep_dgram.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_ep_dgram.lo
+ -rm -f prov/sockets/src/sock_ep_msg.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_ep_msg.lo
+ -rm -f prov/sockets/src/sock_ep_rdm.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_ep_rdm.lo
+ -rm -f prov/sockets/src/sock_epoll.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_epoll.lo
+ -rm -f prov/sockets/src/sock_eq.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_eq.lo
+ -rm -f prov/sockets/src/sock_fabric.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_fabric.lo
+ -rm -f prov/sockets/src/sock_msg.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_msg.lo
+ -rm -f prov/sockets/src/sock_poll.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_poll.lo
+ -rm -f prov/sockets/src/sock_progress.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_progress.lo
+ -rm -f prov/sockets/src/sock_rma.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_rma.lo
+ -rm -f prov/sockets/src/sock_rx_entry.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_rx_entry.lo
+ -rm -f prov/sockets/src/sock_trigger.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_trigger.lo
+ -rm -f prov/sockets/src/sock_wait.$(OBJEXT)
+ -rm -f prov/sockets/src/sock_wait.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_atomic.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_atomic.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_av.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_av.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_cntr.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_cntr.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_comm.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_comm.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_conn.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_conn.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_cq.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_cq.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_ctx.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_ctx.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_dom.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_dom.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_ep.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_ep.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_ep_dgram.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_ep_dgram.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_ep_msg.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_ep_msg.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_ep_rdm.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_ep_rdm.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_epoll.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_epoll.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_eq.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_eq.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_fabric.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_fabric.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_msg.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_msg.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_poll.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_poll.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_progress.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_progress.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_rma.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_rma.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_rx_entry.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_rx_entry.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_trigger.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_trigger.lo
+ -rm -f prov/sockets/src/src_libfabric_la-sock_wait.$(OBJEXT)
+ -rm -f prov/sockets/src/src_libfabric_la-sock_wait.lo
+ -rm -f prov/udp/src/src_libfabric_la-udpx_attr.$(OBJEXT)
+ -rm -f prov/udp/src/src_libfabric_la-udpx_attr.lo
+ -rm -f prov/udp/src/src_libfabric_la-udpx_cq.$(OBJEXT)
+ -rm -f prov/udp/src/src_libfabric_la-udpx_cq.lo
+ -rm -f prov/udp/src/src_libfabric_la-udpx_domain.$(OBJEXT)
+ -rm -f prov/udp/src/src_libfabric_la-udpx_domain.lo
+ -rm -f prov/udp/src/src_libfabric_la-udpx_ep.$(OBJEXT)
+ -rm -f prov/udp/src/src_libfabric_la-udpx_ep.lo
+ -rm -f prov/udp/src/src_libfabric_la-udpx_fabric.$(OBJEXT)
+ -rm -f prov/udp/src/src_libfabric_la-udpx_fabric.lo
+ -rm -f prov/udp/src/src_libfabric_la-udpx_init.$(OBJEXT)
+ -rm -f prov/udp/src/src_libfabric_la-udpx_init.lo
+ -rm -f prov/udp/src/udpx_attr.$(OBJEXT)
+ -rm -f prov/udp/src/udpx_attr.lo
+ -rm -f prov/udp/src/udpx_cq.$(OBJEXT)
+ -rm -f prov/udp/src/udpx_cq.lo
+ -rm -f prov/udp/src/udpx_domain.$(OBJEXT)
+ -rm -f prov/udp/src/udpx_domain.lo
+ -rm -f prov/udp/src/udpx_ep.$(OBJEXT)
+ -rm -f prov/udp/src/udpx_ep.lo
+ -rm -f prov/udp/src/udpx_fabric.$(OBJEXT)
+ -rm -f prov/udp/src/udpx_fabric.lo
+ -rm -f prov/udp/src/udpx_init.$(OBJEXT)
+ -rm -f prov/udp/src/udpx_init.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_av.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_av.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_cm.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_cm.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_cq.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_cq.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_dgram.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_dgram.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_domain.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_domain.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_endpoint.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_endpoint.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_ep_dgram.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_ep_dgram.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_ep_msg.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_ep_msg.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_ep_rdm.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_ep_rdm.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_eq.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_eq.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_ext.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_ext.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_fabric.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_fabric.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_fake_ibv.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_fake_ibv.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_mem.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_mem.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_msg.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_msg.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_pep.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_pep.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_poll.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_poll.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_progress.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_progress.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_rdm.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_rdm.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_timer.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_timer.lo
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_wait.$(OBJEXT)
+ -rm -f prov/usnic/src/libusnic_fi_la-usdf_wait.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_av.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_av.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_cm.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_cm.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_cq.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_cq.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_dgram.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_dgram.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_domain.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_domain.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_endpoint.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_endpoint.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_ep_dgram.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_ep_dgram.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_ep_msg.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_ep_msg.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_ep_rdm.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_ep_rdm.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_eq.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_eq.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_ext.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_ext.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_fabric.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_fabric.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_fake_ibv.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_fake_ibv.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_mem.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_mem.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_msg.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_msg.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_pep.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_pep.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_poll.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_poll.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_progress.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_progress.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_rdm.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_rdm.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_timer.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_timer.lo
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_wait.$(OBJEXT)
+ -rm -f prov/usnic/src/src_libfabric_la-usdf_wait.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-libnl_utils_common.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-libnl_utils_common.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_caps.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_caps.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_dest.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_dest.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_device.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_device.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_enum.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_enum.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_event.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_event.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_cmd.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_cmd.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_sysfs.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_sysfs.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_mem.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_mem.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_poll.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_poll.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_pio_udp.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_pio_udp.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_raw.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_raw.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_udp.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_udp.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_queues.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_queues.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_socket.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_socket.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_vnic.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usd_vnic.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usnic_ip_utils.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-usnic_ip_utils.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_cq.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_cq.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_dev.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_dev.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_intr.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_intr.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_rq.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_rq.lo
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_wq.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_wq.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-libnl_utils_common.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-libnl_utils_common.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_caps.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_caps.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_dest.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_dest.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_device.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_device.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_enum.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_enum.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_event.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_event.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_cmd.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_cmd.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_sysfs.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_sysfs.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_mem.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_mem.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_poll.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_poll.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_post.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_post.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_pio_udp.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_pio_udp.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_raw.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_raw.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_udp.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_udp.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_queues.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_queues.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_socket.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_socket.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_vnic.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usd_vnic.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usnic_ip_utils.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-usnic_ip_utils.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-vnic_cq.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-vnic_cq.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-vnic_dev.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-vnic_dev.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-vnic_intr.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-vnic_intr.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-vnic_rq.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-vnic_rq.lo
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-vnic_wq.$(OBJEXT)
+ -rm -f prov/usnic/src/usnic_direct/src_libfabric_la-vnic_wq.lo
+ -rm -f prov/util/src/libgnix_fi_la-util_attr.$(OBJEXT)
+ -rm -f prov/util/src/libgnix_fi_la-util_attr.lo
+ -rm -f prov/util/src/libgnix_fi_la-util_av.$(OBJEXT)
+ -rm -f prov/util/src/libgnix_fi_la-util_av.lo
+ -rm -f prov/util/src/libgnix_fi_la-util_buf.$(OBJEXT)
+ -rm -f prov/util/src/libgnix_fi_la-util_buf.lo
+ -rm -f prov/util/src/libgnix_fi_la-util_cq.$(OBJEXT)
+ -rm -f prov/util/src/libgnix_fi_la-util_cq.lo
+ -rm -f prov/util/src/libgnix_fi_la-util_domain.$(OBJEXT)
+ -rm -f prov/util/src/libgnix_fi_la-util_domain.lo
+ -rm -f prov/util/src/libgnix_fi_la-util_ep.$(OBJEXT)
+ -rm -f prov/util/src/libgnix_fi_la-util_ep.lo
+ -rm -f prov/util/src/libgnix_fi_la-util_eq.$(OBJEXT)
+ -rm -f prov/util/src/libgnix_fi_la-util_eq.lo
+ -rm -f prov/util/src/libgnix_fi_la-util_fabric.$(OBJEXT)
+ -rm -f prov/util/src/libgnix_fi_la-util_fabric.lo
+ -rm -f prov/util/src/libgnix_fi_la-util_main.$(OBJEXT)
+ -rm -f prov/util/src/libgnix_fi_la-util_main.lo
+ -rm -f prov/util/src/libgnix_fi_la-util_mr.$(OBJEXT)
+ -rm -f prov/util/src/libgnix_fi_la-util_mr.lo
+ -rm -f prov/util/src/libgnix_fi_la-util_poll.$(OBJEXT)
+ -rm -f prov/util/src/libgnix_fi_la-util_poll.lo
+ -rm -f prov/util/src/libgnix_fi_la-util_wait.$(OBJEXT)
+ -rm -f prov/util/src/libgnix_fi_la-util_wait.lo
+ -rm -f prov/util/src/libmlxm_fi_la-util_attr.$(OBJEXT)
+ -rm -f prov/util/src/libmlxm_fi_la-util_attr.lo
+ -rm -f prov/util/src/libmlxm_fi_la-util_av.$(OBJEXT)
+ -rm -f prov/util/src/libmlxm_fi_la-util_av.lo
+ -rm -f prov/util/src/libmlxm_fi_la-util_buf.$(OBJEXT)
+ -rm -f prov/util/src/libmlxm_fi_la-util_buf.lo
+ -rm -f prov/util/src/libmlxm_fi_la-util_cq.$(OBJEXT)
+ -rm -f prov/util/src/libmlxm_fi_la-util_cq.lo
+ -rm -f prov/util/src/libmlxm_fi_la-util_domain.$(OBJEXT)
+ -rm -f prov/util/src/libmlxm_fi_la-util_domain.lo
+ -rm -f prov/util/src/libmlxm_fi_la-util_ep.$(OBJEXT)
+ -rm -f prov/util/src/libmlxm_fi_la-util_ep.lo
+ -rm -f prov/util/src/libmlxm_fi_la-util_eq.$(OBJEXT)
+ -rm -f prov/util/src/libmlxm_fi_la-util_eq.lo
+ -rm -f prov/util/src/libmlxm_fi_la-util_fabric.$(OBJEXT)
+ -rm -f prov/util/src/libmlxm_fi_la-util_fabric.lo
+ -rm -f prov/util/src/libmlxm_fi_la-util_main.$(OBJEXT)
+ -rm -f prov/util/src/libmlxm_fi_la-util_main.lo
+ -rm -f prov/util/src/libmlxm_fi_la-util_mr.$(OBJEXT)
+ -rm -f prov/util/src/libmlxm_fi_la-util_mr.lo
+ -rm -f prov/util/src/libmlxm_fi_la-util_poll.$(OBJEXT)
+ -rm -f prov/util/src/libmlxm_fi_la-util_poll.lo
+ -rm -f prov/util/src/libmlxm_fi_la-util_wait.$(OBJEXT)
+ -rm -f prov/util/src/libmlxm_fi_la-util_wait.lo
+ -rm -f prov/util/src/libpsmx2_fi_la-util_attr.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx2_fi_la-util_attr.lo
+ -rm -f prov/util/src/libpsmx2_fi_la-util_av.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx2_fi_la-util_av.lo
+ -rm -f prov/util/src/libpsmx2_fi_la-util_buf.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx2_fi_la-util_buf.lo
+ -rm -f prov/util/src/libpsmx2_fi_la-util_cq.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx2_fi_la-util_cq.lo
+ -rm -f prov/util/src/libpsmx2_fi_la-util_domain.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx2_fi_la-util_domain.lo
+ -rm -f prov/util/src/libpsmx2_fi_la-util_ep.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx2_fi_la-util_ep.lo
+ -rm -f prov/util/src/libpsmx2_fi_la-util_eq.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx2_fi_la-util_eq.lo
+ -rm -f prov/util/src/libpsmx2_fi_la-util_fabric.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx2_fi_la-util_fabric.lo
+ -rm -f prov/util/src/libpsmx2_fi_la-util_main.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx2_fi_la-util_main.lo
+ -rm -f prov/util/src/libpsmx2_fi_la-util_mr.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx2_fi_la-util_mr.lo
+ -rm -f prov/util/src/libpsmx2_fi_la-util_poll.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx2_fi_la-util_poll.lo
+ -rm -f prov/util/src/libpsmx2_fi_la-util_wait.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx2_fi_la-util_wait.lo
+ -rm -f prov/util/src/libpsmx_fi_la-util_attr.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx_fi_la-util_attr.lo
+ -rm -f prov/util/src/libpsmx_fi_la-util_av.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx_fi_la-util_av.lo
+ -rm -f prov/util/src/libpsmx_fi_la-util_buf.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx_fi_la-util_buf.lo
+ -rm -f prov/util/src/libpsmx_fi_la-util_cq.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx_fi_la-util_cq.lo
+ -rm -f prov/util/src/libpsmx_fi_la-util_domain.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx_fi_la-util_domain.lo
+ -rm -f prov/util/src/libpsmx_fi_la-util_ep.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx_fi_la-util_ep.lo
+ -rm -f prov/util/src/libpsmx_fi_la-util_eq.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx_fi_la-util_eq.lo
+ -rm -f prov/util/src/libpsmx_fi_la-util_fabric.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx_fi_la-util_fabric.lo
+ -rm -f prov/util/src/libpsmx_fi_la-util_main.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx_fi_la-util_main.lo
+ -rm -f prov/util/src/libpsmx_fi_la-util_mr.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx_fi_la-util_mr.lo
+ -rm -f prov/util/src/libpsmx_fi_la-util_poll.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx_fi_la-util_poll.lo
+ -rm -f prov/util/src/libpsmx_fi_la-util_wait.$(OBJEXT)
+ -rm -f prov/util/src/libpsmx_fi_la-util_wait.lo
+ -rm -f prov/util/src/libusnic_fi_la-util_attr.$(OBJEXT)
+ -rm -f prov/util/src/libusnic_fi_la-util_attr.lo
+ -rm -f prov/util/src/libusnic_fi_la-util_av.$(OBJEXT)
+ -rm -f prov/util/src/libusnic_fi_la-util_av.lo
+ -rm -f prov/util/src/libusnic_fi_la-util_buf.$(OBJEXT)
+ -rm -f prov/util/src/libusnic_fi_la-util_buf.lo
+ -rm -f prov/util/src/libusnic_fi_la-util_cq.$(OBJEXT)
+ -rm -f prov/util/src/libusnic_fi_la-util_cq.lo
+ -rm -f prov/util/src/libusnic_fi_la-util_domain.$(OBJEXT)
+ -rm -f prov/util/src/libusnic_fi_la-util_domain.lo
+ -rm -f prov/util/src/libusnic_fi_la-util_ep.$(OBJEXT)
+ -rm -f prov/util/src/libusnic_fi_la-util_ep.lo
+ -rm -f prov/util/src/libusnic_fi_la-util_eq.$(OBJEXT)
+ -rm -f prov/util/src/libusnic_fi_la-util_eq.lo
+ -rm -f prov/util/src/libusnic_fi_la-util_fabric.$(OBJEXT)
+ -rm -f prov/util/src/libusnic_fi_la-util_fabric.lo
+ -rm -f prov/util/src/libusnic_fi_la-util_main.$(OBJEXT)
+ -rm -f prov/util/src/libusnic_fi_la-util_main.lo
+ -rm -f prov/util/src/libusnic_fi_la-util_mr.$(OBJEXT)
+ -rm -f prov/util/src/libusnic_fi_la-util_mr.lo
+ -rm -f prov/util/src/libusnic_fi_la-util_poll.$(OBJEXT)
+ -rm -f prov/util/src/libusnic_fi_la-util_poll.lo
+ -rm -f prov/util/src/libusnic_fi_la-util_wait.$(OBJEXT)
+ -rm -f prov/util/src/libusnic_fi_la-util_wait.lo
+ -rm -f prov/util/src/libverbs_fi_la-util_attr.$(OBJEXT)
+ -rm -f prov/util/src/libverbs_fi_la-util_attr.lo
+ -rm -f prov/util/src/libverbs_fi_la-util_av.$(OBJEXT)
+ -rm -f prov/util/src/libverbs_fi_la-util_av.lo
+ -rm -f prov/util/src/libverbs_fi_la-util_buf.$(OBJEXT)
+ -rm -f prov/util/src/libverbs_fi_la-util_buf.lo
+ -rm -f prov/util/src/libverbs_fi_la-util_cq.$(OBJEXT)
+ -rm -f prov/util/src/libverbs_fi_la-util_cq.lo
+ -rm -f prov/util/src/libverbs_fi_la-util_domain.$(OBJEXT)
+ -rm -f prov/util/src/libverbs_fi_la-util_domain.lo
+ -rm -f prov/util/src/libverbs_fi_la-util_ep.$(OBJEXT)
+ -rm -f prov/util/src/libverbs_fi_la-util_ep.lo
+ -rm -f prov/util/src/libverbs_fi_la-util_eq.$(OBJEXT)
+ -rm -f prov/util/src/libverbs_fi_la-util_eq.lo
+ -rm -f prov/util/src/libverbs_fi_la-util_fabric.$(OBJEXT)
+ -rm -f prov/util/src/libverbs_fi_la-util_fabric.lo
+ -rm -f prov/util/src/libverbs_fi_la-util_main.$(OBJEXT)
+ -rm -f prov/util/src/libverbs_fi_la-util_main.lo
+ -rm -f prov/util/src/libverbs_fi_la-util_mr.$(OBJEXT)
+ -rm -f prov/util/src/libverbs_fi_la-util_mr.lo
+ -rm -f prov/util/src/libverbs_fi_la-util_poll.$(OBJEXT)
+ -rm -f prov/util/src/libverbs_fi_la-util_poll.lo
+ -rm -f prov/util/src/libverbs_fi_la-util_wait.$(OBJEXT)
+ -rm -f prov/util/src/libverbs_fi_la-util_wait.lo
+ -rm -f prov/util/src/src_libfabric_la-util_attr.$(OBJEXT)
+ -rm -f prov/util/src/src_libfabric_la-util_attr.lo
+ -rm -f prov/util/src/src_libfabric_la-util_av.$(OBJEXT)
+ -rm -f prov/util/src/src_libfabric_la-util_av.lo
+ -rm -f prov/util/src/src_libfabric_la-util_buf.$(OBJEXT)
+ -rm -f prov/util/src/src_libfabric_la-util_buf.lo
+ -rm -f prov/util/src/src_libfabric_la-util_cq.$(OBJEXT)
+ -rm -f prov/util/src/src_libfabric_la-util_cq.lo
+ -rm -f prov/util/src/src_libfabric_la-util_domain.$(OBJEXT)
+ -rm -f prov/util/src/src_libfabric_la-util_domain.lo
+ -rm -f prov/util/src/src_libfabric_la-util_ep.$(OBJEXT)
+ -rm -f prov/util/src/src_libfabric_la-util_ep.lo
+ -rm -f prov/util/src/src_libfabric_la-util_eq.$(OBJEXT)
+ -rm -f prov/util/src/src_libfabric_la-util_eq.lo
+ -rm -f prov/util/src/src_libfabric_la-util_fabric.$(OBJEXT)
+ -rm -f prov/util/src/src_libfabric_la-util_fabric.lo
+ -rm -f prov/util/src/src_libfabric_la-util_main.$(OBJEXT)
+ -rm -f prov/util/src/src_libfabric_la-util_main.lo
+ -rm -f prov/util/src/src_libfabric_la-util_mr.$(OBJEXT)
+ -rm -f prov/util/src/src_libfabric_la-util_mr.lo
+ -rm -f prov/util/src/src_libfabric_la-util_poll.$(OBJEXT)
+ -rm -f prov/util/src/src_libfabric_la-util_poll.lo
+ -rm -f prov/util/src/src_libfabric_la-util_wait.$(OBJEXT)
+ -rm -f prov/util/src/src_libfabric_la-util_wait.lo
+ -rm -f prov/util/src/util_attr.$(OBJEXT)
+ -rm -f prov/util/src/util_attr.lo
+ -rm -f prov/util/src/util_av.$(OBJEXT)
+ -rm -f prov/util/src/util_av.lo
+ -rm -f prov/util/src/util_buf.$(OBJEXT)
+ -rm -f prov/util/src/util_buf.lo
+ -rm -f prov/util/src/util_cq.$(OBJEXT)
+ -rm -f prov/util/src/util_cq.lo
+ -rm -f prov/util/src/util_domain.$(OBJEXT)
+ -rm -f prov/util/src/util_domain.lo
+ -rm -f prov/util/src/util_ep.$(OBJEXT)
+ -rm -f prov/util/src/util_ep.lo
+ -rm -f prov/util/src/util_eq.$(OBJEXT)
+ -rm -f prov/util/src/util_eq.lo
+ -rm -f prov/util/src/util_fabric.$(OBJEXT)
+ -rm -f prov/util/src/util_fabric.lo
+ -rm -f prov/util/src/util_main.$(OBJEXT)
+ -rm -f prov/util/src/util_main.lo
+ -rm -f prov/util/src/util_mr.$(OBJEXT)
+ -rm -f prov/util/src/util_mr.lo
+ -rm -f prov/util/src/util_poll.$(OBJEXT)
+ -rm -f prov/util/src/util_poll.lo
+ -rm -f prov/util/src/util_wait.$(OBJEXT)
+ -rm -f prov/util/src/util_wait.lo
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_av_ep_rdm.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_av_ep_rdm.lo
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_cq_ep_rdm.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_cq_ep_rdm.lo
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_ep_rdm.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_ep_rdm.lo
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cm.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cm.lo
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cntr.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cntr.lo
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_msg.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_msg.lo
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm.lo
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm_states.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm_states.lo
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_utils.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_utils.lo
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_av_ep_rdm.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_av_ep_rdm.lo
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_cq_ep_rdm.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_cq_ep_rdm.lo
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_ep_rdm.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_ep_rdm.lo
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cm.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cm.lo
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cntr.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cntr.lo
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_msg.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_msg.lo
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm.lo
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm_states.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm_states.lo
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_utils.$(OBJEXT)
+ -rm -f prov/verbs/src/ep_rdm/src_libfabric_la-verbs_utils.lo
+ -rm -f prov/verbs/src/libverbs_fi_la-fi_verbs.$(OBJEXT)
+ -rm -f prov/verbs/src/libverbs_fi_la-fi_verbs.lo
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_atomic.$(OBJEXT)
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_atomic.lo
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_av.$(OBJEXT)
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_av.lo
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_cm.$(OBJEXT)
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_cm.lo
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_cq.$(OBJEXT)
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_cq.lo
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_domain.$(OBJEXT)
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_domain.lo
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_eq.$(OBJEXT)
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_eq.lo
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_info.$(OBJEXT)
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_info.lo
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_msg.$(OBJEXT)
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_msg.lo
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_msg_ep.$(OBJEXT)
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_msg_ep.lo
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_rma.$(OBJEXT)
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_rma.lo
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_srq.$(OBJEXT)
+ -rm -f prov/verbs/src/libverbs_fi_la-verbs_srq.lo
+ -rm -f prov/verbs/src/src_libfabric_la-fi_verbs.$(OBJEXT)
+ -rm -f prov/verbs/src/src_libfabric_la-fi_verbs.lo
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_atomic.$(OBJEXT)
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_atomic.lo
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_av.$(OBJEXT)
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_av.lo
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_cm.$(OBJEXT)
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_cm.lo
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_cq.$(OBJEXT)
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_cq.lo
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_domain.$(OBJEXT)
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_domain.lo
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_eq.$(OBJEXT)
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_eq.lo
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_info.$(OBJEXT)
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_info.lo
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_msg.$(OBJEXT)
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_msg.lo
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_msg_ep.$(OBJEXT)
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_msg_ep.lo
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_rma.$(OBJEXT)
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_rma.lo
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_srq.$(OBJEXT)
+ -rm -f prov/verbs/src/src_libfabric_la-verbs_srq.lo
+ -rm -f src/common.$(OBJEXT)
+ -rm -f src/common.lo
+ -rm -f src/enosys.$(OBJEXT)
+ -rm -f src/enosys.lo
+ -rm -f src/fasthash.$(OBJEXT)
+ -rm -f src/fasthash.lo
+ -rm -f src/indexer.$(OBJEXT)
+ -rm -f src/indexer.lo
+ -rm -f src/libgnix_fi_la-common.$(OBJEXT)
+ -rm -f src/libgnix_fi_la-common.lo
+ -rm -f src/libgnix_fi_la-enosys.$(OBJEXT)
+ -rm -f src/libgnix_fi_la-enosys.lo
+ -rm -f src/libgnix_fi_la-fasthash.$(OBJEXT)
+ -rm -f src/libgnix_fi_la-fasthash.lo
+ -rm -f src/libgnix_fi_la-indexer.$(OBJEXT)
+ -rm -f src/libgnix_fi_la-indexer.lo
+ -rm -f src/libgnix_fi_la-rbtree.$(OBJEXT)
+ -rm -f src/libgnix_fi_la-rbtree.lo
+ -rm -f src/libmlxm_fi_la-common.$(OBJEXT)
+ -rm -f src/libmlxm_fi_la-common.lo
+ -rm -f src/libmlxm_fi_la-enosys.$(OBJEXT)
+ -rm -f src/libmlxm_fi_la-enosys.lo
+ -rm -f src/libmlxm_fi_la-fasthash.$(OBJEXT)
+ -rm -f src/libmlxm_fi_la-fasthash.lo
+ -rm -f src/libmlxm_fi_la-indexer.$(OBJEXT)
+ -rm -f src/libmlxm_fi_la-indexer.lo
+ -rm -f src/libmlxm_fi_la-rbtree.$(OBJEXT)
+ -rm -f src/libmlxm_fi_la-rbtree.lo
+ -rm -f src/libpsmx2_fi_la-common.$(OBJEXT)
+ -rm -f src/libpsmx2_fi_la-common.lo
+ -rm -f src/libpsmx2_fi_la-enosys.$(OBJEXT)
+ -rm -f src/libpsmx2_fi_la-enosys.lo
+ -rm -f src/libpsmx2_fi_la-fasthash.$(OBJEXT)
+ -rm -f src/libpsmx2_fi_la-fasthash.lo
+ -rm -f src/libpsmx2_fi_la-indexer.$(OBJEXT)
+ -rm -f src/libpsmx2_fi_la-indexer.lo
+ -rm -f src/libpsmx2_fi_la-rbtree.$(OBJEXT)
+ -rm -f src/libpsmx2_fi_la-rbtree.lo
+ -rm -f src/libpsmx_fi_la-common.$(OBJEXT)
+ -rm -f src/libpsmx_fi_la-common.lo
+ -rm -f src/libpsmx_fi_la-enosys.$(OBJEXT)
+ -rm -f src/libpsmx_fi_la-enosys.lo
+ -rm -f src/libpsmx_fi_la-fasthash.$(OBJEXT)
+ -rm -f src/libpsmx_fi_la-fasthash.lo
+ -rm -f src/libpsmx_fi_la-indexer.$(OBJEXT)
+ -rm -f src/libpsmx_fi_la-indexer.lo
+ -rm -f src/libpsmx_fi_la-rbtree.$(OBJEXT)
+ -rm -f src/libpsmx_fi_la-rbtree.lo
+ -rm -f src/libusnic_fi_la-common.$(OBJEXT)
+ -rm -f src/libusnic_fi_la-common.lo
+ -rm -f src/libusnic_fi_la-enosys.$(OBJEXT)
+ -rm -f src/libusnic_fi_la-enosys.lo
+ -rm -f src/libusnic_fi_la-fasthash.$(OBJEXT)
+ -rm -f src/libusnic_fi_la-fasthash.lo
+ -rm -f src/libusnic_fi_la-indexer.$(OBJEXT)
+ -rm -f src/libusnic_fi_la-indexer.lo
+ -rm -f src/libusnic_fi_la-rbtree.$(OBJEXT)
+ -rm -f src/libusnic_fi_la-rbtree.lo
+ -rm -f src/libverbs_fi_la-common.$(OBJEXT)
+ -rm -f src/libverbs_fi_la-common.lo
+ -rm -f src/libverbs_fi_la-enosys.$(OBJEXT)
+ -rm -f src/libverbs_fi_la-enosys.lo
+ -rm -f src/libverbs_fi_la-fasthash.$(OBJEXT)
+ -rm -f src/libverbs_fi_la-fasthash.lo
+ -rm -f src/libverbs_fi_la-indexer.$(OBJEXT)
+ -rm -f src/libverbs_fi_la-indexer.lo
+ -rm -f src/libverbs_fi_la-rbtree.$(OBJEXT)
+ -rm -f src/libverbs_fi_la-rbtree.lo
+ -rm -f src/osx/libgnix_fi_la-osd.$(OBJEXT)
+ -rm -f src/osx/libgnix_fi_la-osd.lo
+ -rm -f src/osx/libmlxm_fi_la-osd.$(OBJEXT)
+ -rm -f src/osx/libmlxm_fi_la-osd.lo
+ -rm -f src/osx/libpsmx2_fi_la-osd.$(OBJEXT)
+ -rm -f src/osx/libpsmx2_fi_la-osd.lo
+ -rm -f src/osx/libpsmx_fi_la-osd.$(OBJEXT)
+ -rm -f src/osx/libpsmx_fi_la-osd.lo
+ -rm -f src/osx/libusnic_fi_la-osd.$(OBJEXT)
+ -rm -f src/osx/libusnic_fi_la-osd.lo
+ -rm -f src/osx/libverbs_fi_la-osd.$(OBJEXT)
+ -rm -f src/osx/libverbs_fi_la-osd.lo
+ -rm -f src/osx/osd.$(OBJEXT)
+ -rm -f src/osx/osd.lo
+ -rm -f src/osx/src_libfabric_la-osd.$(OBJEXT)
+ -rm -f src/osx/src_libfabric_la-osd.lo
+ -rm -f src/rbtree.$(OBJEXT)
+ -rm -f src/rbtree.lo
+ -rm -f src/src_libfabric_la-common.$(OBJEXT)
+ -rm -f src/src_libfabric_la-common.lo
+ -rm -f src/src_libfabric_la-enosys.$(OBJEXT)
+ -rm -f src/src_libfabric_la-enosys.lo
+ -rm -f src/src_libfabric_la-fabric.$(OBJEXT)
+ -rm -f src/src_libfabric_la-fabric.lo
+ -rm -f src/src_libfabric_la-fasthash.$(OBJEXT)
+ -rm -f src/src_libfabric_la-fasthash.lo
+ -rm -f src/src_libfabric_la-fi_tostr.$(OBJEXT)
+ -rm -f src/src_libfabric_la-fi_tostr.lo
+ -rm -f src/src_libfabric_la-indexer.$(OBJEXT)
+ -rm -f src/src_libfabric_la-indexer.lo
+ -rm -f src/src_libfabric_la-log.$(OBJEXT)
+ -rm -f src/src_libfabric_la-log.lo
+ -rm -f src/src_libfabric_la-rbtree.$(OBJEXT)
+ -rm -f src/src_libfabric_la-rbtree.lo
+ -rm -f src/src_libfabric_la-var.$(OBJEXT)
+ -rm -f src/src_libfabric_la-var.lo
+ -rm -f src/unix/libgnix_fi_la-osd.$(OBJEXT)
+ -rm -f src/unix/libgnix_fi_la-osd.lo
+ -rm -f src/unix/libmlxm_fi_la-osd.$(OBJEXT)
+ -rm -f src/unix/libmlxm_fi_la-osd.lo
+ -rm -f src/unix/libpsmx2_fi_la-osd.$(OBJEXT)
+ -rm -f src/unix/libpsmx2_fi_la-osd.lo
+ -rm -f src/unix/libpsmx_fi_la-osd.$(OBJEXT)
+ -rm -f src/unix/libpsmx_fi_la-osd.lo
+ -rm -f src/unix/libusnic_fi_la-osd.$(OBJEXT)
+ -rm -f src/unix/libusnic_fi_la-osd.lo
+ -rm -f src/unix/libverbs_fi_la-osd.$(OBJEXT)
+ -rm -f src/unix/libverbs_fi_la-osd.lo
+ -rm -f src/unix/osd.$(OBJEXT)
+ -rm -f src/unix/osd.lo
+ -rm -f src/unix/src_libfabric_la-osd.$(OBJEXT)
+ -rm -f src/unix/src_libfabric_la-osd.lo
+ -rm -f util/info.$(OBJEXT)
+ -rm -f util/pingpong.$(OBJEXT)
+ -rm -f util/strerror.$(OBJEXT)
+
+distclean-compile:
+ -rm -f *.tab.c
+
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_atomic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_bitmap.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_buddy_allocator.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cm_nic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cntr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_datagram.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_dom.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_freelist.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_hashtable.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mbox_allocator.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mr_cache.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mr_notifier.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_nameserver.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_nic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_queue.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_rma.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_tags.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_trigger.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_util.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_vc.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_vector.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_xpmem.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_atomic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_bitmap.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_buddy_allocator.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cm_nic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cntr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_datagram.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_dom.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_freelist.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_hashtable.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mbox_allocator.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mr_cache.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mr_notifier.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_nameserver.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_nic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_queue.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_rma.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_tags.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_trigger.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_util.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_vc.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_vector.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_xpmem.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-allocator.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cntr.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cq.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-av.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-bitmap.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-buddy_allocator.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cancel.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cntr.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-common.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cq.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-datagram.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dlist-utils.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dom.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-ep.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-eq.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-freelist.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-hashtable.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr_notifier.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-nic.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-pmi_utils.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-queue.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_atomic.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_dgram_rma.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_rx_overrun.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_sr.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_tagged_sr.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-tags.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-utils.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc_lookup.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vector.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-wait.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_tagged.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_tagged.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_am.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_atomic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_cntr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_msg2.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_rma.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_tagged.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_util.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_am.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_atomic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_cntr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_msg2.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_rma.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_tagged.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_util.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_am.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_atomic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_cntr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_rma.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_tagged.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_util.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_am.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_atomic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_cntr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_rma.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_tagged.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_util.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/rxd_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/rxd_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/rxd_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/rxd_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/rxd_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/rxd_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/rxd_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/rxd_rma.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_rma.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/rxm_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/rxm_conn.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/rxm_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/rxm_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/rxm_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/rxm_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/rxm_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_conn.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_atomic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_cntr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_comm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_conn.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_ctx.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_dom.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_ep_dgram.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_ep_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_ep_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_epoll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_progress.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_rma.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_rx_entry.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_trigger.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/sock_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_atomic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_cntr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_comm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_conn.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ctx.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_dom.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep_dgram.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_epoll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_progress.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_rma.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_rx_entry.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_trigger.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/udp/src/$(DEPDIR)/udpx_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/udp/src/$(DEPDIR)/udpx_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/udp/src/$(DEPDIR)/udpx_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/udp/src/$(DEPDIR)/udpx_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/udp/src/$(DEPDIR)/udpx_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/udp/src/$(DEPDIR)/udpx_init.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_dgram.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_endpoint.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ep_dgram.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ep_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ep_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ext.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_fake_ibv.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_mem.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_pep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_progress.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_timer.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_dgram.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_endpoint.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ep_dgram.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ep_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ep_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ext.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_fake_ibv.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_mem.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_pep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_progress.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_timer.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-libnl_utils_common.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_caps.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_dest.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_device.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_enum.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_event.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_ib_cmd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_ib_sysfs.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_mem.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post_ud_pio_udp.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post_ud_raw.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post_ud_udp.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_queues.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_socket.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_vnic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usnic_ip_utils.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_dev.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_intr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_rq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_wq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-libnl_utils_common.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_caps.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_dest.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_device.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_enum.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_event.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_ib_cmd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_ib_sysfs.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_mem.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post_ud_pio_udp.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post_ud_raw.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post_ud_udp.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_queues.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_socket.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_vnic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usnic_ip_utils.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_dev.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_intr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_rq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_wq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libgnix_fi_la-util_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libgnix_fi_la-util_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libgnix_fi_la-util_buf.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libgnix_fi_la-util_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libgnix_fi_la-util_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libgnix_fi_la-util_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libgnix_fi_la-util_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libgnix_fi_la-util_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libgnix_fi_la-util_main.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libgnix_fi_la-util_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libgnix_fi_la-util_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libgnix_fi_la-util_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_buf.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_main.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_buf.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_main.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_buf.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_main.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libusnic_fi_la-util_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libusnic_fi_la-util_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libusnic_fi_la-util_buf.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libusnic_fi_la-util_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libusnic_fi_la-util_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libusnic_fi_la-util_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libusnic_fi_la-util_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libusnic_fi_la-util_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libusnic_fi_la-util_main.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libusnic_fi_la-util_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libusnic_fi_la-util_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libusnic_fi_la-util_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libverbs_fi_la-util_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libverbs_fi_la-util_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libverbs_fi_la-util_buf.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libverbs_fi_la-util_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libverbs_fi_la-util_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libverbs_fi_la-util_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libverbs_fi_la-util_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libverbs_fi_la-util_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libverbs_fi_la-util_main.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libverbs_fi_la-util_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libverbs_fi_la-util_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/libverbs_fi_la-util_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/src_libfabric_la-util_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/src_libfabric_la-util_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/src_libfabric_la-util_buf.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/src_libfabric_la-util_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/src_libfabric_la-util_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/src_libfabric_la-util_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/src_libfabric_la-util_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/src_libfabric_la-util_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/src_libfabric_la-util_main.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/src_libfabric_la-util_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/src_libfabric_la-util_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/src_libfabric_la-util_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/util_attr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/util_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/util_buf.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/util_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/util_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/util_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/util_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/util_fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/util_main.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/util_mr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/util_poll.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/util/src/$(DEPDIR)/util_wait.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/libverbs_fi_la-fi_verbs.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_atomic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_info.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_msg_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_rma.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_srq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/src_libfabric_la-fi_verbs.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_atomic.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_av.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_cq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_domain.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_eq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_info.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_msg_ep.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_rma.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_srq.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_av_ep_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_cq_ep_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_ep_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_rdm_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_rdm_cntr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_rdm_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_tagged_ep_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_tagged_ep_rdm_states.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_utils.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_av_ep_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_cq_ep_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_ep_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_rdm_cm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_rdm_cntr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_rdm_msg.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_tagged_ep_rdm.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_tagged_ep_rdm_states.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_utils.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/common.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/enosys.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/fasthash.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/indexer.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libgnix_fi_la-common.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libgnix_fi_la-enosys.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libgnix_fi_la-fasthash.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libgnix_fi_la-indexer.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libgnix_fi_la-rbtree.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libmlxm_fi_la-common.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libmlxm_fi_la-enosys.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libmlxm_fi_la-fasthash.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libmlxm_fi_la-indexer.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libmlxm_fi_la-rbtree.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libpsmx2_fi_la-common.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libpsmx2_fi_la-enosys.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libpsmx2_fi_la-fasthash.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libpsmx2_fi_la-indexer.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libpsmx2_fi_la-rbtree.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libpsmx_fi_la-common.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libpsmx_fi_la-enosys.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libpsmx_fi_la-fasthash.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libpsmx_fi_la-indexer.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libpsmx_fi_la-rbtree.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libusnic_fi_la-common.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libusnic_fi_la-enosys.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libusnic_fi_la-fasthash.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libusnic_fi_la-indexer.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libusnic_fi_la-rbtree.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libverbs_fi_la-common.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libverbs_fi_la-enosys.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libverbs_fi_la-fasthash.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libverbs_fi_la-indexer.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/libverbs_fi_la-rbtree.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/rbtree.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/src_libfabric_la-common.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/src_libfabric_la-enosys.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/src_libfabric_la-fabric.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/src_libfabric_la-fasthash.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/src_libfabric_la-fi_tostr.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/src_libfabric_la-indexer.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/src_libfabric_la-log.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/src_libfabric_la-rbtree.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/src_libfabric_la-var.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/osx/$(DEPDIR)/libgnix_fi_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/osx/$(DEPDIR)/libmlxm_fi_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/osx/$(DEPDIR)/libpsmx2_fi_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/osx/$(DEPDIR)/libpsmx_fi_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/osx/$(DEPDIR)/libusnic_fi_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/osx/$(DEPDIR)/libverbs_fi_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/osx/$(DEPDIR)/osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/osx/$(DEPDIR)/src_libfabric_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/unix/$(DEPDIR)/libgnix_fi_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/unix/$(DEPDIR)/libmlxm_fi_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/unix/$(DEPDIR)/libpsmx2_fi_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/unix/$(DEPDIR)/libpsmx_fi_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/unix/$(DEPDIR)/libusnic_fi_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/unix/$(DEPDIR)/libverbs_fi_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/unix/$(DEPDIR)/osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at src/unix/$(DEPDIR)/src_libfabric_la-osd.Plo at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at util/$(DEPDIR)/info.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at util/$(DEPDIR)/pingpong.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at util/$(DEPDIR)/strerror.Po at am__quote@
+
+.c.o:
+ at am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.o$$||'`;\
+ at am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\
+ at am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(COMPILE) -c -o $@ $<
+
+.c.obj:
+ at am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.obj$$||'`;\
+ at am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ `$(CYGPATH_W) '$<'` &&\
+ at am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+ at am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.lo$$||'`;\
+ at am__fastdepCC_TRUE@ $(LTCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\
+ at am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LTCOMPILE) -c -o $@ $<
+
+prov/gni/src/libgnix_fi_la-gnix_atomic.lo: prov/gni/src/gnix_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_atomic.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_atomic.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_atomic.lo `test -f 'prov/gni/src/gnix_atomic.c' || echo '$(srcdir)/'`prov/gni/src/gnix_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_atomic.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_atomic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_atomic.c' object='prov/gni/src/libgnix_fi_la-gnix_atomic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_atomic.lo `test -f 'prov/gni/src/gnix_atomic.c' || echo '$(srcdir)/'`prov/gni/src/gnix_atomic.c
+
+prov/gni/src/libgnix_fi_la-gnix_av.lo: prov/gni/src/gnix_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_av.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_av.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_av.lo `test -f 'prov/gni/src/gnix_av.c' || echo '$(srcdir)/'`prov/gni/src/gnix_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_av.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_av.c' object='prov/gni/src/libgnix_fi_la-gnix_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_av.lo `test -f 'prov/gni/src/gnix_av.c' || echo '$(srcdir)/'`prov/gni/src/gnix_av.c
+
+prov/gni/src/libgnix_fi_la-gnix_bitmap.lo: prov/gni/src/gnix_bitmap.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_bitmap.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_bitmap.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_bitmap.lo `test -f 'prov/gni/src/gnix_bitmap.c' || echo '$(srcdir)/'`prov/gni/src/gnix_bitmap.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_bitmap.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_bitmap.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_bitmap.c' object='prov/gni/src/libgnix_fi_la-gnix_bitmap.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_bitmap.lo `test -f 'prov/gni/src/gnix_bitmap.c' || echo '$(srcdir)/'`prov/gni/src/gnix_bitmap.c
+
+prov/gni/src/libgnix_fi_la-gnix_buddy_allocator.lo: prov/gni/src/gnix_buddy_allocator.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_buddy_allocator.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_buddy_allocator.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_buddy_allocator.lo `test -f 'prov/gni/src/gnix_buddy_allocator.c' || echo '$(srcdir)/'`prov/gni/src/gnix_b [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_buddy_allocator.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_buddy_allocator.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_buddy_allocator.c' object='prov/gni/src/libgnix_fi_la-gnix_buddy_allocator.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_buddy_allocator.lo `test -f 'prov/gni/src/gnix_buddy_allocator.c' || echo '$(srcdir)/'`prov/gni/src/gnix_buddy_allocator.c
+
+prov/gni/src/libgnix_fi_la-gnix_cm.lo: prov/gni/src/gnix_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_cm.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cm.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_cm.lo `test -f 'prov/gni/src/gnix_cm.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cm.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_cm.c' object='prov/gni/src/libgnix_fi_la-gnix_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_cm.lo `test -f 'prov/gni/src/gnix_cm.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cm.c
+
+prov/gni/src/libgnix_fi_la-gnix_cm_nic.lo: prov/gni/src/gnix_cm_nic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_cm_nic.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cm_nic.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_cm_nic.lo `test -f 'prov/gni/src/gnix_cm_nic.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cm_nic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cm_nic.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cm_nic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_cm_nic.c' object='prov/gni/src/libgnix_fi_la-gnix_cm_nic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_cm_nic.lo `test -f 'prov/gni/src/gnix_cm_nic.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cm_nic.c
+
+prov/gni/src/libgnix_fi_la-gnix_cntr.lo: prov/gni/src/gnix_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_cntr.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cntr.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_cntr.lo `test -f 'prov/gni/src/gnix_cntr.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cntr.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cntr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_cntr.c' object='prov/gni/src/libgnix_fi_la-gnix_cntr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_cntr.lo `test -f 'prov/gni/src/gnix_cntr.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cntr.c
+
+prov/gni/src/libgnix_fi_la-gnix_cq.lo: prov/gni/src/gnix_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_cq.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cq.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_cq.lo `test -f 'prov/gni/src/gnix_cq.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cq.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_cq.c' object='prov/gni/src/libgnix_fi_la-gnix_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_cq.lo `test -f 'prov/gni/src/gnix_cq.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cq.c
+
+prov/gni/src/libgnix_fi_la-gnix_datagram.lo: prov/gni/src/gnix_datagram.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_datagram.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_datagram.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_datagram.lo `test -f 'prov/gni/src/gnix_datagram.c' || echo '$(srcdir)/'`prov/gni/src/gnix_datagram.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_datagram.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_datagram.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_datagram.c' object='prov/gni/src/libgnix_fi_la-gnix_datagram.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_datagram.lo `test -f 'prov/gni/src/gnix_datagram.c' || echo '$(srcdir)/'`prov/gni/src/gnix_datagram.c
+
+prov/gni/src/libgnix_fi_la-gnix_dom.lo: prov/gni/src/gnix_dom.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_dom.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_dom.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_dom.lo `test -f 'prov/gni/src/gnix_dom.c' || echo '$(srcdir)/'`prov/gni/src/gnix_dom.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_dom.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_dom.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_dom.c' object='prov/gni/src/libgnix_fi_la-gnix_dom.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_dom.lo `test -f 'prov/gni/src/gnix_dom.c' || echo '$(srcdir)/'`prov/gni/src/gnix_dom.c
+
+prov/gni/src/libgnix_fi_la-gnix_ep.lo: prov/gni/src/gnix_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_ep.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_ep.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_ep.lo `test -f 'prov/gni/src/gnix_ep.c' || echo '$(srcdir)/'`prov/gni/src/gnix_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_ep.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_ep.c' object='prov/gni/src/libgnix_fi_la-gnix_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_ep.lo `test -f 'prov/gni/src/gnix_ep.c' || echo '$(srcdir)/'`prov/gni/src/gnix_ep.c
+
+prov/gni/src/libgnix_fi_la-gnix_eq.lo: prov/gni/src/gnix_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_eq.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_eq.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_eq.lo `test -f 'prov/gni/src/gnix_eq.c' || echo '$(srcdir)/'`prov/gni/src/gnix_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_eq.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_eq.c' object='prov/gni/src/libgnix_fi_la-gnix_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_eq.lo `test -f 'prov/gni/src/gnix_eq.c' || echo '$(srcdir)/'`prov/gni/src/gnix_eq.c
+
+prov/gni/src/libgnix_fi_la-gnix_fabric.lo: prov/gni/src/gnix_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_fabric.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_fabric.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_fabric.lo `test -f 'prov/gni/src/gnix_fabric.c' || echo '$(srcdir)/'`prov/gni/src/gnix_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_fabric.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_fabric.c' object='prov/gni/src/libgnix_fi_la-gnix_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_fabric.lo `test -f 'prov/gni/src/gnix_fabric.c' || echo '$(srcdir)/'`prov/gni/src/gnix_fabric.c
+
+prov/gni/src/libgnix_fi_la-gnix_freelist.lo: prov/gni/src/gnix_freelist.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_freelist.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_freelist.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_freelist.lo `test -f 'prov/gni/src/gnix_freelist.c' || echo '$(srcdir)/'`prov/gni/src/gnix_freelist.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_freelist.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_freelist.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_freelist.c' object='prov/gni/src/libgnix_fi_la-gnix_freelist.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_freelist.lo `test -f 'prov/gni/src/gnix_freelist.c' || echo '$(srcdir)/'`prov/gni/src/gnix_freelist.c
+
+prov/gni/src/libgnix_fi_la-gnix_hashtable.lo: prov/gni/src/gnix_hashtable.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_hashtable.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_hashtable.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_hashtable.lo `test -f 'prov/gni/src/gnix_hashtable.c' || echo '$(srcdir)/'`prov/gni/src/gnix_hashtable.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_hashtable.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_hashtable.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_hashtable.c' object='prov/gni/src/libgnix_fi_la-gnix_hashtable.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_hashtable.lo `test -f 'prov/gni/src/gnix_hashtable.c' || echo '$(srcdir)/'`prov/gni/src/gnix_hashtable.c
+
+prov/gni/src/libgnix_fi_la-gnix_init.lo: prov/gni/src/gnix_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_init.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_init.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_init.lo `test -f 'prov/gni/src/gnix_init.c' || echo '$(srcdir)/'`prov/gni/src/gnix_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_init.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_init.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_init.c' object='prov/gni/src/libgnix_fi_la-gnix_init.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_init.lo `test -f 'prov/gni/src/gnix_init.c' || echo '$(srcdir)/'`prov/gni/src/gnix_init.c
+
+prov/gni/src/libgnix_fi_la-gnix_mbox_allocator.lo: prov/gni/src/gnix_mbox_allocator.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_mbox_allocator.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mbox_allocator.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_mbox_allocator.lo `test -f 'prov/gni/src/gnix_mbox_allocator.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mbox_ [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mbox_allocator.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mbox_allocator.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_mbox_allocator.c' object='prov/gni/src/libgnix_fi_la-gnix_mbox_allocator.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_mbox_allocator.lo `test -f 'prov/gni/src/gnix_mbox_allocator.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mbox_allocator.c
+
+prov/gni/src/libgnix_fi_la-gnix_mr.lo: prov/gni/src/gnix_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_mr.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mr.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_mr.lo `test -f 'prov/gni/src/gnix_mr.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mr.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_mr.c' object='prov/gni/src/libgnix_fi_la-gnix_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_mr.lo `test -f 'prov/gni/src/gnix_mr.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mr.c
+
+prov/gni/src/libgnix_fi_la-gnix_mr_cache.lo: prov/gni/src/gnix_mr_cache.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_mr_cache.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mr_cache.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_mr_cache.lo `test -f 'prov/gni/src/gnix_mr_cache.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mr_cache.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mr_cache.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mr_cache.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_mr_cache.c' object='prov/gni/src/libgnix_fi_la-gnix_mr_cache.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_mr_cache.lo `test -f 'prov/gni/src/gnix_mr_cache.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mr_cache.c
+
+prov/gni/src/libgnix_fi_la-gnix_mr_notifier.lo: prov/gni/src/gnix_mr_notifier.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_mr_notifier.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mr_notifier.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_mr_notifier.lo `test -f 'prov/gni/src/gnix_mr_notifier.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mr_notifier.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mr_notifier.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_mr_notifier.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_mr_notifier.c' object='prov/gni/src/libgnix_fi_la-gnix_mr_notifier.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_mr_notifier.lo `test -f 'prov/gni/src/gnix_mr_notifier.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mr_notifier.c
+
+prov/gni/src/libgnix_fi_la-gnix_msg.lo: prov/gni/src/gnix_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_msg.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_msg.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_msg.lo `test -f 'prov/gni/src/gnix_msg.c' || echo '$(srcdir)/'`prov/gni/src/gnix_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_msg.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_msg.c' object='prov/gni/src/libgnix_fi_la-gnix_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_msg.lo `test -f 'prov/gni/src/gnix_msg.c' || echo '$(srcdir)/'`prov/gni/src/gnix_msg.c
+
+prov/gni/src/libgnix_fi_la-gnix_nameserver.lo: prov/gni/src/gnix_nameserver.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_nameserver.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_nameserver.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_nameserver.lo `test -f 'prov/gni/src/gnix_nameserver.c' || echo '$(srcdir)/'`prov/gni/src/gnix_nameserver.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_nameserver.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_nameserver.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_nameserver.c' object='prov/gni/src/libgnix_fi_la-gnix_nameserver.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_nameserver.lo `test -f 'prov/gni/src/gnix_nameserver.c' || echo '$(srcdir)/'`prov/gni/src/gnix_nameserver.c
+
+prov/gni/src/libgnix_fi_la-gnix_nic.lo: prov/gni/src/gnix_nic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_nic.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_nic.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_nic.lo `test -f 'prov/gni/src/gnix_nic.c' || echo '$(srcdir)/'`prov/gni/src/gnix_nic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_nic.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_nic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_nic.c' object='prov/gni/src/libgnix_fi_la-gnix_nic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_nic.lo `test -f 'prov/gni/src/gnix_nic.c' || echo '$(srcdir)/'`prov/gni/src/gnix_nic.c
+
+prov/gni/src/libgnix_fi_la-gnix_poll.lo: prov/gni/src/gnix_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_poll.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_poll.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_poll.lo `test -f 'prov/gni/src/gnix_poll.c' || echo '$(srcdir)/'`prov/gni/src/gnix_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_poll.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_poll.c' object='prov/gni/src/libgnix_fi_la-gnix_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_poll.lo `test -f 'prov/gni/src/gnix_poll.c' || echo '$(srcdir)/'`prov/gni/src/gnix_poll.c
+
+prov/gni/src/libgnix_fi_la-gnix_queue.lo: prov/gni/src/gnix_queue.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_queue.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_queue.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_queue.lo `test -f 'prov/gni/src/gnix_queue.c' || echo '$(srcdir)/'`prov/gni/src/gnix_queue.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_queue.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_queue.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_queue.c' object='prov/gni/src/libgnix_fi_la-gnix_queue.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_queue.lo `test -f 'prov/gni/src/gnix_queue.c' || echo '$(srcdir)/'`prov/gni/src/gnix_queue.c
+
+prov/gni/src/libgnix_fi_la-gnix_rma.lo: prov/gni/src/gnix_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_rma.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_rma.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_rma.lo `test -f 'prov/gni/src/gnix_rma.c' || echo '$(srcdir)/'`prov/gni/src/gnix_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_rma.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_rma.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_rma.c' object='prov/gni/src/libgnix_fi_la-gnix_rma.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_rma.lo `test -f 'prov/gni/src/gnix_rma.c' || echo '$(srcdir)/'`prov/gni/src/gnix_rma.c
+
+prov/gni/src/libgnix_fi_la-gnix_tags.lo: prov/gni/src/gnix_tags.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_tags.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_tags.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_tags.lo `test -f 'prov/gni/src/gnix_tags.c' || echo '$(srcdir)/'`prov/gni/src/gnix_tags.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_tags.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_tags.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_tags.c' object='prov/gni/src/libgnix_fi_la-gnix_tags.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_tags.lo `test -f 'prov/gni/src/gnix_tags.c' || echo '$(srcdir)/'`prov/gni/src/gnix_tags.c
+
+prov/gni/src/libgnix_fi_la-gnix_trigger.lo: prov/gni/src/gnix_trigger.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_trigger.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_trigger.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_trigger.lo `test -f 'prov/gni/src/gnix_trigger.c' || echo '$(srcdir)/'`prov/gni/src/gnix_trigger.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_trigger.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_trigger.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_trigger.c' object='prov/gni/src/libgnix_fi_la-gnix_trigger.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_trigger.lo `test -f 'prov/gni/src/gnix_trigger.c' || echo '$(srcdir)/'`prov/gni/src/gnix_trigger.c
+
+prov/gni/src/libgnix_fi_la-gnix_util.lo: prov/gni/src/gnix_util.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_util.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_util.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_util.lo `test -f 'prov/gni/src/gnix_util.c' || echo '$(srcdir)/'`prov/gni/src/gnix_util.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_util.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_util.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_util.c' object='prov/gni/src/libgnix_fi_la-gnix_util.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_util.lo `test -f 'prov/gni/src/gnix_util.c' || echo '$(srcdir)/'`prov/gni/src/gnix_util.c
+
+prov/gni/src/libgnix_fi_la-gnix_vc.lo: prov/gni/src/gnix_vc.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_vc.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_vc.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_vc.lo `test -f 'prov/gni/src/gnix_vc.c' || echo '$(srcdir)/'`prov/gni/src/gnix_vc.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_vc.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_vc.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_vc.c' object='prov/gni/src/libgnix_fi_la-gnix_vc.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_vc.lo `test -f 'prov/gni/src/gnix_vc.c' || echo '$(srcdir)/'`prov/gni/src/gnix_vc.c
+
+prov/gni/src/libgnix_fi_la-gnix_vector.lo: prov/gni/src/gnix_vector.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_vector.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_vector.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_vector.lo `test -f 'prov/gni/src/gnix_vector.c' || echo '$(srcdir)/'`prov/gni/src/gnix_vector.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_vector.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_vector.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_vector.c' object='prov/gni/src/libgnix_fi_la-gnix_vector.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_vector.lo `test -f 'prov/gni/src/gnix_vector.c' || echo '$(srcdir)/'`prov/gni/src/gnix_vector.c
+
+prov/gni/src/libgnix_fi_la-gnix_xpmem.lo: prov/gni/src/gnix_xpmem.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_xpmem.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_xpmem.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_xpmem.lo `test -f 'prov/gni/src/gnix_xpmem.c' || echo '$(srcdir)/'`prov/gni/src/gnix_xpmem.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_xpmem.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_xpmem.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_xpmem.c' object='prov/gni/src/libgnix_fi_la-gnix_xpmem.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_xpmem.lo `test -f 'prov/gni/src/gnix_xpmem.c' || echo '$(srcdir)/'`prov/gni/src/gnix_xpmem.c
+
+prov/gni/src/libgnix_fi_la-gnix_wait.lo: prov/gni/src/gnix_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/libgnix_fi_la-gnix_wait.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_wait.Tpo -c -o prov/gni/src/libgnix_fi_la-gnix_wait.lo `test -f 'prov/gni/src/gnix_wait.c' || echo '$(srcdir)/'`prov/gni/src/gnix_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_wait.Tpo prov/gni/src/$(DEPDIR)/libgnix_fi_la-gnix_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_wait.c' object='prov/gni/src/libgnix_fi_la-gnix_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/libgnix_fi_la-gnix_wait.lo `test -f 'prov/gni/src/gnix_wait.c' || echo '$(srcdir)/'`prov/gni/src/gnix_wait.c
+
+src/libgnix_fi_la-common.lo: src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libgnix_fi_la-common.lo -MD -MP -MF src/$(DEPDIR)/libgnix_fi_la-common.Tpo -c -o src/libgnix_fi_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libgnix_fi_la-common.Tpo src/$(DEPDIR)/libgnix_fi_la-common.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/common.c' object='src/libgnix_fi_la-common.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libgnix_fi_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+
+src/libgnix_fi_la-enosys.lo: src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libgnix_fi_la-enosys.lo -MD -MP -MF src/$(DEPDIR)/libgnix_fi_la-enosys.Tpo -c -o src/libgnix_fi_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libgnix_fi_la-enosys.Tpo src/$(DEPDIR)/libgnix_fi_la-enosys.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/enosys.c' object='src/libgnix_fi_la-enosys.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libgnix_fi_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+
+src/libgnix_fi_la-rbtree.lo: src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libgnix_fi_la-rbtree.lo -MD -MP -MF src/$(DEPDIR)/libgnix_fi_la-rbtree.Tpo -c -o src/libgnix_fi_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libgnix_fi_la-rbtree.Tpo src/$(DEPDIR)/libgnix_fi_la-rbtree.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/rbtree.c' object='src/libgnix_fi_la-rbtree.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libgnix_fi_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+
+src/libgnix_fi_la-fasthash.lo: src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libgnix_fi_la-fasthash.lo -MD -MP -MF src/$(DEPDIR)/libgnix_fi_la-fasthash.Tpo -c -o src/libgnix_fi_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libgnix_fi_la-fasthash.Tpo src/$(DEPDIR)/libgnix_fi_la-fasthash.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/fasthash.c' object='src/libgnix_fi_la-fasthash.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libgnix_fi_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+
+src/libgnix_fi_la-indexer.lo: src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libgnix_fi_la-indexer.lo -MD -MP -MF src/$(DEPDIR)/libgnix_fi_la-indexer.Tpo -c -o src/libgnix_fi_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libgnix_fi_la-indexer.Tpo src/$(DEPDIR)/libgnix_fi_la-indexer.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/indexer.c' object='src/libgnix_fi_la-indexer.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libgnix_fi_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+
+prov/util/src/libgnix_fi_la-util_attr.lo: prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libgnix_fi_la-util_attr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libgnix_fi_la-util_attr.Tpo -c -o prov/util/src/libgnix_fi_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libgnix_fi_la-util_attr.Tpo prov/util/src/$(DEPDIR)/libgnix_fi_la-util_attr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_attr.c' object='prov/util/src/libgnix_fi_la-util_attr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libgnix_fi_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+
+prov/util/src/libgnix_fi_la-util_av.lo: prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libgnix_fi_la-util_av.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libgnix_fi_la-util_av.Tpo -c -o prov/util/src/libgnix_fi_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libgnix_fi_la-util_av.Tpo prov/util/src/$(DEPDIR)/libgnix_fi_la-util_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_av.c' object='prov/util/src/libgnix_fi_la-util_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libgnix_fi_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+
+prov/util/src/libgnix_fi_la-util_cq.lo: prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libgnix_fi_la-util_cq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libgnix_fi_la-util_cq.Tpo -c -o prov/util/src/libgnix_fi_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libgnix_fi_la-util_cq.Tpo prov/util/src/$(DEPDIR)/libgnix_fi_la-util_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_cq.c' object='prov/util/src/libgnix_fi_la-util_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libgnix_fi_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+
+prov/util/src/libgnix_fi_la-util_domain.lo: prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libgnix_fi_la-util_domain.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libgnix_fi_la-util_domain.Tpo -c -o prov/util/src/libgnix_fi_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libgnix_fi_la-util_domain.Tpo prov/util/src/$(DEPDIR)/libgnix_fi_la-util_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_domain.c' object='prov/util/src/libgnix_fi_la-util_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libgnix_fi_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+
+prov/util/src/libgnix_fi_la-util_ep.lo: prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libgnix_fi_la-util_ep.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libgnix_fi_la-util_ep.Tpo -c -o prov/util/src/libgnix_fi_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libgnix_fi_la-util_ep.Tpo prov/util/src/$(DEPDIR)/libgnix_fi_la-util_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_ep.c' object='prov/util/src/libgnix_fi_la-util_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libgnix_fi_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+
+prov/util/src/libgnix_fi_la-util_eq.lo: prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libgnix_fi_la-util_eq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libgnix_fi_la-util_eq.Tpo -c -o prov/util/src/libgnix_fi_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libgnix_fi_la-util_eq.Tpo prov/util/src/$(DEPDIR)/libgnix_fi_la-util_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_eq.c' object='prov/util/src/libgnix_fi_la-util_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libgnix_fi_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+
+prov/util/src/libgnix_fi_la-util_fabric.lo: prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libgnix_fi_la-util_fabric.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libgnix_fi_la-util_fabric.Tpo -c -o prov/util/src/libgnix_fi_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libgnix_fi_la-util_fabric.Tpo prov/util/src/$(DEPDIR)/libgnix_fi_la-util_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_fabric.c' object='prov/util/src/libgnix_fi_la-util_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libgnix_fi_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+
+prov/util/src/libgnix_fi_la-util_main.lo: prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libgnix_fi_la-util_main.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libgnix_fi_la-util_main.Tpo -c -o prov/util/src/libgnix_fi_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libgnix_fi_la-util_main.Tpo prov/util/src/$(DEPDIR)/libgnix_fi_la-util_main.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_main.c' object='prov/util/src/libgnix_fi_la-util_main.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libgnix_fi_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+
+prov/util/src/libgnix_fi_la-util_poll.lo: prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libgnix_fi_la-util_poll.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libgnix_fi_la-util_poll.Tpo -c -o prov/util/src/libgnix_fi_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libgnix_fi_la-util_poll.Tpo prov/util/src/$(DEPDIR)/libgnix_fi_la-util_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_poll.c' object='prov/util/src/libgnix_fi_la-util_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libgnix_fi_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+
+prov/util/src/libgnix_fi_la-util_wait.lo: prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libgnix_fi_la-util_wait.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libgnix_fi_la-util_wait.Tpo -c -o prov/util/src/libgnix_fi_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libgnix_fi_la-util_wait.Tpo prov/util/src/$(DEPDIR)/libgnix_fi_la-util_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_wait.c' object='prov/util/src/libgnix_fi_la-util_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libgnix_fi_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+
+prov/util/src/libgnix_fi_la-util_buf.lo: prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libgnix_fi_la-util_buf.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libgnix_fi_la-util_buf.Tpo -c -o prov/util/src/libgnix_fi_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libgnix_fi_la-util_buf.Tpo prov/util/src/$(DEPDIR)/libgnix_fi_la-util_buf.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_buf.c' object='prov/util/src/libgnix_fi_la-util_buf.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libgnix_fi_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+
+prov/util/src/libgnix_fi_la-util_mr.lo: prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libgnix_fi_la-util_mr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libgnix_fi_la-util_mr.Tpo -c -o prov/util/src/libgnix_fi_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libgnix_fi_la-util_mr.Tpo prov/util/src/$(DEPDIR)/libgnix_fi_la-util_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_mr.c' object='prov/util/src/libgnix_fi_la-util_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libgnix_fi_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+
+src/osx/libgnix_fi_la-osd.lo: src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/osx/libgnix_fi_la-osd.lo -MD -MP -MF src/osx/$(DEPDIR)/libgnix_fi_la-osd.Tpo -c -o src/osx/libgnix_fi_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/osx/$(DEPDIR)/libgnix_fi_la-osd.Tpo src/osx/$(DEPDIR)/libgnix_fi_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/osx/osd.c' object='src/osx/libgnix_fi_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/osx/libgnix_fi_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+
+src/unix/libgnix_fi_la-osd.lo: src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/unix/libgnix_fi_la-osd.lo -MD -MP -MF src/unix/$(DEPDIR)/libgnix_fi_la-osd.Tpo -c -o src/unix/libgnix_fi_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/unix/$(DEPDIR)/libgnix_fi_la-osd.Tpo src/unix/$(DEPDIR)/libgnix_fi_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/unix/osd.c' object='src/unix/libgnix_fi_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnix_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/unix/libgnix_fi_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+
+prov/mxm/src/libmlxm_fi_la-mlxm_init.lo: prov/mxm/src/mlxm_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/libmlxm_fi_la-mlxm_init.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_init.Tpo -c -o prov/mxm/src/libmlxm_fi_la-mlxm_init.lo `test -f 'prov/mxm/src/mlxm_init.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_init.Tpo prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_init.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_init.c' object='prov/mxm/src/libmlxm_fi_la-mlxm_init.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/libmlxm_fi_la-mlxm_init.lo `test -f 'prov/mxm/src/mlxm_init.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_init.c
+
+prov/mxm/src/libmlxm_fi_la-mlxm_domain.lo: prov/mxm/src/mlxm_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/libmlxm_fi_la-mlxm_domain.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_domain.Tpo -c -o prov/mxm/src/libmlxm_fi_la-mlxm_domain.lo `test -f 'prov/mxm/src/mlxm_domain.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_domain.Tpo prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_domain.c' object='prov/mxm/src/libmlxm_fi_la-mlxm_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/libmlxm_fi_la-mlxm_domain.lo `test -f 'prov/mxm/src/mlxm_domain.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_domain.c
+
+prov/mxm/src/libmlxm_fi_la-mlxm_cq.lo: prov/mxm/src/mlxm_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/libmlxm_fi_la-mlxm_cq.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_cq.Tpo -c -o prov/mxm/src/libmlxm_fi_la-mlxm_cq.lo `test -f 'prov/mxm/src/mlxm_cq.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_cq.Tpo prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_cq.c' object='prov/mxm/src/libmlxm_fi_la-mlxm_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/libmlxm_fi_la-mlxm_cq.lo `test -f 'prov/mxm/src/mlxm_cq.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_cq.c
+
+prov/mxm/src/libmlxm_fi_la-mlxm_av.lo: prov/mxm/src/mlxm_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/libmlxm_fi_la-mlxm_av.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_av.Tpo -c -o prov/mxm/src/libmlxm_fi_la-mlxm_av.lo `test -f 'prov/mxm/src/mlxm_av.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_av.Tpo prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_av.c' object='prov/mxm/src/libmlxm_fi_la-mlxm_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/libmlxm_fi_la-mlxm_av.lo `test -f 'prov/mxm/src/mlxm_av.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_av.c
+
+prov/mxm/src/libmlxm_fi_la-mlxm_ep.lo: prov/mxm/src/mlxm_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/libmlxm_fi_la-mlxm_ep.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_ep.Tpo -c -o prov/mxm/src/libmlxm_fi_la-mlxm_ep.lo `test -f 'prov/mxm/src/mlxm_ep.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_ep.Tpo prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_ep.c' object='prov/mxm/src/libmlxm_fi_la-mlxm_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/libmlxm_fi_la-mlxm_ep.lo `test -f 'prov/mxm/src/mlxm_ep.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_ep.c
+
+prov/mxm/src/libmlxm_fi_la-mlxm_cm.lo: prov/mxm/src/mlxm_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/libmlxm_fi_la-mlxm_cm.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_cm.Tpo -c -o prov/mxm/src/libmlxm_fi_la-mlxm_cm.lo `test -f 'prov/mxm/src/mlxm_cm.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_cm.Tpo prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_cm.c' object='prov/mxm/src/libmlxm_fi_la-mlxm_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/libmlxm_fi_la-mlxm_cm.lo `test -f 'prov/mxm/src/mlxm_cm.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_cm.c
+
+prov/mxm/src/libmlxm_fi_la-mlxm_tagged.lo: prov/mxm/src/mlxm_tagged.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/libmlxm_fi_la-mlxm_tagged.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_tagged.Tpo -c -o prov/mxm/src/libmlxm_fi_la-mlxm_tagged.lo `test -f 'prov/mxm/src/mlxm_tagged.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_tagged.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_tagged.Tpo prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_tagged.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_tagged.c' object='prov/mxm/src/libmlxm_fi_la-mlxm_tagged.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/libmlxm_fi_la-mlxm_tagged.lo `test -f 'prov/mxm/src/mlxm_tagged.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_tagged.c
+
+prov/mxm/src/libmlxm_fi_la-mlxm_mr.lo: prov/mxm/src/mlxm_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/libmlxm_fi_la-mlxm_mr.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_mr.Tpo -c -o prov/mxm/src/libmlxm_fi_la-mlxm_mr.lo `test -f 'prov/mxm/src/mlxm_mr.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_mr.Tpo prov/mxm/src/$(DEPDIR)/libmlxm_fi_la-mlxm_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_mr.c' object='prov/mxm/src/libmlxm_fi_la-mlxm_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/libmlxm_fi_la-mlxm_mr.lo `test -f 'prov/mxm/src/mlxm_mr.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_mr.c
+
+src/libmlxm_fi_la-common.lo: src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libmlxm_fi_la-common.lo -MD -MP -MF src/$(DEPDIR)/libmlxm_fi_la-common.Tpo -c -o src/libmlxm_fi_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libmlxm_fi_la-common.Tpo src/$(DEPDIR)/libmlxm_fi_la-common.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/common.c' object='src/libmlxm_fi_la-common.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libmlxm_fi_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+
+src/libmlxm_fi_la-enosys.lo: src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libmlxm_fi_la-enosys.lo -MD -MP -MF src/$(DEPDIR)/libmlxm_fi_la-enosys.Tpo -c -o src/libmlxm_fi_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libmlxm_fi_la-enosys.Tpo src/$(DEPDIR)/libmlxm_fi_la-enosys.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/enosys.c' object='src/libmlxm_fi_la-enosys.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libmlxm_fi_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+
+src/libmlxm_fi_la-rbtree.lo: src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libmlxm_fi_la-rbtree.lo -MD -MP -MF src/$(DEPDIR)/libmlxm_fi_la-rbtree.Tpo -c -o src/libmlxm_fi_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libmlxm_fi_la-rbtree.Tpo src/$(DEPDIR)/libmlxm_fi_la-rbtree.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/rbtree.c' object='src/libmlxm_fi_la-rbtree.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libmlxm_fi_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+
+src/libmlxm_fi_la-fasthash.lo: src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libmlxm_fi_la-fasthash.lo -MD -MP -MF src/$(DEPDIR)/libmlxm_fi_la-fasthash.Tpo -c -o src/libmlxm_fi_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libmlxm_fi_la-fasthash.Tpo src/$(DEPDIR)/libmlxm_fi_la-fasthash.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/fasthash.c' object='src/libmlxm_fi_la-fasthash.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libmlxm_fi_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+
+src/libmlxm_fi_la-indexer.lo: src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libmlxm_fi_la-indexer.lo -MD -MP -MF src/$(DEPDIR)/libmlxm_fi_la-indexer.Tpo -c -o src/libmlxm_fi_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libmlxm_fi_la-indexer.Tpo src/$(DEPDIR)/libmlxm_fi_la-indexer.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/indexer.c' object='src/libmlxm_fi_la-indexer.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libmlxm_fi_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+
+prov/util/src/libmlxm_fi_la-util_attr.lo: prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libmlxm_fi_la-util_attr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_attr.Tpo -c -o prov/util/src/libmlxm_fi_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_attr.Tpo prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_attr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_attr.c' object='prov/util/src/libmlxm_fi_la-util_attr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libmlxm_fi_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+
+prov/util/src/libmlxm_fi_la-util_av.lo: prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libmlxm_fi_la-util_av.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_av.Tpo -c -o prov/util/src/libmlxm_fi_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_av.Tpo prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_av.c' object='prov/util/src/libmlxm_fi_la-util_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libmlxm_fi_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+
+prov/util/src/libmlxm_fi_la-util_cq.lo: prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libmlxm_fi_la-util_cq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_cq.Tpo -c -o prov/util/src/libmlxm_fi_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_cq.Tpo prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_cq.c' object='prov/util/src/libmlxm_fi_la-util_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libmlxm_fi_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+
+prov/util/src/libmlxm_fi_la-util_domain.lo: prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libmlxm_fi_la-util_domain.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_domain.Tpo -c -o prov/util/src/libmlxm_fi_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_domain.Tpo prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_domain.c' object='prov/util/src/libmlxm_fi_la-util_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libmlxm_fi_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+
+prov/util/src/libmlxm_fi_la-util_ep.lo: prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libmlxm_fi_la-util_ep.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_ep.Tpo -c -o prov/util/src/libmlxm_fi_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_ep.Tpo prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_ep.c' object='prov/util/src/libmlxm_fi_la-util_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libmlxm_fi_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+
+prov/util/src/libmlxm_fi_la-util_eq.lo: prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libmlxm_fi_la-util_eq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_eq.Tpo -c -o prov/util/src/libmlxm_fi_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_eq.Tpo prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_eq.c' object='prov/util/src/libmlxm_fi_la-util_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libmlxm_fi_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+
+prov/util/src/libmlxm_fi_la-util_fabric.lo: prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libmlxm_fi_la-util_fabric.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_fabric.Tpo -c -o prov/util/src/libmlxm_fi_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_fabric.Tpo prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_fabric.c' object='prov/util/src/libmlxm_fi_la-util_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libmlxm_fi_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+
+prov/util/src/libmlxm_fi_la-util_main.lo: prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libmlxm_fi_la-util_main.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_main.Tpo -c -o prov/util/src/libmlxm_fi_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_main.Tpo prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_main.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_main.c' object='prov/util/src/libmlxm_fi_la-util_main.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libmlxm_fi_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+
+prov/util/src/libmlxm_fi_la-util_poll.lo: prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libmlxm_fi_la-util_poll.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_poll.Tpo -c -o prov/util/src/libmlxm_fi_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_poll.Tpo prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_poll.c' object='prov/util/src/libmlxm_fi_la-util_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libmlxm_fi_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+
+prov/util/src/libmlxm_fi_la-util_wait.lo: prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libmlxm_fi_la-util_wait.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_wait.Tpo -c -o prov/util/src/libmlxm_fi_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_wait.Tpo prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_wait.c' object='prov/util/src/libmlxm_fi_la-util_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libmlxm_fi_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+
+prov/util/src/libmlxm_fi_la-util_buf.lo: prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libmlxm_fi_la-util_buf.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_buf.Tpo -c -o prov/util/src/libmlxm_fi_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_buf.Tpo prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_buf.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_buf.c' object='prov/util/src/libmlxm_fi_la-util_buf.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libmlxm_fi_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+
+prov/util/src/libmlxm_fi_la-util_mr.lo: prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libmlxm_fi_la-util_mr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_mr.Tpo -c -o prov/util/src/libmlxm_fi_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_mr.Tpo prov/util/src/$(DEPDIR)/libmlxm_fi_la-util_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_mr.c' object='prov/util/src/libmlxm_fi_la-util_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libmlxm_fi_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+
+src/osx/libmlxm_fi_la-osd.lo: src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/osx/libmlxm_fi_la-osd.lo -MD -MP -MF src/osx/$(DEPDIR)/libmlxm_fi_la-osd.Tpo -c -o src/osx/libmlxm_fi_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/osx/$(DEPDIR)/libmlxm_fi_la-osd.Tpo src/osx/$(DEPDIR)/libmlxm_fi_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/osx/osd.c' object='src/osx/libmlxm_fi_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/osx/libmlxm_fi_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+
+src/unix/libmlxm_fi_la-osd.lo: src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/unix/libmlxm_fi_la-osd.lo -MD -MP -MF src/unix/$(DEPDIR)/libmlxm_fi_la-osd.Tpo -c -o src/unix/libmlxm_fi_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/unix/$(DEPDIR)/libmlxm_fi_la-osd.Tpo src/unix/$(DEPDIR)/libmlxm_fi_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/unix/osd.c' object='src/unix/libmlxm_fi_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libmlxm_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/unix/libmlxm_fi_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+
+prov/psm/src/libpsmx_fi_la-psmx_init.lo: prov/psm/src/psmx_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_init.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_init.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_init.lo `test -f 'prov/psm/src/psmx_init.c' || echo '$(srcdir)/'`prov/psm/src/psmx_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_init.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_init.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_init.c' object='prov/psm/src/libpsmx_fi_la-psmx_init.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_init.lo `test -f 'prov/psm/src/psmx_init.c' || echo '$(srcdir)/'`prov/psm/src/psmx_init.c
+
+prov/psm/src/libpsmx_fi_la-psmx_fabric.lo: prov/psm/src/psmx_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_fabric.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_fabric.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_fabric.lo `test -f 'prov/psm/src/psmx_fabric.c' || echo '$(srcdir)/'`prov/psm/src/psmx_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_fabric.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_fabric.c' object='prov/psm/src/libpsmx_fi_la-psmx_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_fabric.lo `test -f 'prov/psm/src/psmx_fabric.c' || echo '$(srcdir)/'`prov/psm/src/psmx_fabric.c
+
+prov/psm/src/libpsmx_fi_la-psmx_domain.lo: prov/psm/src/psmx_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_domain.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_domain.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_domain.lo `test -f 'prov/psm/src/psmx_domain.c' || echo '$(srcdir)/'`prov/psm/src/psmx_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_domain.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_domain.c' object='prov/psm/src/libpsmx_fi_la-psmx_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_domain.lo `test -f 'prov/psm/src/psmx_domain.c' || echo '$(srcdir)/'`prov/psm/src/psmx_domain.c
+
+prov/psm/src/libpsmx_fi_la-psmx_cq.lo: prov/psm/src/psmx_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_cq.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_cq.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_cq.lo `test -f 'prov/psm/src/psmx_cq.c' || echo '$(srcdir)/'`prov/psm/src/psmx_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_cq.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_cq.c' object='prov/psm/src/libpsmx_fi_la-psmx_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_cq.lo `test -f 'prov/psm/src/psmx_cq.c' || echo '$(srcdir)/'`prov/psm/src/psmx_cq.c
+
+prov/psm/src/libpsmx_fi_la-psmx_cntr.lo: prov/psm/src/psmx_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_cntr.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_cntr.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_cntr.lo `test -f 'prov/psm/src/psmx_cntr.c' || echo '$(srcdir)/'`prov/psm/src/psmx_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_cntr.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_cntr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_cntr.c' object='prov/psm/src/libpsmx_fi_la-psmx_cntr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_cntr.lo `test -f 'prov/psm/src/psmx_cntr.c' || echo '$(srcdir)/'`prov/psm/src/psmx_cntr.c
+
+prov/psm/src/libpsmx_fi_la-psmx_av.lo: prov/psm/src/psmx_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_av.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_av.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_av.lo `test -f 'prov/psm/src/psmx_av.c' || echo '$(srcdir)/'`prov/psm/src/psmx_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_av.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_av.c' object='prov/psm/src/libpsmx_fi_la-psmx_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_av.lo `test -f 'prov/psm/src/psmx_av.c' || echo '$(srcdir)/'`prov/psm/src/psmx_av.c
+
+prov/psm/src/libpsmx_fi_la-psmx_ep.lo: prov/psm/src/psmx_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_ep.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_ep.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_ep.lo `test -f 'prov/psm/src/psmx_ep.c' || echo '$(srcdir)/'`prov/psm/src/psmx_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_ep.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_ep.c' object='prov/psm/src/libpsmx_fi_la-psmx_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_ep.lo `test -f 'prov/psm/src/psmx_ep.c' || echo '$(srcdir)/'`prov/psm/src/psmx_ep.c
+
+prov/psm/src/libpsmx_fi_la-psmx_cm.lo: prov/psm/src/psmx_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_cm.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_cm.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_cm.lo `test -f 'prov/psm/src/psmx_cm.c' || echo '$(srcdir)/'`prov/psm/src/psmx_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_cm.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_cm.c' object='prov/psm/src/libpsmx_fi_la-psmx_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_cm.lo `test -f 'prov/psm/src/psmx_cm.c' || echo '$(srcdir)/'`prov/psm/src/psmx_cm.c
+
+prov/psm/src/libpsmx_fi_la-psmx_tagged.lo: prov/psm/src/psmx_tagged.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_tagged.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_tagged.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_tagged.lo `test -f 'prov/psm/src/psmx_tagged.c' || echo '$(srcdir)/'`prov/psm/src/psmx_tagged.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_tagged.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_tagged.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_tagged.c' object='prov/psm/src/libpsmx_fi_la-psmx_tagged.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_tagged.lo `test -f 'prov/psm/src/psmx_tagged.c' || echo '$(srcdir)/'`prov/psm/src/psmx_tagged.c
+
+prov/psm/src/libpsmx_fi_la-psmx_msg.lo: prov/psm/src/psmx_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_msg.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_msg.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_msg.lo `test -f 'prov/psm/src/psmx_msg.c' || echo '$(srcdir)/'`prov/psm/src/psmx_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_msg.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_msg.c' object='prov/psm/src/libpsmx_fi_la-psmx_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_msg.lo `test -f 'prov/psm/src/psmx_msg.c' || echo '$(srcdir)/'`prov/psm/src/psmx_msg.c
+
+prov/psm/src/libpsmx_fi_la-psmx_msg2.lo: prov/psm/src/psmx_msg2.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_msg2.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_msg2.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_msg2.lo `test -f 'prov/psm/src/psmx_msg2.c' || echo '$(srcdir)/'`prov/psm/src/psmx_msg2.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_msg2.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_msg2.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_msg2.c' object='prov/psm/src/libpsmx_fi_la-psmx_msg2.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_msg2.lo `test -f 'prov/psm/src/psmx_msg2.c' || echo '$(srcdir)/'`prov/psm/src/psmx_msg2.c
+
+prov/psm/src/libpsmx_fi_la-psmx_rma.lo: prov/psm/src/psmx_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_rma.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_rma.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_rma.lo `test -f 'prov/psm/src/psmx_rma.c' || echo '$(srcdir)/'`prov/psm/src/psmx_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_rma.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_rma.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_rma.c' object='prov/psm/src/libpsmx_fi_la-psmx_rma.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_rma.lo `test -f 'prov/psm/src/psmx_rma.c' || echo '$(srcdir)/'`prov/psm/src/psmx_rma.c
+
+prov/psm/src/libpsmx_fi_la-psmx_atomic.lo: prov/psm/src/psmx_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_atomic.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_atomic.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_atomic.lo `test -f 'prov/psm/src/psmx_atomic.c' || echo '$(srcdir)/'`prov/psm/src/psmx_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_atomic.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_atomic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_atomic.c' object='prov/psm/src/libpsmx_fi_la-psmx_atomic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_atomic.lo `test -f 'prov/psm/src/psmx_atomic.c' || echo '$(srcdir)/'`prov/psm/src/psmx_atomic.c
+
+prov/psm/src/libpsmx_fi_la-psmx_am.lo: prov/psm/src/psmx_am.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_am.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_am.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_am.lo `test -f 'prov/psm/src/psmx_am.c' || echo '$(srcdir)/'`prov/psm/src/psmx_am.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_am.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_am.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_am.c' object='prov/psm/src/libpsmx_fi_la-psmx_am.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_am.lo `test -f 'prov/psm/src/psmx_am.c' || echo '$(srcdir)/'`prov/psm/src/psmx_am.c
+
+prov/psm/src/libpsmx_fi_la-psmx_mr.lo: prov/psm/src/psmx_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_mr.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_mr.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_mr.lo `test -f 'prov/psm/src/psmx_mr.c' || echo '$(srcdir)/'`prov/psm/src/psmx_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_mr.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_mr.c' object='prov/psm/src/libpsmx_fi_la-psmx_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_mr.lo `test -f 'prov/psm/src/psmx_mr.c' || echo '$(srcdir)/'`prov/psm/src/psmx_mr.c
+
+prov/psm/src/libpsmx_fi_la-psmx_wait.lo: prov/psm/src/psmx_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_wait.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_wait.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_wait.lo `test -f 'prov/psm/src/psmx_wait.c' || echo '$(srcdir)/'`prov/psm/src/psmx_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_wait.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_wait.c' object='prov/psm/src/libpsmx_fi_la-psmx_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_wait.lo `test -f 'prov/psm/src/psmx_wait.c' || echo '$(srcdir)/'`prov/psm/src/psmx_wait.c
+
+prov/psm/src/libpsmx_fi_la-psmx_util.lo: prov/psm/src/psmx_util.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_fi_la-psmx_util.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_util.Tpo -c -o prov/psm/src/libpsmx_fi_la-psmx_util.lo `test -f 'prov/psm/src/psmx_util.c' || echo '$(srcdir)/'`prov/psm/src/psmx_util.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_util.Tpo prov/psm/src/$(DEPDIR)/libpsmx_fi_la-psmx_util.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_util.c' object='prov/psm/src/libpsmx_fi_la-psmx_util.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_fi_la-psmx_util.lo `test -f 'prov/psm/src/psmx_util.c' || echo '$(srcdir)/'`prov/psm/src/psmx_util.c
+
+src/libpsmx_fi_la-common.lo: src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libpsmx_fi_la-common.lo -MD -MP -MF src/$(DEPDIR)/libpsmx_fi_la-common.Tpo -c -o src/libpsmx_fi_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libpsmx_fi_la-common.Tpo src/$(DEPDIR)/libpsmx_fi_la-common.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/common.c' object='src/libpsmx_fi_la-common.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libpsmx_fi_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+
+src/libpsmx_fi_la-enosys.lo: src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libpsmx_fi_la-enosys.lo -MD -MP -MF src/$(DEPDIR)/libpsmx_fi_la-enosys.Tpo -c -o src/libpsmx_fi_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libpsmx_fi_la-enosys.Tpo src/$(DEPDIR)/libpsmx_fi_la-enosys.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/enosys.c' object='src/libpsmx_fi_la-enosys.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libpsmx_fi_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+
+src/libpsmx_fi_la-rbtree.lo: src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libpsmx_fi_la-rbtree.lo -MD -MP -MF src/$(DEPDIR)/libpsmx_fi_la-rbtree.Tpo -c -o src/libpsmx_fi_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libpsmx_fi_la-rbtree.Tpo src/$(DEPDIR)/libpsmx_fi_la-rbtree.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/rbtree.c' object='src/libpsmx_fi_la-rbtree.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libpsmx_fi_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+
+src/libpsmx_fi_la-fasthash.lo: src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libpsmx_fi_la-fasthash.lo -MD -MP -MF src/$(DEPDIR)/libpsmx_fi_la-fasthash.Tpo -c -o src/libpsmx_fi_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libpsmx_fi_la-fasthash.Tpo src/$(DEPDIR)/libpsmx_fi_la-fasthash.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/fasthash.c' object='src/libpsmx_fi_la-fasthash.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libpsmx_fi_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+
+src/libpsmx_fi_la-indexer.lo: src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libpsmx_fi_la-indexer.lo -MD -MP -MF src/$(DEPDIR)/libpsmx_fi_la-indexer.Tpo -c -o src/libpsmx_fi_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libpsmx_fi_la-indexer.Tpo src/$(DEPDIR)/libpsmx_fi_la-indexer.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/indexer.c' object='src/libpsmx_fi_la-indexer.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libpsmx_fi_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+
+prov/util/src/libpsmx_fi_la-util_attr.lo: prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx_fi_la-util_attr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_attr.Tpo -c -o prov/util/src/libpsmx_fi_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_attr.Tpo prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_attr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_attr.c' object='prov/util/src/libpsmx_fi_la-util_attr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx_fi_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+
+prov/util/src/libpsmx_fi_la-util_av.lo: prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx_fi_la-util_av.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_av.Tpo -c -o prov/util/src/libpsmx_fi_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_av.Tpo prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_av.c' object='prov/util/src/libpsmx_fi_la-util_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx_fi_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+
+prov/util/src/libpsmx_fi_la-util_cq.lo: prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx_fi_la-util_cq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_cq.Tpo -c -o prov/util/src/libpsmx_fi_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_cq.Tpo prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_cq.c' object='prov/util/src/libpsmx_fi_la-util_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx_fi_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+
+prov/util/src/libpsmx_fi_la-util_domain.lo: prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx_fi_la-util_domain.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_domain.Tpo -c -o prov/util/src/libpsmx_fi_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_domain.Tpo prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_domain.c' object='prov/util/src/libpsmx_fi_la-util_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx_fi_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+
+prov/util/src/libpsmx_fi_la-util_ep.lo: prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx_fi_la-util_ep.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_ep.Tpo -c -o prov/util/src/libpsmx_fi_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_ep.Tpo prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_ep.c' object='prov/util/src/libpsmx_fi_la-util_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx_fi_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+
+prov/util/src/libpsmx_fi_la-util_eq.lo: prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx_fi_la-util_eq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_eq.Tpo -c -o prov/util/src/libpsmx_fi_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_eq.Tpo prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_eq.c' object='prov/util/src/libpsmx_fi_la-util_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx_fi_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+
+prov/util/src/libpsmx_fi_la-util_fabric.lo: prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx_fi_la-util_fabric.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_fabric.Tpo -c -o prov/util/src/libpsmx_fi_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_fabric.Tpo prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_fabric.c' object='prov/util/src/libpsmx_fi_la-util_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx_fi_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+
+prov/util/src/libpsmx_fi_la-util_main.lo: prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx_fi_la-util_main.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_main.Tpo -c -o prov/util/src/libpsmx_fi_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_main.Tpo prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_main.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_main.c' object='prov/util/src/libpsmx_fi_la-util_main.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx_fi_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+
+prov/util/src/libpsmx_fi_la-util_poll.lo: prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx_fi_la-util_poll.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_poll.Tpo -c -o prov/util/src/libpsmx_fi_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_poll.Tpo prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_poll.c' object='prov/util/src/libpsmx_fi_la-util_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx_fi_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+
+prov/util/src/libpsmx_fi_la-util_wait.lo: prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx_fi_la-util_wait.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_wait.Tpo -c -o prov/util/src/libpsmx_fi_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_wait.Tpo prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_wait.c' object='prov/util/src/libpsmx_fi_la-util_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx_fi_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+
+prov/util/src/libpsmx_fi_la-util_buf.lo: prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx_fi_la-util_buf.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_buf.Tpo -c -o prov/util/src/libpsmx_fi_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_buf.Tpo prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_buf.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_buf.c' object='prov/util/src/libpsmx_fi_la-util_buf.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx_fi_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+
+prov/util/src/libpsmx_fi_la-util_mr.lo: prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx_fi_la-util_mr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_mr.Tpo -c -o prov/util/src/libpsmx_fi_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_mr.Tpo prov/util/src/$(DEPDIR)/libpsmx_fi_la-util_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_mr.c' object='prov/util/src/libpsmx_fi_la-util_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx_fi_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+
+src/osx/libpsmx_fi_la-osd.lo: src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/osx/libpsmx_fi_la-osd.lo -MD -MP -MF src/osx/$(DEPDIR)/libpsmx_fi_la-osd.Tpo -c -o src/osx/libpsmx_fi_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/osx/$(DEPDIR)/libpsmx_fi_la-osd.Tpo src/osx/$(DEPDIR)/libpsmx_fi_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/osx/osd.c' object='src/osx/libpsmx_fi_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/osx/libpsmx_fi_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+
+src/unix/libpsmx_fi_la-osd.lo: src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/unix/libpsmx_fi_la-osd.lo -MD -MP -MF src/unix/$(DEPDIR)/libpsmx_fi_la-osd.Tpo -c -o src/unix/libpsmx_fi_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/unix/$(DEPDIR)/libpsmx_fi_la-osd.Tpo src/unix/$(DEPDIR)/libpsmx_fi_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/unix/osd.c' object='src/unix/libpsmx_fi_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/unix/libpsmx_fi_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+
+prov/psm/src/libpsmx_la-psmx_init.lo: prov/psm/src/psmx_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_init.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_init.Tpo -c -o prov/psm/src/libpsmx_la-psmx_init.lo `test -f 'prov/psm/src/psmx_init.c' || echo '$(srcdir)/'`prov/psm/src/psmx_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_init.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_init.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_init.c' object='prov/psm/src/libpsmx_la-psmx_init.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_init.lo `test -f 'prov/psm/src/psmx_init.c' || echo '$(srcdir)/'`prov/psm/src/psmx_init.c
+
+prov/psm/src/libpsmx_la-psmx_fabric.lo: prov/psm/src/psmx_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_fabric.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_fabric.Tpo -c -o prov/psm/src/libpsmx_la-psmx_fabric.lo `test -f 'prov/psm/src/psmx_fabric.c' || echo '$(srcdir)/'`prov/psm/src/psmx_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_fabric.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_fabric.c' object='prov/psm/src/libpsmx_la-psmx_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_fabric.lo `test -f 'prov/psm/src/psmx_fabric.c' || echo '$(srcdir)/'`prov/psm/src/psmx_fabric.c
+
+prov/psm/src/libpsmx_la-psmx_domain.lo: prov/psm/src/psmx_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_domain.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_domain.Tpo -c -o prov/psm/src/libpsmx_la-psmx_domain.lo `test -f 'prov/psm/src/psmx_domain.c' || echo '$(srcdir)/'`prov/psm/src/psmx_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_domain.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_domain.c' object='prov/psm/src/libpsmx_la-psmx_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_domain.lo `test -f 'prov/psm/src/psmx_domain.c' || echo '$(srcdir)/'`prov/psm/src/psmx_domain.c
+
+prov/psm/src/libpsmx_la-psmx_cq.lo: prov/psm/src/psmx_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_cq.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_cq.Tpo -c -o prov/psm/src/libpsmx_la-psmx_cq.lo `test -f 'prov/psm/src/psmx_cq.c' || echo '$(srcdir)/'`prov/psm/src/psmx_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_cq.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_cq.c' object='prov/psm/src/libpsmx_la-psmx_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_cq.lo `test -f 'prov/psm/src/psmx_cq.c' || echo '$(srcdir)/'`prov/psm/src/psmx_cq.c
+
+prov/psm/src/libpsmx_la-psmx_cntr.lo: prov/psm/src/psmx_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_cntr.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_cntr.Tpo -c -o prov/psm/src/libpsmx_la-psmx_cntr.lo `test -f 'prov/psm/src/psmx_cntr.c' || echo '$(srcdir)/'`prov/psm/src/psmx_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_cntr.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_cntr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_cntr.c' object='prov/psm/src/libpsmx_la-psmx_cntr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_cntr.lo `test -f 'prov/psm/src/psmx_cntr.c' || echo '$(srcdir)/'`prov/psm/src/psmx_cntr.c
+
+prov/psm/src/libpsmx_la-psmx_av.lo: prov/psm/src/psmx_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_av.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_av.Tpo -c -o prov/psm/src/libpsmx_la-psmx_av.lo `test -f 'prov/psm/src/psmx_av.c' || echo '$(srcdir)/'`prov/psm/src/psmx_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_av.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_av.c' object='prov/psm/src/libpsmx_la-psmx_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_av.lo `test -f 'prov/psm/src/psmx_av.c' || echo '$(srcdir)/'`prov/psm/src/psmx_av.c
+
+prov/psm/src/libpsmx_la-psmx_ep.lo: prov/psm/src/psmx_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_ep.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_ep.Tpo -c -o prov/psm/src/libpsmx_la-psmx_ep.lo `test -f 'prov/psm/src/psmx_ep.c' || echo '$(srcdir)/'`prov/psm/src/psmx_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_ep.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_ep.c' object='prov/psm/src/libpsmx_la-psmx_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_ep.lo `test -f 'prov/psm/src/psmx_ep.c' || echo '$(srcdir)/'`prov/psm/src/psmx_ep.c
+
+prov/psm/src/libpsmx_la-psmx_cm.lo: prov/psm/src/psmx_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_cm.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_cm.Tpo -c -o prov/psm/src/libpsmx_la-psmx_cm.lo `test -f 'prov/psm/src/psmx_cm.c' || echo '$(srcdir)/'`prov/psm/src/psmx_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_cm.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_cm.c' object='prov/psm/src/libpsmx_la-psmx_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_cm.lo `test -f 'prov/psm/src/psmx_cm.c' || echo '$(srcdir)/'`prov/psm/src/psmx_cm.c
+
+prov/psm/src/libpsmx_la-psmx_tagged.lo: prov/psm/src/psmx_tagged.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_tagged.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_tagged.Tpo -c -o prov/psm/src/libpsmx_la-psmx_tagged.lo `test -f 'prov/psm/src/psmx_tagged.c' || echo '$(srcdir)/'`prov/psm/src/psmx_tagged.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_tagged.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_tagged.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_tagged.c' object='prov/psm/src/libpsmx_la-psmx_tagged.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_tagged.lo `test -f 'prov/psm/src/psmx_tagged.c' || echo '$(srcdir)/'`prov/psm/src/psmx_tagged.c
+
+prov/psm/src/libpsmx_la-psmx_msg.lo: prov/psm/src/psmx_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_msg.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_msg.Tpo -c -o prov/psm/src/libpsmx_la-psmx_msg.lo `test -f 'prov/psm/src/psmx_msg.c' || echo '$(srcdir)/'`prov/psm/src/psmx_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_msg.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_msg.c' object='prov/psm/src/libpsmx_la-psmx_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_msg.lo `test -f 'prov/psm/src/psmx_msg.c' || echo '$(srcdir)/'`prov/psm/src/psmx_msg.c
+
+prov/psm/src/libpsmx_la-psmx_msg2.lo: prov/psm/src/psmx_msg2.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_msg2.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_msg2.Tpo -c -o prov/psm/src/libpsmx_la-psmx_msg2.lo `test -f 'prov/psm/src/psmx_msg2.c' || echo '$(srcdir)/'`prov/psm/src/psmx_msg2.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_msg2.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_msg2.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_msg2.c' object='prov/psm/src/libpsmx_la-psmx_msg2.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_msg2.lo `test -f 'prov/psm/src/psmx_msg2.c' || echo '$(srcdir)/'`prov/psm/src/psmx_msg2.c
+
+prov/psm/src/libpsmx_la-psmx_rma.lo: prov/psm/src/psmx_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_rma.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_rma.Tpo -c -o prov/psm/src/libpsmx_la-psmx_rma.lo `test -f 'prov/psm/src/psmx_rma.c' || echo '$(srcdir)/'`prov/psm/src/psmx_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_rma.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_rma.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_rma.c' object='prov/psm/src/libpsmx_la-psmx_rma.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_rma.lo `test -f 'prov/psm/src/psmx_rma.c' || echo '$(srcdir)/'`prov/psm/src/psmx_rma.c
+
+prov/psm/src/libpsmx_la-psmx_atomic.lo: prov/psm/src/psmx_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_atomic.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_atomic.Tpo -c -o prov/psm/src/libpsmx_la-psmx_atomic.lo `test -f 'prov/psm/src/psmx_atomic.c' || echo '$(srcdir)/'`prov/psm/src/psmx_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_atomic.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_atomic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_atomic.c' object='prov/psm/src/libpsmx_la-psmx_atomic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_atomic.lo `test -f 'prov/psm/src/psmx_atomic.c' || echo '$(srcdir)/'`prov/psm/src/psmx_atomic.c
+
+prov/psm/src/libpsmx_la-psmx_am.lo: prov/psm/src/psmx_am.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_am.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_am.Tpo -c -o prov/psm/src/libpsmx_la-psmx_am.lo `test -f 'prov/psm/src/psmx_am.c' || echo '$(srcdir)/'`prov/psm/src/psmx_am.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_am.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_am.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_am.c' object='prov/psm/src/libpsmx_la-psmx_am.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_am.lo `test -f 'prov/psm/src/psmx_am.c' || echo '$(srcdir)/'`prov/psm/src/psmx_am.c
+
+prov/psm/src/libpsmx_la-psmx_mr.lo: prov/psm/src/psmx_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_mr.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_mr.Tpo -c -o prov/psm/src/libpsmx_la-psmx_mr.lo `test -f 'prov/psm/src/psmx_mr.c' || echo '$(srcdir)/'`prov/psm/src/psmx_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_mr.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_mr.c' object='prov/psm/src/libpsmx_la-psmx_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_mr.lo `test -f 'prov/psm/src/psmx_mr.c' || echo '$(srcdir)/'`prov/psm/src/psmx_mr.c
+
+prov/psm/src/libpsmx_la-psmx_wait.lo: prov/psm/src/psmx_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_wait.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_wait.Tpo -c -o prov/psm/src/libpsmx_la-psmx_wait.lo `test -f 'prov/psm/src/psmx_wait.c' || echo '$(srcdir)/'`prov/psm/src/psmx_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_wait.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_wait.c' object='prov/psm/src/libpsmx_la-psmx_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_wait.lo `test -f 'prov/psm/src/psmx_wait.c' || echo '$(srcdir)/'`prov/psm/src/psmx_wait.c
+
+prov/psm/src/libpsmx_la-psmx_util.lo: prov/psm/src/psmx_util.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm/src/libpsmx_la-psmx_util.lo -MD -MP -MF prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_util.Tpo -c -o prov/psm/src/libpsmx_la-psmx_util.lo `test -f 'prov/psm/src/psmx_util.c' || echo '$(srcdir)/'`prov/psm/src/psmx_util.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_util.Tpo prov/psm/src/$(DEPDIR)/libpsmx_la-psmx_util.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm/src/psmx_util.c' object='prov/psm/src/libpsmx_la-psmx_util.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm/src/libpsmx_la-psmx_util.lo `test -f 'prov/psm/src/psmx_util.c' || echo '$(srcdir)/'`prov/psm/src/psmx_util.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_init.lo: prov/psm2/src/psmx2_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_init.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_init.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_init.lo `test -f 'prov/psm2/src/psmx2_init.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_init.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_init.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_init.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_init.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_init.lo `test -f 'prov/psm2/src/psmx2_init.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_init.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_fabric.lo: prov/psm2/src/psmx2_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_fabric.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_fabric.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_fabric.lo `test -f 'prov/psm2/src/psmx2_fabric.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_fabric.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_fabric.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_fabric.lo `test -f 'prov/psm2/src/psmx2_fabric.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_fabric.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_domain.lo: prov/psm2/src/psmx2_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_domain.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_domain.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_domain.lo `test -f 'prov/psm2/src/psmx2_domain.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_domain.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_domain.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_domain.lo `test -f 'prov/psm2/src/psmx2_domain.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_domain.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_cq.lo: prov/psm2/src/psmx2_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_cq.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_cq.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_cq.lo `test -f 'prov/psm2/src/psmx2_cq.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_cq.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_cq.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_cq.lo `test -f 'prov/psm2/src/psmx2_cq.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_cq.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_cntr.lo: prov/psm2/src/psmx2_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_cntr.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_cntr.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_cntr.lo `test -f 'prov/psm2/src/psmx2_cntr.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_cntr.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_cntr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_cntr.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_cntr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_cntr.lo `test -f 'prov/psm2/src/psmx2_cntr.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_cntr.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_av.lo: prov/psm2/src/psmx2_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_av.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_av.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_av.lo `test -f 'prov/psm2/src/psmx2_av.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_av.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_av.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_av.lo `test -f 'prov/psm2/src/psmx2_av.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_av.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_ep.lo: prov/psm2/src/psmx2_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_ep.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_ep.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_ep.lo `test -f 'prov/psm2/src/psmx2_ep.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_ep.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_ep.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_ep.lo `test -f 'prov/psm2/src/psmx2_ep.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_ep.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_cm.lo: prov/psm2/src/psmx2_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_cm.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_cm.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_cm.lo `test -f 'prov/psm2/src/psmx2_cm.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_cm.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_cm.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_cm.lo `test -f 'prov/psm2/src/psmx2_cm.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_cm.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_tagged.lo: prov/psm2/src/psmx2_tagged.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_tagged.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_tagged.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_tagged.lo `test -f 'prov/psm2/src/psmx2_tagged.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_tagged.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_tagged.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_tagged.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_tagged.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_tagged.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_tagged.lo `test -f 'prov/psm2/src/psmx2_tagged.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_tagged.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_msg.lo: prov/psm2/src/psmx2_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_msg.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_msg.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_msg.lo `test -f 'prov/psm2/src/psmx2_msg.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_msg.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_msg.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_msg.lo `test -f 'prov/psm2/src/psmx2_msg.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_msg.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_rma.lo: prov/psm2/src/psmx2_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_rma.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_rma.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_rma.lo `test -f 'prov/psm2/src/psmx2_rma.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_rma.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_rma.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_rma.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_rma.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_rma.lo `test -f 'prov/psm2/src/psmx2_rma.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_rma.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_atomic.lo: prov/psm2/src/psmx2_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_atomic.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_atomic.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_atomic.lo `test -f 'prov/psm2/src/psmx2_atomic.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_atomic.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_atomic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_atomic.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_atomic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_atomic.lo `test -f 'prov/psm2/src/psmx2_atomic.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_atomic.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_am.lo: prov/psm2/src/psmx2_am.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_am.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_am.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_am.lo `test -f 'prov/psm2/src/psmx2_am.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_am.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_am.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_am.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_am.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_am.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_am.lo `test -f 'prov/psm2/src/psmx2_am.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_am.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_mr.lo: prov/psm2/src/psmx2_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_mr.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_mr.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_mr.lo `test -f 'prov/psm2/src/psmx2_mr.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_mr.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_mr.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_mr.lo `test -f 'prov/psm2/src/psmx2_mr.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_mr.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_wait.lo: prov/psm2/src/psmx2_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_wait.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_wait.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_wait.lo `test -f 'prov/psm2/src/psmx2_wait.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_wait.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_wait.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_wait.lo `test -f 'prov/psm2/src/psmx2_wait.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_wait.c
+
+prov/psm2/src/libpsmx2_fi_la-psmx2_util.lo: prov/psm2/src/psmx2_util.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_fi_la-psmx2_util.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_util.Tpo -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_util.lo `test -f 'prov/psm2/src/psmx2_util.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_util.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_util.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_fi_la-psmx2_util.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_util.c' object='prov/psm2/src/libpsmx2_fi_la-psmx2_util.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_fi_la-psmx2_util.lo `test -f 'prov/psm2/src/psmx2_util.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_util.c
+
+src/libpsmx2_fi_la-common.lo: src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libpsmx2_fi_la-common.lo -MD -MP -MF src/$(DEPDIR)/libpsmx2_fi_la-common.Tpo -c -o src/libpsmx2_fi_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libpsmx2_fi_la-common.Tpo src/$(DEPDIR)/libpsmx2_fi_la-common.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/common.c' object='src/libpsmx2_fi_la-common.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libpsmx2_fi_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+
+src/libpsmx2_fi_la-enosys.lo: src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libpsmx2_fi_la-enosys.lo -MD -MP -MF src/$(DEPDIR)/libpsmx2_fi_la-enosys.Tpo -c -o src/libpsmx2_fi_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libpsmx2_fi_la-enosys.Tpo src/$(DEPDIR)/libpsmx2_fi_la-enosys.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/enosys.c' object='src/libpsmx2_fi_la-enosys.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libpsmx2_fi_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+
+src/libpsmx2_fi_la-rbtree.lo: src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libpsmx2_fi_la-rbtree.lo -MD -MP -MF src/$(DEPDIR)/libpsmx2_fi_la-rbtree.Tpo -c -o src/libpsmx2_fi_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libpsmx2_fi_la-rbtree.Tpo src/$(DEPDIR)/libpsmx2_fi_la-rbtree.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/rbtree.c' object='src/libpsmx2_fi_la-rbtree.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libpsmx2_fi_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+
+src/libpsmx2_fi_la-fasthash.lo: src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libpsmx2_fi_la-fasthash.lo -MD -MP -MF src/$(DEPDIR)/libpsmx2_fi_la-fasthash.Tpo -c -o src/libpsmx2_fi_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libpsmx2_fi_la-fasthash.Tpo src/$(DEPDIR)/libpsmx2_fi_la-fasthash.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/fasthash.c' object='src/libpsmx2_fi_la-fasthash.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libpsmx2_fi_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+
+src/libpsmx2_fi_la-indexer.lo: src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libpsmx2_fi_la-indexer.lo -MD -MP -MF src/$(DEPDIR)/libpsmx2_fi_la-indexer.Tpo -c -o src/libpsmx2_fi_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libpsmx2_fi_la-indexer.Tpo src/$(DEPDIR)/libpsmx2_fi_la-indexer.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/indexer.c' object='src/libpsmx2_fi_la-indexer.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libpsmx2_fi_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+
+prov/util/src/libpsmx2_fi_la-util_attr.lo: prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx2_fi_la-util_attr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_attr.Tpo -c -o prov/util/src/libpsmx2_fi_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_attr.Tpo prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_attr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_attr.c' object='prov/util/src/libpsmx2_fi_la-util_attr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx2_fi_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+
+prov/util/src/libpsmx2_fi_la-util_av.lo: prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx2_fi_la-util_av.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_av.Tpo -c -o prov/util/src/libpsmx2_fi_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_av.Tpo prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_av.c' object='prov/util/src/libpsmx2_fi_la-util_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx2_fi_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+
+prov/util/src/libpsmx2_fi_la-util_cq.lo: prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx2_fi_la-util_cq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_cq.Tpo -c -o prov/util/src/libpsmx2_fi_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_cq.Tpo prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_cq.c' object='prov/util/src/libpsmx2_fi_la-util_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx2_fi_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+
+prov/util/src/libpsmx2_fi_la-util_domain.lo: prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx2_fi_la-util_domain.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_domain.Tpo -c -o prov/util/src/libpsmx2_fi_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_domain.Tpo prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_domain.c' object='prov/util/src/libpsmx2_fi_la-util_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx2_fi_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+
+prov/util/src/libpsmx2_fi_la-util_ep.lo: prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx2_fi_la-util_ep.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_ep.Tpo -c -o prov/util/src/libpsmx2_fi_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_ep.Tpo prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_ep.c' object='prov/util/src/libpsmx2_fi_la-util_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx2_fi_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+
+prov/util/src/libpsmx2_fi_la-util_eq.lo: prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx2_fi_la-util_eq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_eq.Tpo -c -o prov/util/src/libpsmx2_fi_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_eq.Tpo prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_eq.c' object='prov/util/src/libpsmx2_fi_la-util_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx2_fi_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+
+prov/util/src/libpsmx2_fi_la-util_fabric.lo: prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx2_fi_la-util_fabric.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_fabric.Tpo -c -o prov/util/src/libpsmx2_fi_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_fabric.Tpo prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_fabric.c' object='prov/util/src/libpsmx2_fi_la-util_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx2_fi_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+
+prov/util/src/libpsmx2_fi_la-util_main.lo: prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx2_fi_la-util_main.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_main.Tpo -c -o prov/util/src/libpsmx2_fi_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_main.Tpo prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_main.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_main.c' object='prov/util/src/libpsmx2_fi_la-util_main.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx2_fi_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+
+prov/util/src/libpsmx2_fi_la-util_poll.lo: prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx2_fi_la-util_poll.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_poll.Tpo -c -o prov/util/src/libpsmx2_fi_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_poll.Tpo prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_poll.c' object='prov/util/src/libpsmx2_fi_la-util_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx2_fi_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+
+prov/util/src/libpsmx2_fi_la-util_wait.lo: prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx2_fi_la-util_wait.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_wait.Tpo -c -o prov/util/src/libpsmx2_fi_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_wait.Tpo prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_wait.c' object='prov/util/src/libpsmx2_fi_la-util_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx2_fi_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+
+prov/util/src/libpsmx2_fi_la-util_buf.lo: prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx2_fi_la-util_buf.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_buf.Tpo -c -o prov/util/src/libpsmx2_fi_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_buf.Tpo prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_buf.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_buf.c' object='prov/util/src/libpsmx2_fi_la-util_buf.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx2_fi_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+
+prov/util/src/libpsmx2_fi_la-util_mr.lo: prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libpsmx2_fi_la-util_mr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_mr.Tpo -c -o prov/util/src/libpsmx2_fi_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_mr.Tpo prov/util/src/$(DEPDIR)/libpsmx2_fi_la-util_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_mr.c' object='prov/util/src/libpsmx2_fi_la-util_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libpsmx2_fi_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+
+src/osx/libpsmx2_fi_la-osd.lo: src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/osx/libpsmx2_fi_la-osd.lo -MD -MP -MF src/osx/$(DEPDIR)/libpsmx2_fi_la-osd.Tpo -c -o src/osx/libpsmx2_fi_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/osx/$(DEPDIR)/libpsmx2_fi_la-osd.Tpo src/osx/$(DEPDIR)/libpsmx2_fi_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/osx/osd.c' object='src/osx/libpsmx2_fi_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/osx/libpsmx2_fi_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+
+src/unix/libpsmx2_fi_la-osd.lo: src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/unix/libpsmx2_fi_la-osd.lo -MD -MP -MF src/unix/$(DEPDIR)/libpsmx2_fi_la-osd.Tpo -c -o src/unix/libpsmx2_fi_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/unix/$(DEPDIR)/libpsmx2_fi_la-osd.Tpo src/unix/$(DEPDIR)/libpsmx2_fi_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/unix/osd.c' object='src/unix/libpsmx2_fi_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/unix/libpsmx2_fi_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+
+prov/psm2/src/libpsmx2_la-psmx2_init.lo: prov/psm2/src/psmx2_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_init.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_init.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_init.lo `test -f 'prov/psm2/src/psmx2_init.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_init.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_init.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_init.c' object='prov/psm2/src/libpsmx2_la-psmx2_init.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_init.lo `test -f 'prov/psm2/src/psmx2_init.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_init.c
+
+prov/psm2/src/libpsmx2_la-psmx2_fabric.lo: prov/psm2/src/psmx2_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_fabric.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_fabric.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_fabric.lo `test -f 'prov/psm2/src/psmx2_fabric.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_fabric.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_fabric.c' object='prov/psm2/src/libpsmx2_la-psmx2_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_fabric.lo `test -f 'prov/psm2/src/psmx2_fabric.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_fabric.c
+
+prov/psm2/src/libpsmx2_la-psmx2_domain.lo: prov/psm2/src/psmx2_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_domain.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_domain.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_domain.lo `test -f 'prov/psm2/src/psmx2_domain.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_domain.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_domain.c' object='prov/psm2/src/libpsmx2_la-psmx2_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_domain.lo `test -f 'prov/psm2/src/psmx2_domain.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_domain.c
+
+prov/psm2/src/libpsmx2_la-psmx2_cq.lo: prov/psm2/src/psmx2_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_cq.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_cq.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_cq.lo `test -f 'prov/psm2/src/psmx2_cq.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_cq.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_cq.c' object='prov/psm2/src/libpsmx2_la-psmx2_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_cq.lo `test -f 'prov/psm2/src/psmx2_cq.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_cq.c
+
+prov/psm2/src/libpsmx2_la-psmx2_cntr.lo: prov/psm2/src/psmx2_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_cntr.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_cntr.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_cntr.lo `test -f 'prov/psm2/src/psmx2_cntr.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_cntr.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_cntr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_cntr.c' object='prov/psm2/src/libpsmx2_la-psmx2_cntr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_cntr.lo `test -f 'prov/psm2/src/psmx2_cntr.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_cntr.c
+
+prov/psm2/src/libpsmx2_la-psmx2_av.lo: prov/psm2/src/psmx2_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_av.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_av.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_av.lo `test -f 'prov/psm2/src/psmx2_av.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_av.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_av.c' object='prov/psm2/src/libpsmx2_la-psmx2_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_av.lo `test -f 'prov/psm2/src/psmx2_av.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_av.c
+
+prov/psm2/src/libpsmx2_la-psmx2_ep.lo: prov/psm2/src/psmx2_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_ep.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_ep.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_ep.lo `test -f 'prov/psm2/src/psmx2_ep.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_ep.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_ep.c' object='prov/psm2/src/libpsmx2_la-psmx2_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_ep.lo `test -f 'prov/psm2/src/psmx2_ep.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_ep.c
+
+prov/psm2/src/libpsmx2_la-psmx2_cm.lo: prov/psm2/src/psmx2_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_cm.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_cm.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_cm.lo `test -f 'prov/psm2/src/psmx2_cm.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_cm.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_cm.c' object='prov/psm2/src/libpsmx2_la-psmx2_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_cm.lo `test -f 'prov/psm2/src/psmx2_cm.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_cm.c
+
+prov/psm2/src/libpsmx2_la-psmx2_tagged.lo: prov/psm2/src/psmx2_tagged.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_tagged.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_tagged.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_tagged.lo `test -f 'prov/psm2/src/psmx2_tagged.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_tagged.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_tagged.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_tagged.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_tagged.c' object='prov/psm2/src/libpsmx2_la-psmx2_tagged.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_tagged.lo `test -f 'prov/psm2/src/psmx2_tagged.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_tagged.c
+
+prov/psm2/src/libpsmx2_la-psmx2_msg.lo: prov/psm2/src/psmx2_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_msg.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_msg.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_msg.lo `test -f 'prov/psm2/src/psmx2_msg.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_msg.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_msg.c' object='prov/psm2/src/libpsmx2_la-psmx2_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_msg.lo `test -f 'prov/psm2/src/psmx2_msg.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_msg.c
+
+prov/psm2/src/libpsmx2_la-psmx2_rma.lo: prov/psm2/src/psmx2_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_rma.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_rma.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_rma.lo `test -f 'prov/psm2/src/psmx2_rma.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_rma.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_rma.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_rma.c' object='prov/psm2/src/libpsmx2_la-psmx2_rma.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_rma.lo `test -f 'prov/psm2/src/psmx2_rma.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_rma.c
+
+prov/psm2/src/libpsmx2_la-psmx2_atomic.lo: prov/psm2/src/psmx2_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_atomic.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_atomic.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_atomic.lo `test -f 'prov/psm2/src/psmx2_atomic.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_atomic.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_atomic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_atomic.c' object='prov/psm2/src/libpsmx2_la-psmx2_atomic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_atomic.lo `test -f 'prov/psm2/src/psmx2_atomic.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_atomic.c
+
+prov/psm2/src/libpsmx2_la-psmx2_am.lo: prov/psm2/src/psmx2_am.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_am.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_am.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_am.lo `test -f 'prov/psm2/src/psmx2_am.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_am.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_am.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_am.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_am.c' object='prov/psm2/src/libpsmx2_la-psmx2_am.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_am.lo `test -f 'prov/psm2/src/psmx2_am.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_am.c
+
+prov/psm2/src/libpsmx2_la-psmx2_mr.lo: prov/psm2/src/psmx2_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_mr.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_mr.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_mr.lo `test -f 'prov/psm2/src/psmx2_mr.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_mr.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_mr.c' object='prov/psm2/src/libpsmx2_la-psmx2_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_mr.lo `test -f 'prov/psm2/src/psmx2_mr.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_mr.c
+
+prov/psm2/src/libpsmx2_la-psmx2_wait.lo: prov/psm2/src/psmx2_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_wait.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_wait.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_wait.lo `test -f 'prov/psm2/src/psmx2_wait.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_wait.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_wait.c' object='prov/psm2/src/libpsmx2_la-psmx2_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_wait.lo `test -f 'prov/psm2/src/psmx2_wait.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_wait.c
+
+prov/psm2/src/libpsmx2_la-psmx2_util.lo: prov/psm2/src/psmx2_util.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/psm2/src/libpsmx2_la-psmx2_util.lo -MD -MP -MF prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_util.Tpo -c -o prov/psm2/src/libpsmx2_la-psmx2_util.lo `test -f 'prov/psm2/src/psmx2_util.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_util.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_util.Tpo prov/psm2/src/$(DEPDIR)/libpsmx2_la-psmx2_util.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/psm2/src/psmx2_util.c' object='prov/psm2/src/libpsmx2_la-psmx2_util.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libpsmx2_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/psm2/src/libpsmx2_la-psmx2_util.lo `test -f 'prov/psm2/src/psmx2_util.c' || echo '$(srcdir)/'`prov/psm2/src/psmx2_util.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-libnl_utils_common.lo: prov/usnic/src/usnic_direct/libnl_utils_common.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-libnl_utils_common.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-libnl_utils_common.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-libnl_utils_common.lo `test -f 'prov/usnic/src/usnic_direct/libnl_ [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-libnl_utils_common.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-libnl_utils_common.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/libnl_utils_common.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-libnl_utils_common.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-libnl_utils_common.lo `test -f 'prov/usnic/src/usnic_direct/libnl_utils_common.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/libnl_utils_common.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_caps.lo: prov/usnic/src/usnic_direct/usd_caps.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_caps.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_caps.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_caps.lo `test -f 'prov/usnic/src/usnic_direct/usd_caps.c' || echo '$(srcdir)/'`pro [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_caps.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_caps.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_caps.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_caps.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_caps.lo `test -f 'prov/usnic/src/usnic_direct/usd_caps.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_caps.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_dest.lo: prov/usnic/src/usnic_direct/usd_dest.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_dest.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_dest.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_dest.lo `test -f 'prov/usnic/src/usnic_direct/usd_dest.c' || echo '$(srcdir)/'`pro [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_dest.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_dest.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_dest.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_dest.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_dest.lo `test -f 'prov/usnic/src/usnic_direct/usd_dest.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_dest.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_device.lo: prov/usnic/src/usnic_direct/usd_device.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_device.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_device.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_device.lo `test -f 'prov/usnic/src/usnic_direct/usd_device.c' || echo '$(srcdi [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_device.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_device.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_device.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_device.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_device.lo `test -f 'prov/usnic/src/usnic_direct/usd_device.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_device.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_event.lo: prov/usnic/src/usnic_direct/usd_event.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_event.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_event.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_event.lo `test -f 'prov/usnic/src/usnic_direct/usd_event.c' || echo '$(srcdir)/' [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_event.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_event.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_event.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_event.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_event.lo `test -f 'prov/usnic/src/usnic_direct/usd_event.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_event.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_enum.lo: prov/usnic/src/usnic_direct/usd_enum.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_enum.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_enum.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_enum.lo `test -f 'prov/usnic/src/usnic_direct/usd_enum.c' || echo '$(srcdir)/'`pro [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_enum.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_enum.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_enum.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_enum.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_enum.lo `test -f 'prov/usnic/src/usnic_direct/usd_enum.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_enum.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_cmd.lo: prov/usnic/src/usnic_direct/usd_ib_cmd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_cmd.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_ib_cmd.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_cmd.lo `test -f 'prov/usnic/src/usnic_direct/usd_ib_cmd.c' || echo '$(srcdi [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_ib_cmd.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_ib_cmd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_ib_cmd.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_cmd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_cmd.lo `test -f 'prov/usnic/src/usnic_direct/usd_ib_cmd.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_ib_cmd.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_sysfs.lo: prov/usnic/src/usnic_direct/usd_ib_sysfs.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_sysfs.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_ib_sysfs.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_sysfs.lo `test -f 'prov/usnic/src/usnic_direct/usd_ib_sysfs.c' || echo [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_ib_sysfs.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_ib_sysfs.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_ib_sysfs.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_sysfs.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_ib_sysfs.lo `test -f 'prov/usnic/src/usnic_direct/usd_ib_sysfs.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_ib_sysfs.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_mem.lo: prov/usnic/src/usnic_direct/usd_mem.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_mem.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_mem.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_mem.lo `test -f 'prov/usnic/src/usnic_direct/usd_mem.c' || echo '$(srcdir)/'`prov/us [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_mem.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_mem.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_mem.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_mem.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_mem.lo `test -f 'prov/usnic/src/usnic_direct/usd_mem.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_mem.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_poll.lo: prov/usnic/src/usnic_direct/usd_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_poll.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_poll.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_poll.lo `test -f 'prov/usnic/src/usnic_direct/usd_poll.c' || echo '$(srcdir)/'`pro [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_poll.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_poll.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_poll.lo `test -f 'prov/usnic/src/usnic_direct/usd_poll.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_poll.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post.lo: prov/usnic/src/usnic_direct/usd_post.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post.lo `test -f 'prov/usnic/src/usnic_direct/usd_post.c' || echo '$(srcdir)/'`pro [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_post.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post.lo `test -f 'prov/usnic/src/usnic_direct/usd_post.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_post.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_raw.lo: prov/usnic/src/usnic_direct/usd_post_ud_raw.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_raw.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post_ud_raw.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_raw.lo `test -f 'prov/usnic/src/usnic_direct/usd_post_ud_raw [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post_ud_raw.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post_ud_raw.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_post_ud_raw.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_raw.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_raw.lo `test -f 'prov/usnic/src/usnic_direct/usd_post_ud_raw.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_post_ud_raw.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_udp.lo: prov/usnic/src/usnic_direct/usd_post_ud_udp.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_udp.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post_ud_udp.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_udp.lo `test -f 'prov/usnic/src/usnic_direct/usd_post_ud_udp [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post_ud_udp.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post_ud_udp.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_post_ud_udp.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_udp.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_udp.lo `test -f 'prov/usnic/src/usnic_direct/usd_post_ud_udp.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_post_ud_udp.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_pio_udp.lo: prov/usnic/src/usnic_direct/usd_post_ud_pio_udp.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_pio_udp.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post_ud_pio_udp.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_pio_udp.lo `test -f 'prov/usnic/src/usnic_direct/usd [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post_ud_pio_udp.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_post_ud_pio_udp.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_post_ud_pio_udp.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_pio_udp.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_post_ud_pio_udp.lo `test -f 'prov/usnic/src/usnic_direct/usd_post_ud_pio_udp.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_post_ud_pio_udp.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_queues.lo: prov/usnic/src/usnic_direct/usd_queues.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_queues.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_queues.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_queues.lo `test -f 'prov/usnic/src/usnic_direct/usd_queues.c' || echo '$(srcdi [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_queues.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_queues.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_queues.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_queues.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_queues.lo `test -f 'prov/usnic/src/usnic_direct/usd_queues.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_queues.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_socket.lo: prov/usnic/src/usnic_direct/usd_socket.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_socket.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_socket.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_socket.lo `test -f 'prov/usnic/src/usnic_direct/usd_socket.c' || echo '$(srcdi [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_socket.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_socket.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_socket.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_socket.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_socket.lo `test -f 'prov/usnic/src/usnic_direct/usd_socket.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_socket.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usd_vnic.lo: prov/usnic/src/usnic_direct/usd_vnic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usd_vnic.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_vnic.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_vnic.lo `test -f 'prov/usnic/src/usnic_direct/usd_vnic.c' || echo '$(srcdir)/'`pro [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_vnic.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usd_vnic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_vnic.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usd_vnic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usd_vnic.lo `test -f 'prov/usnic/src/usnic_direct/usd_vnic.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_vnic.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-usnic_ip_utils.lo: prov/usnic/src/usnic_direct/usnic_ip_utils.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-usnic_ip_utils.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usnic_ip_utils.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usnic_ip_utils.lo `test -f 'prov/usnic/src/usnic_direct/usnic_ip_utils.c' [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usnic_ip_utils.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-usnic_ip_utils.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usnic_ip_utils.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-usnic_ip_utils.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-usnic_ip_utils.lo `test -f 'prov/usnic/src/usnic_direct/usnic_ip_utils.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usnic_ip_utils.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_cq.lo: prov/usnic/src/usnic_direct/vnic_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_cq.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_cq.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_cq.lo `test -f 'prov/usnic/src/usnic_direct/vnic_cq.c' || echo '$(srcdir)/'`prov/us [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_cq.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/vnic_cq.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_cq.lo `test -f 'prov/usnic/src/usnic_direct/vnic_cq.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/vnic_cq.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_dev.lo: prov/usnic/src/usnic_direct/vnic_dev.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_dev.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_dev.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_dev.lo `test -f 'prov/usnic/src/usnic_direct/vnic_dev.c' || echo '$(srcdir)/'`pro [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_dev.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_dev.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/vnic_dev.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_dev.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_dev.lo `test -f 'prov/usnic/src/usnic_direct/vnic_dev.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/vnic_dev.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_rq.lo: prov/usnic/src/usnic_direct/vnic_rq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_rq.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_rq.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_rq.lo `test -f 'prov/usnic/src/usnic_direct/vnic_rq.c' || echo '$(srcdir)/'`prov/us [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_rq.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_rq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/vnic_rq.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_rq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_rq.lo `test -f 'prov/usnic/src/usnic_direct/vnic_rq.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/vnic_rq.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_wq.lo: prov/usnic/src/usnic_direct/vnic_wq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_wq.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_wq.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_wq.lo `test -f 'prov/usnic/src/usnic_direct/vnic_wq.c' || echo '$(srcdir)/'`prov/us [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_wq.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_wq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/vnic_wq.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_wq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_wq.lo `test -f 'prov/usnic/src/usnic_direct/vnic_wq.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/vnic_wq.c
+
+prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_intr.lo: prov/usnic/src/usnic_direct/vnic_intr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_intr.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_intr.Tpo -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_intr.lo `test -f 'prov/usnic/src/usnic_direct/vnic_intr.c' || echo '$(srcdir)/' [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_intr.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/libusnic_fi_la-vnic_intr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/vnic_intr.c' object='prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_intr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/libusnic_fi_la-vnic_intr.lo `test -f 'prov/usnic/src/usnic_direct/vnic_intr.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/vnic_intr.c
+
+prov/usnic/src/libusnic_fi_la-usdf_av.lo: prov/usnic/src/usdf_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_av.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_av.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_av.lo `test -f 'prov/usnic/src/usdf_av.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_av.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_av.c' object='prov/usnic/src/libusnic_fi_la-usdf_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_av.lo `test -f 'prov/usnic/src/usdf_av.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_av.c
+
+prov/usnic/src/libusnic_fi_la-usdf_cm.lo: prov/usnic/src/usdf_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_cm.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_cm.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_cm.lo `test -f 'prov/usnic/src/usdf_cm.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_cm.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_cm.c' object='prov/usnic/src/libusnic_fi_la-usdf_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_cm.lo `test -f 'prov/usnic/src/usdf_cm.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_cm.c
+
+prov/usnic/src/libusnic_fi_la-usdf_cq.lo: prov/usnic/src/usdf_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_cq.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_cq.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_cq.lo `test -f 'prov/usnic/src/usdf_cq.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_cq.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_cq.c' object='prov/usnic/src/libusnic_fi_la-usdf_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_cq.lo `test -f 'prov/usnic/src/usdf_cq.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_cq.c
+
+prov/usnic/src/libusnic_fi_la-usdf_dgram.lo: prov/usnic/src/usdf_dgram.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_dgram.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_dgram.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_dgram.lo `test -f 'prov/usnic/src/usdf_dgram.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_dgram.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_dgram.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_dgram.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_dgram.c' object='prov/usnic/src/libusnic_fi_la-usdf_dgram.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_dgram.lo `test -f 'prov/usnic/src/usdf_dgram.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_dgram.c
+
+prov/usnic/src/libusnic_fi_la-usdf_domain.lo: prov/usnic/src/usdf_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_domain.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_domain.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_domain.lo `test -f 'prov/usnic/src/usdf_domain.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_domain.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_domain.c' object='prov/usnic/src/libusnic_fi_la-usdf_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_domain.lo `test -f 'prov/usnic/src/usdf_domain.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_domain.c
+
+prov/usnic/src/libusnic_fi_la-usdf_endpoint.lo: prov/usnic/src/usdf_endpoint.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_endpoint.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_endpoint.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_endpoint.lo `test -f 'prov/usnic/src/usdf_endpoint.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_endpoint.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_endpoint.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_endpoint.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_endpoint.c' object='prov/usnic/src/libusnic_fi_la-usdf_endpoint.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_endpoint.lo `test -f 'prov/usnic/src/usdf_endpoint.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_endpoint.c
+
+prov/usnic/src/libusnic_fi_la-usdf_ep_dgram.lo: prov/usnic/src/usdf_ep_dgram.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_ep_dgram.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ep_dgram.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_ep_dgram.lo `test -f 'prov/usnic/src/usdf_ep_dgram.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ep_dgram.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ep_dgram.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ep_dgram.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_ep_dgram.c' object='prov/usnic/src/libusnic_fi_la-usdf_ep_dgram.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_ep_dgram.lo `test -f 'prov/usnic/src/usdf_ep_dgram.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ep_dgram.c
+
+prov/usnic/src/libusnic_fi_la-usdf_ep_msg.lo: prov/usnic/src/usdf_ep_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_ep_msg.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ep_msg.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_ep_msg.lo `test -f 'prov/usnic/src/usdf_ep_msg.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ep_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ep_msg.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ep_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_ep_msg.c' object='prov/usnic/src/libusnic_fi_la-usdf_ep_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_ep_msg.lo `test -f 'prov/usnic/src/usdf_ep_msg.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ep_msg.c
+
+prov/usnic/src/libusnic_fi_la-usdf_ep_rdm.lo: prov/usnic/src/usdf_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_ep_rdm.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ep_rdm.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_ep_rdm.lo `test -f 'prov/usnic/src/usdf_ep_rdm.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ep_rdm.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ep_rdm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_ep_rdm.c' object='prov/usnic/src/libusnic_fi_la-usdf_ep_rdm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_ep_rdm.lo `test -f 'prov/usnic/src/usdf_ep_rdm.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ep_rdm.c
+
+prov/usnic/src/libusnic_fi_la-usdf_eq.lo: prov/usnic/src/usdf_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_eq.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_eq.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_eq.lo `test -f 'prov/usnic/src/usdf_eq.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_eq.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_eq.c' object='prov/usnic/src/libusnic_fi_la-usdf_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_eq.lo `test -f 'prov/usnic/src/usdf_eq.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_eq.c
+
+prov/usnic/src/libusnic_fi_la-usdf_fabric.lo: prov/usnic/src/usdf_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_fabric.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_fabric.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_fabric.lo `test -f 'prov/usnic/src/usdf_fabric.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_fabric.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_fabric.c' object='prov/usnic/src/libusnic_fi_la-usdf_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_fabric.lo `test -f 'prov/usnic/src/usdf_fabric.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_fabric.c
+
+prov/usnic/src/libusnic_fi_la-usdf_mem.lo: prov/usnic/src/usdf_mem.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_mem.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_mem.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_mem.lo `test -f 'prov/usnic/src/usdf_mem.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_mem.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_mem.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_mem.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_mem.c' object='prov/usnic/src/libusnic_fi_la-usdf_mem.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_mem.lo `test -f 'prov/usnic/src/usdf_mem.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_mem.c
+
+prov/usnic/src/libusnic_fi_la-usdf_msg.lo: prov/usnic/src/usdf_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_msg.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_msg.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_msg.lo `test -f 'prov/usnic/src/usdf_msg.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_msg.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_msg.c' object='prov/usnic/src/libusnic_fi_la-usdf_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_msg.lo `test -f 'prov/usnic/src/usdf_msg.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_msg.c
+
+prov/usnic/src/libusnic_fi_la-usdf_pep.lo: prov/usnic/src/usdf_pep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_pep.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_pep.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_pep.lo `test -f 'prov/usnic/src/usdf_pep.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_pep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_pep.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_pep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_pep.c' object='prov/usnic/src/libusnic_fi_la-usdf_pep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_pep.lo `test -f 'prov/usnic/src/usdf_pep.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_pep.c
+
+prov/usnic/src/libusnic_fi_la-usdf_progress.lo: prov/usnic/src/usdf_progress.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_progress.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_progress.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_progress.lo `test -f 'prov/usnic/src/usdf_progress.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_progress.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_progress.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_progress.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_progress.c' object='prov/usnic/src/libusnic_fi_la-usdf_progress.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_progress.lo `test -f 'prov/usnic/src/usdf_progress.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_progress.c
+
+prov/usnic/src/libusnic_fi_la-usdf_rdm.lo: prov/usnic/src/usdf_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_rdm.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_rdm.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_rdm.lo `test -f 'prov/usnic/src/usdf_rdm.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_rdm.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_rdm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_rdm.c' object='prov/usnic/src/libusnic_fi_la-usdf_rdm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_rdm.lo `test -f 'prov/usnic/src/usdf_rdm.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_rdm.c
+
+prov/usnic/src/libusnic_fi_la-usdf_timer.lo: prov/usnic/src/usdf_timer.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_timer.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_timer.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_timer.lo `test -f 'prov/usnic/src/usdf_timer.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_timer.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_timer.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_timer.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_timer.c' object='prov/usnic/src/libusnic_fi_la-usdf_timer.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_timer.lo `test -f 'prov/usnic/src/usdf_timer.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_timer.c
+
+prov/usnic/src/libusnic_fi_la-usdf_poll.lo: prov/usnic/src/usdf_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_poll.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_poll.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_poll.lo `test -f 'prov/usnic/src/usdf_poll.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_poll.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_poll.c' object='prov/usnic/src/libusnic_fi_la-usdf_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_poll.lo `test -f 'prov/usnic/src/usdf_poll.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_poll.c
+
+prov/usnic/src/libusnic_fi_la-usdf_ext.lo: prov/usnic/src/usdf_ext.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_ext.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ext.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_ext.lo `test -f 'prov/usnic/src/usdf_ext.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ext.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ext.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_ext.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_ext.c' object='prov/usnic/src/libusnic_fi_la-usdf_ext.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_ext.lo `test -f 'prov/usnic/src/usdf_ext.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ext.c
+
+prov/usnic/src/libusnic_fi_la-usdf_wait.lo: prov/usnic/src/usdf_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_wait.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_wait.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_wait.lo `test -f 'prov/usnic/src/usdf_wait.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_wait.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_wait.c' object='prov/usnic/src/libusnic_fi_la-usdf_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_wait.lo `test -f 'prov/usnic/src/usdf_wait.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_wait.c
+
+prov/usnic/src/libusnic_fi_la-usdf_fake_ibv.lo: prov/usnic/src/usdf_fake_ibv.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/libusnic_fi_la-usdf_fake_ibv.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_fake_ibv.Tpo -c -o prov/usnic/src/libusnic_fi_la-usdf_fake_ibv.lo `test -f 'prov/usnic/src/usdf_fake_ibv.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_fake_ibv.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_fake_ibv.Tpo prov/usnic/src/$(DEPDIR)/libusnic_fi_la-usdf_fake_ibv.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_fake_ibv.c' object='prov/usnic/src/libusnic_fi_la-usdf_fake_ibv.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/libusnic_fi_la-usdf_fake_ibv.lo `test -f 'prov/usnic/src/usdf_fake_ibv.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_fake_ibv.c
+
+src/libusnic_fi_la-common.lo: src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libusnic_fi_la-common.lo -MD -MP -MF src/$(DEPDIR)/libusnic_fi_la-common.Tpo -c -o src/libusnic_fi_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libusnic_fi_la-common.Tpo src/$(DEPDIR)/libusnic_fi_la-common.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/common.c' object='src/libusnic_fi_la-common.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libusnic_fi_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+
+src/libusnic_fi_la-enosys.lo: src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libusnic_fi_la-enosys.lo -MD -MP -MF src/$(DEPDIR)/libusnic_fi_la-enosys.Tpo -c -o src/libusnic_fi_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libusnic_fi_la-enosys.Tpo src/$(DEPDIR)/libusnic_fi_la-enosys.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/enosys.c' object='src/libusnic_fi_la-enosys.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libusnic_fi_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+
+src/libusnic_fi_la-rbtree.lo: src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libusnic_fi_la-rbtree.lo -MD -MP -MF src/$(DEPDIR)/libusnic_fi_la-rbtree.Tpo -c -o src/libusnic_fi_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libusnic_fi_la-rbtree.Tpo src/$(DEPDIR)/libusnic_fi_la-rbtree.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/rbtree.c' object='src/libusnic_fi_la-rbtree.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libusnic_fi_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+
+src/libusnic_fi_la-fasthash.lo: src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libusnic_fi_la-fasthash.lo -MD -MP -MF src/$(DEPDIR)/libusnic_fi_la-fasthash.Tpo -c -o src/libusnic_fi_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libusnic_fi_la-fasthash.Tpo src/$(DEPDIR)/libusnic_fi_la-fasthash.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/fasthash.c' object='src/libusnic_fi_la-fasthash.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libusnic_fi_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+
+src/libusnic_fi_la-indexer.lo: src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libusnic_fi_la-indexer.lo -MD -MP -MF src/$(DEPDIR)/libusnic_fi_la-indexer.Tpo -c -o src/libusnic_fi_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libusnic_fi_la-indexer.Tpo src/$(DEPDIR)/libusnic_fi_la-indexer.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/indexer.c' object='src/libusnic_fi_la-indexer.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libusnic_fi_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+
+prov/util/src/libusnic_fi_la-util_attr.lo: prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libusnic_fi_la-util_attr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libusnic_fi_la-util_attr.Tpo -c -o prov/util/src/libusnic_fi_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libusnic_fi_la-util_attr.Tpo prov/util/src/$(DEPDIR)/libusnic_fi_la-util_attr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_attr.c' object='prov/util/src/libusnic_fi_la-util_attr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libusnic_fi_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+
+prov/util/src/libusnic_fi_la-util_av.lo: prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libusnic_fi_la-util_av.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libusnic_fi_la-util_av.Tpo -c -o prov/util/src/libusnic_fi_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libusnic_fi_la-util_av.Tpo prov/util/src/$(DEPDIR)/libusnic_fi_la-util_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_av.c' object='prov/util/src/libusnic_fi_la-util_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libusnic_fi_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+
+prov/util/src/libusnic_fi_la-util_cq.lo: prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libusnic_fi_la-util_cq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libusnic_fi_la-util_cq.Tpo -c -o prov/util/src/libusnic_fi_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libusnic_fi_la-util_cq.Tpo prov/util/src/$(DEPDIR)/libusnic_fi_la-util_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_cq.c' object='prov/util/src/libusnic_fi_la-util_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libusnic_fi_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+
+prov/util/src/libusnic_fi_la-util_domain.lo: prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libusnic_fi_la-util_domain.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libusnic_fi_la-util_domain.Tpo -c -o prov/util/src/libusnic_fi_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libusnic_fi_la-util_domain.Tpo prov/util/src/$(DEPDIR)/libusnic_fi_la-util_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_domain.c' object='prov/util/src/libusnic_fi_la-util_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libusnic_fi_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+
+prov/util/src/libusnic_fi_la-util_ep.lo: prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libusnic_fi_la-util_ep.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libusnic_fi_la-util_ep.Tpo -c -o prov/util/src/libusnic_fi_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libusnic_fi_la-util_ep.Tpo prov/util/src/$(DEPDIR)/libusnic_fi_la-util_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_ep.c' object='prov/util/src/libusnic_fi_la-util_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libusnic_fi_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+
+prov/util/src/libusnic_fi_la-util_eq.lo: prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libusnic_fi_la-util_eq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libusnic_fi_la-util_eq.Tpo -c -o prov/util/src/libusnic_fi_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libusnic_fi_la-util_eq.Tpo prov/util/src/$(DEPDIR)/libusnic_fi_la-util_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_eq.c' object='prov/util/src/libusnic_fi_la-util_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libusnic_fi_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+
+prov/util/src/libusnic_fi_la-util_fabric.lo: prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libusnic_fi_la-util_fabric.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libusnic_fi_la-util_fabric.Tpo -c -o prov/util/src/libusnic_fi_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libusnic_fi_la-util_fabric.Tpo prov/util/src/$(DEPDIR)/libusnic_fi_la-util_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_fabric.c' object='prov/util/src/libusnic_fi_la-util_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libusnic_fi_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+
+prov/util/src/libusnic_fi_la-util_main.lo: prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libusnic_fi_la-util_main.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libusnic_fi_la-util_main.Tpo -c -o prov/util/src/libusnic_fi_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libusnic_fi_la-util_main.Tpo prov/util/src/$(DEPDIR)/libusnic_fi_la-util_main.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_main.c' object='prov/util/src/libusnic_fi_la-util_main.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libusnic_fi_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+
+prov/util/src/libusnic_fi_la-util_poll.lo: prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libusnic_fi_la-util_poll.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libusnic_fi_la-util_poll.Tpo -c -o prov/util/src/libusnic_fi_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libusnic_fi_la-util_poll.Tpo prov/util/src/$(DEPDIR)/libusnic_fi_la-util_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_poll.c' object='prov/util/src/libusnic_fi_la-util_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libusnic_fi_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+
+prov/util/src/libusnic_fi_la-util_wait.lo: prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libusnic_fi_la-util_wait.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libusnic_fi_la-util_wait.Tpo -c -o prov/util/src/libusnic_fi_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libusnic_fi_la-util_wait.Tpo prov/util/src/$(DEPDIR)/libusnic_fi_la-util_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_wait.c' object='prov/util/src/libusnic_fi_la-util_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libusnic_fi_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+
+prov/util/src/libusnic_fi_la-util_buf.lo: prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libusnic_fi_la-util_buf.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libusnic_fi_la-util_buf.Tpo -c -o prov/util/src/libusnic_fi_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libusnic_fi_la-util_buf.Tpo prov/util/src/$(DEPDIR)/libusnic_fi_la-util_buf.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_buf.c' object='prov/util/src/libusnic_fi_la-util_buf.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libusnic_fi_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+
+prov/util/src/libusnic_fi_la-util_mr.lo: prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libusnic_fi_la-util_mr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libusnic_fi_la-util_mr.Tpo -c -o prov/util/src/libusnic_fi_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libusnic_fi_la-util_mr.Tpo prov/util/src/$(DEPDIR)/libusnic_fi_la-util_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_mr.c' object='prov/util/src/libusnic_fi_la-util_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libusnic_fi_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+
+src/osx/libusnic_fi_la-osd.lo: src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/osx/libusnic_fi_la-osd.lo -MD -MP -MF src/osx/$(DEPDIR)/libusnic_fi_la-osd.Tpo -c -o src/osx/libusnic_fi_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/osx/$(DEPDIR)/libusnic_fi_la-osd.Tpo src/osx/$(DEPDIR)/libusnic_fi_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/osx/osd.c' object='src/osx/libusnic_fi_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/osx/libusnic_fi_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+
+src/unix/libusnic_fi_la-osd.lo: src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/unix/libusnic_fi_la-osd.lo -MD -MP -MF src/unix/$(DEPDIR)/libusnic_fi_la-osd.Tpo -c -o src/unix/libusnic_fi_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/unix/$(DEPDIR)/libusnic_fi_la-osd.Tpo src/unix/$(DEPDIR)/libusnic_fi_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/unix/osd.c' object='src/unix/libusnic_fi_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libusnic_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/unix/libusnic_fi_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+
+prov/verbs/src/libverbs_fi_la-fi_verbs.lo: prov/verbs/src/fi_verbs.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/libverbs_fi_la-fi_verbs.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/libverbs_fi_la-fi_verbs.Tpo -c -o prov/verbs/src/libverbs_fi_la-fi_verbs.lo `test -f 'prov/verbs/src/fi_verbs.c' || echo '$(srcdir)/'`prov/verbs/src/fi_verbs.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/libverbs_fi_la-fi_verbs.Tpo prov/verbs/src/$(DEPDIR)/libverbs_fi_la-fi_verbs.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/fi_verbs.c' object='prov/verbs/src/libverbs_fi_la-fi_verbs.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/libverbs_fi_la-fi_verbs.lo `test -f 'prov/verbs/src/fi_verbs.c' || echo '$(srcdir)/'`prov/verbs/src/fi_verbs.c
+
+prov/verbs/src/libverbs_fi_la-verbs_atomic.lo: prov/verbs/src/verbs_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/libverbs_fi_la-verbs_atomic.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_atomic.Tpo -c -o prov/verbs/src/libverbs_fi_la-verbs_atomic.lo `test -f 'prov/verbs/src/verbs_atomic.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_atomic.Tpo prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_atomic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_atomic.c' object='prov/verbs/src/libverbs_fi_la-verbs_atomic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/libverbs_fi_la-verbs_atomic.lo `test -f 'prov/verbs/src/verbs_atomic.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_atomic.c
+
+prov/verbs/src/libverbs_fi_la-verbs_av.lo: prov/verbs/src/verbs_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/libverbs_fi_la-verbs_av.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_av.Tpo -c -o prov/verbs/src/libverbs_fi_la-verbs_av.lo `test -f 'prov/verbs/src/verbs_av.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_av.Tpo prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_av.c' object='prov/verbs/src/libverbs_fi_la-verbs_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/libverbs_fi_la-verbs_av.lo `test -f 'prov/verbs/src/verbs_av.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_av.c
+
+prov/verbs/src/libverbs_fi_la-verbs_cm.lo: prov/verbs/src/verbs_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/libverbs_fi_la-verbs_cm.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_cm.Tpo -c -o prov/verbs/src/libverbs_fi_la-verbs_cm.lo `test -f 'prov/verbs/src/verbs_cm.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_cm.Tpo prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_cm.c' object='prov/verbs/src/libverbs_fi_la-verbs_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/libverbs_fi_la-verbs_cm.lo `test -f 'prov/verbs/src/verbs_cm.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_cm.c
+
+prov/verbs/src/libverbs_fi_la-verbs_cq.lo: prov/verbs/src/verbs_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/libverbs_fi_la-verbs_cq.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_cq.Tpo -c -o prov/verbs/src/libverbs_fi_la-verbs_cq.lo `test -f 'prov/verbs/src/verbs_cq.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_cq.Tpo prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_cq.c' object='prov/verbs/src/libverbs_fi_la-verbs_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/libverbs_fi_la-verbs_cq.lo `test -f 'prov/verbs/src/verbs_cq.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_cq.c
+
+prov/verbs/src/libverbs_fi_la-verbs_srq.lo: prov/verbs/src/verbs_srq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/libverbs_fi_la-verbs_srq.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_srq.Tpo -c -o prov/verbs/src/libverbs_fi_la-verbs_srq.lo `test -f 'prov/verbs/src/verbs_srq.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_srq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_srq.Tpo prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_srq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_srq.c' object='prov/verbs/src/libverbs_fi_la-verbs_srq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/libverbs_fi_la-verbs_srq.lo `test -f 'prov/verbs/src/verbs_srq.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_srq.c
+
+prov/verbs/src/libverbs_fi_la-verbs_domain.lo: prov/verbs/src/verbs_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/libverbs_fi_la-verbs_domain.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_domain.Tpo -c -o prov/verbs/src/libverbs_fi_la-verbs_domain.lo `test -f 'prov/verbs/src/verbs_domain.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_domain.Tpo prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_domain.c' object='prov/verbs/src/libverbs_fi_la-verbs_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/libverbs_fi_la-verbs_domain.lo `test -f 'prov/verbs/src/verbs_domain.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_domain.c
+
+prov/verbs/src/libverbs_fi_la-verbs_eq.lo: prov/verbs/src/verbs_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/libverbs_fi_la-verbs_eq.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_eq.Tpo -c -o prov/verbs/src/libverbs_fi_la-verbs_eq.lo `test -f 'prov/verbs/src/verbs_eq.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_eq.Tpo prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_eq.c' object='prov/verbs/src/libverbs_fi_la-verbs_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/libverbs_fi_la-verbs_eq.lo `test -f 'prov/verbs/src/verbs_eq.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_eq.c
+
+prov/verbs/src/libverbs_fi_la-verbs_info.lo: prov/verbs/src/verbs_info.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/libverbs_fi_la-verbs_info.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_info.Tpo -c -o prov/verbs/src/libverbs_fi_la-verbs_info.lo `test -f 'prov/verbs/src/verbs_info.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_info.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_info.Tpo prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_info.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_info.c' object='prov/verbs/src/libverbs_fi_la-verbs_info.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/libverbs_fi_la-verbs_info.lo `test -f 'prov/verbs/src/verbs_info.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_info.c
+
+prov/verbs/src/libverbs_fi_la-verbs_msg.lo: prov/verbs/src/verbs_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/libverbs_fi_la-verbs_msg.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_msg.Tpo -c -o prov/verbs/src/libverbs_fi_la-verbs_msg.lo `test -f 'prov/verbs/src/verbs_msg.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_msg.Tpo prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_msg.c' object='prov/verbs/src/libverbs_fi_la-verbs_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/libverbs_fi_la-verbs_msg.lo `test -f 'prov/verbs/src/verbs_msg.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_msg.c
+
+prov/verbs/src/libverbs_fi_la-verbs_msg_ep.lo: prov/verbs/src/verbs_msg_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/libverbs_fi_la-verbs_msg_ep.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_msg_ep.Tpo -c -o prov/verbs/src/libverbs_fi_la-verbs_msg_ep.lo `test -f 'prov/verbs/src/verbs_msg_ep.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_msg_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_msg_ep.Tpo prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_msg_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_msg_ep.c' object='prov/verbs/src/libverbs_fi_la-verbs_msg_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/libverbs_fi_la-verbs_msg_ep.lo `test -f 'prov/verbs/src/verbs_msg_ep.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_msg_ep.c
+
+prov/verbs/src/libverbs_fi_la-verbs_rma.lo: prov/verbs/src/verbs_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/libverbs_fi_la-verbs_rma.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_rma.Tpo -c -o prov/verbs/src/libverbs_fi_la-verbs_rma.lo `test -f 'prov/verbs/src/verbs_rma.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_rma.Tpo prov/verbs/src/$(DEPDIR)/libverbs_fi_la-verbs_rma.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_rma.c' object='prov/verbs/src/libverbs_fi_la-verbs_rma.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/libverbs_fi_la-verbs_rma.lo `test -f 'prov/verbs/src/verbs_rma.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_rma.c
+
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_av_ep_rdm.lo: prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_av_ep_rdm.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_av_ep_rdm.Tpo -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_av_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c' || echo '$(srcdir)/' [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_av_ep_rdm.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_av_ep_rdm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c' object='prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_av_ep_rdm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_av_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c
+
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_cq_ep_rdm.lo: prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_cq_ep_rdm.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_cq_ep_rdm.Tpo -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_cq_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c' || echo '$(srcdir)/' [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_cq_ep_rdm.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_cq_ep_rdm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c' object='prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_cq_ep_rdm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_cq_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c
+
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_ep_rdm.lo: prov/verbs/src/ep_rdm/verbs_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_ep_rdm.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_ep_rdm.Tpo -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_ep_rdm.c' || echo '$(srcdir)/'`prov/verbs/ [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_ep_rdm.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_ep_rdm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_ep_rdm.c' object='prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_ep_rdm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_ep_rdm.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_ep_rdm.c
+
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cm.lo: prov/verbs/src/ep_rdm/verbs_rdm_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cm.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_rdm_cm.Tpo -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_rdm_cm.c' || echo '$(srcdir)/'`prov/verbs/ [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_rdm_cm.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_rdm_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_rdm_cm.c' object='prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_rdm_cm.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_rdm_cm.c
+
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cntr.lo: prov/verbs/src/ep_rdm/verbs_rdm_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cntr.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_rdm_cntr.Tpo -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cntr.lo `test -f 'prov/verbs/src/ep_rdm/verbs_rdm_cntr.c' || echo '$(srcdir)/'`pro [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_rdm_cntr.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_rdm_cntr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_rdm_cntr.c' object='prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cntr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_cntr.lo `test -f 'prov/verbs/src/ep_rdm/verbs_rdm_cntr.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_rdm_cntr.c
+
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_msg.lo: prov/verbs/src/ep_rdm/verbs_rdm_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_msg.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_rdm_msg.Tpo -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_msg.lo `test -f 'prov/verbs/src/ep_rdm/verbs_rdm_msg.c' || echo '$(srcdir)/'`prov/ve [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_rdm_msg.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_rdm_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_rdm_msg.c' object='prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_rdm_msg.lo `test -f 'prov/verbs/src/ep_rdm/verbs_rdm_msg.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_rdm_msg.c
+
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm_states.lo: prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm_states.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_tagged_ep_rdm_states.Tpo -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm_states.lo `test -f 'prov/verbs/src/ep_rdm/verbs_ [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_tagged_ep_rdm_states.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_tagged_ep_rdm_states.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c' object='prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm_states.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm_states.lo `test -f 'prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c
+
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm.lo: prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_tagged_ep_rdm.Tpo -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c' || e [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_tagged_ep_rdm.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_tagged_ep_rdm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c' object='prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_tagged_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c
+
+prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_utils.lo: prov/verbs/src/ep_rdm/verbs_utils.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_utils.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_utils.Tpo -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_utils.lo `test -f 'prov/verbs/src/ep_rdm/verbs_utils.c' || echo '$(srcdir)/'`prov/verbs/src/ [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_utils.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/libverbs_fi_la-verbs_utils.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_utils.c' object='prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_utils.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/libverbs_fi_la-verbs_utils.lo `test -f 'prov/verbs/src/ep_rdm/verbs_utils.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_utils.c
+
+src/libverbs_fi_la-common.lo: src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libverbs_fi_la-common.lo -MD -MP -MF src/$(DEPDIR)/libverbs_fi_la-common.Tpo -c -o src/libverbs_fi_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libverbs_fi_la-common.Tpo src/$(DEPDIR)/libverbs_fi_la-common.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/common.c' object='src/libverbs_fi_la-common.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libverbs_fi_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+
+src/libverbs_fi_la-enosys.lo: src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libverbs_fi_la-enosys.lo -MD -MP -MF src/$(DEPDIR)/libverbs_fi_la-enosys.Tpo -c -o src/libverbs_fi_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libverbs_fi_la-enosys.Tpo src/$(DEPDIR)/libverbs_fi_la-enosys.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/enosys.c' object='src/libverbs_fi_la-enosys.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libverbs_fi_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+
+src/libverbs_fi_la-rbtree.lo: src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libverbs_fi_la-rbtree.lo -MD -MP -MF src/$(DEPDIR)/libverbs_fi_la-rbtree.Tpo -c -o src/libverbs_fi_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libverbs_fi_la-rbtree.Tpo src/$(DEPDIR)/libverbs_fi_la-rbtree.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/rbtree.c' object='src/libverbs_fi_la-rbtree.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libverbs_fi_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+
+src/libverbs_fi_la-fasthash.lo: src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libverbs_fi_la-fasthash.lo -MD -MP -MF src/$(DEPDIR)/libverbs_fi_la-fasthash.Tpo -c -o src/libverbs_fi_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libverbs_fi_la-fasthash.Tpo src/$(DEPDIR)/libverbs_fi_la-fasthash.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/fasthash.c' object='src/libverbs_fi_la-fasthash.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libverbs_fi_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+
+src/libverbs_fi_la-indexer.lo: src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/libverbs_fi_la-indexer.lo -MD -MP -MF src/$(DEPDIR)/libverbs_fi_la-indexer.Tpo -c -o src/libverbs_fi_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/libverbs_fi_la-indexer.Tpo src/$(DEPDIR)/libverbs_fi_la-indexer.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/indexer.c' object='src/libverbs_fi_la-indexer.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/libverbs_fi_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+
+prov/util/src/libverbs_fi_la-util_attr.lo: prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libverbs_fi_la-util_attr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libverbs_fi_la-util_attr.Tpo -c -o prov/util/src/libverbs_fi_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libverbs_fi_la-util_attr.Tpo prov/util/src/$(DEPDIR)/libverbs_fi_la-util_attr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_attr.c' object='prov/util/src/libverbs_fi_la-util_attr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libverbs_fi_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+
+prov/util/src/libverbs_fi_la-util_av.lo: prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libverbs_fi_la-util_av.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libverbs_fi_la-util_av.Tpo -c -o prov/util/src/libverbs_fi_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libverbs_fi_la-util_av.Tpo prov/util/src/$(DEPDIR)/libverbs_fi_la-util_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_av.c' object='prov/util/src/libverbs_fi_la-util_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libverbs_fi_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+
+prov/util/src/libverbs_fi_la-util_cq.lo: prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libverbs_fi_la-util_cq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libverbs_fi_la-util_cq.Tpo -c -o prov/util/src/libverbs_fi_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libverbs_fi_la-util_cq.Tpo prov/util/src/$(DEPDIR)/libverbs_fi_la-util_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_cq.c' object='prov/util/src/libverbs_fi_la-util_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libverbs_fi_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+
+prov/util/src/libverbs_fi_la-util_domain.lo: prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libverbs_fi_la-util_domain.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libverbs_fi_la-util_domain.Tpo -c -o prov/util/src/libverbs_fi_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libverbs_fi_la-util_domain.Tpo prov/util/src/$(DEPDIR)/libverbs_fi_la-util_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_domain.c' object='prov/util/src/libverbs_fi_la-util_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libverbs_fi_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+
+prov/util/src/libverbs_fi_la-util_ep.lo: prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libverbs_fi_la-util_ep.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libverbs_fi_la-util_ep.Tpo -c -o prov/util/src/libverbs_fi_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libverbs_fi_la-util_ep.Tpo prov/util/src/$(DEPDIR)/libverbs_fi_la-util_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_ep.c' object='prov/util/src/libverbs_fi_la-util_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libverbs_fi_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+
+prov/util/src/libverbs_fi_la-util_eq.lo: prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libverbs_fi_la-util_eq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libverbs_fi_la-util_eq.Tpo -c -o prov/util/src/libverbs_fi_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libverbs_fi_la-util_eq.Tpo prov/util/src/$(DEPDIR)/libverbs_fi_la-util_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_eq.c' object='prov/util/src/libverbs_fi_la-util_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libverbs_fi_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+
+prov/util/src/libverbs_fi_la-util_fabric.lo: prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libverbs_fi_la-util_fabric.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libverbs_fi_la-util_fabric.Tpo -c -o prov/util/src/libverbs_fi_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libverbs_fi_la-util_fabric.Tpo prov/util/src/$(DEPDIR)/libverbs_fi_la-util_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_fabric.c' object='prov/util/src/libverbs_fi_la-util_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libverbs_fi_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+
+prov/util/src/libverbs_fi_la-util_main.lo: prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libverbs_fi_la-util_main.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libverbs_fi_la-util_main.Tpo -c -o prov/util/src/libverbs_fi_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libverbs_fi_la-util_main.Tpo prov/util/src/$(DEPDIR)/libverbs_fi_la-util_main.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_main.c' object='prov/util/src/libverbs_fi_la-util_main.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libverbs_fi_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+
+prov/util/src/libverbs_fi_la-util_poll.lo: prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libverbs_fi_la-util_poll.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libverbs_fi_la-util_poll.Tpo -c -o prov/util/src/libverbs_fi_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libverbs_fi_la-util_poll.Tpo prov/util/src/$(DEPDIR)/libverbs_fi_la-util_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_poll.c' object='prov/util/src/libverbs_fi_la-util_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libverbs_fi_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+
+prov/util/src/libverbs_fi_la-util_wait.lo: prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libverbs_fi_la-util_wait.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libverbs_fi_la-util_wait.Tpo -c -o prov/util/src/libverbs_fi_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libverbs_fi_la-util_wait.Tpo prov/util/src/$(DEPDIR)/libverbs_fi_la-util_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_wait.c' object='prov/util/src/libverbs_fi_la-util_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libverbs_fi_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+
+prov/util/src/libverbs_fi_la-util_buf.lo: prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libverbs_fi_la-util_buf.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libverbs_fi_la-util_buf.Tpo -c -o prov/util/src/libverbs_fi_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libverbs_fi_la-util_buf.Tpo prov/util/src/$(DEPDIR)/libverbs_fi_la-util_buf.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_buf.c' object='prov/util/src/libverbs_fi_la-util_buf.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libverbs_fi_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+
+prov/util/src/libverbs_fi_la-util_mr.lo: prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/libverbs_fi_la-util_mr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/libverbs_fi_la-util_mr.Tpo -c -o prov/util/src/libverbs_fi_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/libverbs_fi_la-util_mr.Tpo prov/util/src/$(DEPDIR)/libverbs_fi_la-util_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_mr.c' object='prov/util/src/libverbs_fi_la-util_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/libverbs_fi_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+
+src/osx/libverbs_fi_la-osd.lo: src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/osx/libverbs_fi_la-osd.lo -MD -MP -MF src/osx/$(DEPDIR)/libverbs_fi_la-osd.Tpo -c -o src/osx/libverbs_fi_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/osx/$(DEPDIR)/libverbs_fi_la-osd.Tpo src/osx/$(DEPDIR)/libverbs_fi_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/osx/osd.c' object='src/osx/libverbs_fi_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/osx/libverbs_fi_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+
+src/unix/libverbs_fi_la-osd.lo: src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/unix/libverbs_fi_la-osd.lo -MD -MP -MF src/unix/$(DEPDIR)/libverbs_fi_la-osd.Tpo -c -o src/unix/libverbs_fi_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/unix/$(DEPDIR)/libverbs_fi_la-osd.Tpo src/unix/$(DEPDIR)/libverbs_fi_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/unix/osd.c' object='src/unix/libverbs_fi_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libverbs_fi_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/unix/libverbs_fi_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+
+src/src_libfabric_la-fabric.lo: src/fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/src_libfabric_la-fabric.lo -MD -MP -MF src/$(DEPDIR)/src_libfabric_la-fabric.Tpo -c -o src/src_libfabric_la-fabric.lo `test -f 'src/fabric.c' || echo '$(srcdir)/'`src/fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/src_libfabric_la-fabric.Tpo src/$(DEPDIR)/src_libfabric_la-fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/fabric.c' object='src/src_libfabric_la-fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/src_libfabric_la-fabric.lo `test -f 'src/fabric.c' || echo '$(srcdir)/'`src/fabric.c
+
+src/src_libfabric_la-fi_tostr.lo: src/fi_tostr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/src_libfabric_la-fi_tostr.lo -MD -MP -MF src/$(DEPDIR)/src_libfabric_la-fi_tostr.Tpo -c -o src/src_libfabric_la-fi_tostr.lo `test -f 'src/fi_tostr.c' || echo '$(srcdir)/'`src/fi_tostr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/src_libfabric_la-fi_tostr.Tpo src/$(DEPDIR)/src_libfabric_la-fi_tostr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/fi_tostr.c' object='src/src_libfabric_la-fi_tostr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/src_libfabric_la-fi_tostr.lo `test -f 'src/fi_tostr.c' || echo '$(srcdir)/'`src/fi_tostr.c
+
+src/src_libfabric_la-log.lo: src/log.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/src_libfabric_la-log.lo -MD -MP -MF src/$(DEPDIR)/src_libfabric_la-log.Tpo -c -o src/src_libfabric_la-log.lo `test -f 'src/log.c' || echo '$(srcdir)/'`src/log.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/src_libfabric_la-log.Tpo src/$(DEPDIR)/src_libfabric_la-log.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/log.c' object='src/src_libfabric_la-log.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/src_libfabric_la-log.lo `test -f 'src/log.c' || echo '$(srcdir)/'`src/log.c
+
+src/src_libfabric_la-var.lo: src/var.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/src_libfabric_la-var.lo -MD -MP -MF src/$(DEPDIR)/src_libfabric_la-var.Tpo -c -o src/src_libfabric_la-var.lo `test -f 'src/var.c' || echo '$(srcdir)/'`src/var.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/src_libfabric_la-var.Tpo src/$(DEPDIR)/src_libfabric_la-var.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/var.c' object='src/src_libfabric_la-var.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/src_libfabric_la-var.lo `test -f 'src/var.c' || echo '$(srcdir)/'`src/var.c
+
+src/src_libfabric_la-common.lo: src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/src_libfabric_la-common.lo -MD -MP -MF src/$(DEPDIR)/src_libfabric_la-common.Tpo -c -o src/src_libfabric_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/src_libfabric_la-common.Tpo src/$(DEPDIR)/src_libfabric_la-common.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/common.c' object='src/src_libfabric_la-common.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/src_libfabric_la-common.lo `test -f 'src/common.c' || echo '$(srcdir)/'`src/common.c
+
+src/src_libfabric_la-enosys.lo: src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/src_libfabric_la-enosys.lo -MD -MP -MF src/$(DEPDIR)/src_libfabric_la-enosys.Tpo -c -o src/src_libfabric_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/src_libfabric_la-enosys.Tpo src/$(DEPDIR)/src_libfabric_la-enosys.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/enosys.c' object='src/src_libfabric_la-enosys.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/src_libfabric_la-enosys.lo `test -f 'src/enosys.c' || echo '$(srcdir)/'`src/enosys.c
+
+src/src_libfabric_la-rbtree.lo: src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/src_libfabric_la-rbtree.lo -MD -MP -MF src/$(DEPDIR)/src_libfabric_la-rbtree.Tpo -c -o src/src_libfabric_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/src_libfabric_la-rbtree.Tpo src/$(DEPDIR)/src_libfabric_la-rbtree.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/rbtree.c' object='src/src_libfabric_la-rbtree.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/src_libfabric_la-rbtree.lo `test -f 'src/rbtree.c' || echo '$(srcdir)/'`src/rbtree.c
+
+src/src_libfabric_la-fasthash.lo: src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/src_libfabric_la-fasthash.lo -MD -MP -MF src/$(DEPDIR)/src_libfabric_la-fasthash.Tpo -c -o src/src_libfabric_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/src_libfabric_la-fasthash.Tpo src/$(DEPDIR)/src_libfabric_la-fasthash.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/fasthash.c' object='src/src_libfabric_la-fasthash.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/src_libfabric_la-fasthash.lo `test -f 'src/fasthash.c' || echo '$(srcdir)/'`src/fasthash.c
+
+src/src_libfabric_la-indexer.lo: src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/src_libfabric_la-indexer.lo -MD -MP -MF src/$(DEPDIR)/src_libfabric_la-indexer.Tpo -c -o src/src_libfabric_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/src_libfabric_la-indexer.Tpo src/$(DEPDIR)/src_libfabric_la-indexer.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/indexer.c' object='src/src_libfabric_la-indexer.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/src_libfabric_la-indexer.lo `test -f 'src/indexer.c' || echo '$(srcdir)/'`src/indexer.c
+
+prov/util/src/src_libfabric_la-util_attr.lo: prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/src_libfabric_la-util_attr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/src_libfabric_la-util_attr.Tpo -c -o prov/util/src/src_libfabric_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/src_libfabric_la-util_attr.Tpo prov/util/src/$(DEPDIR)/src_libfabric_la-util_attr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_attr.c' object='prov/util/src/src_libfabric_la-util_attr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/src_libfabric_la-util_attr.lo `test -f 'prov/util/src/util_attr.c' || echo '$(srcdir)/'`prov/util/src/util_attr.c
+
+prov/util/src/src_libfabric_la-util_av.lo: prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/src_libfabric_la-util_av.lo -MD -MP -MF prov/util/src/$(DEPDIR)/src_libfabric_la-util_av.Tpo -c -o prov/util/src/src_libfabric_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/src_libfabric_la-util_av.Tpo prov/util/src/$(DEPDIR)/src_libfabric_la-util_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_av.c' object='prov/util/src/src_libfabric_la-util_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/src_libfabric_la-util_av.lo `test -f 'prov/util/src/util_av.c' || echo '$(srcdir)/'`prov/util/src/util_av.c
+
+prov/util/src/src_libfabric_la-util_cq.lo: prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/src_libfabric_la-util_cq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/src_libfabric_la-util_cq.Tpo -c -o prov/util/src/src_libfabric_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/src_libfabric_la-util_cq.Tpo prov/util/src/$(DEPDIR)/src_libfabric_la-util_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_cq.c' object='prov/util/src/src_libfabric_la-util_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/src_libfabric_la-util_cq.lo `test -f 'prov/util/src/util_cq.c' || echo '$(srcdir)/'`prov/util/src/util_cq.c
+
+prov/util/src/src_libfabric_la-util_domain.lo: prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/src_libfabric_la-util_domain.lo -MD -MP -MF prov/util/src/$(DEPDIR)/src_libfabric_la-util_domain.Tpo -c -o prov/util/src/src_libfabric_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/src_libfabric_la-util_domain.Tpo prov/util/src/$(DEPDIR)/src_libfabric_la-util_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_domain.c' object='prov/util/src/src_libfabric_la-util_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/src_libfabric_la-util_domain.lo `test -f 'prov/util/src/util_domain.c' || echo '$(srcdir)/'`prov/util/src/util_domain.c
+
+prov/util/src/src_libfabric_la-util_ep.lo: prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/src_libfabric_la-util_ep.lo -MD -MP -MF prov/util/src/$(DEPDIR)/src_libfabric_la-util_ep.Tpo -c -o prov/util/src/src_libfabric_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/src_libfabric_la-util_ep.Tpo prov/util/src/$(DEPDIR)/src_libfabric_la-util_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_ep.c' object='prov/util/src/src_libfabric_la-util_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/src_libfabric_la-util_ep.lo `test -f 'prov/util/src/util_ep.c' || echo '$(srcdir)/'`prov/util/src/util_ep.c
+
+prov/util/src/src_libfabric_la-util_eq.lo: prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/src_libfabric_la-util_eq.lo -MD -MP -MF prov/util/src/$(DEPDIR)/src_libfabric_la-util_eq.Tpo -c -o prov/util/src/src_libfabric_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/src_libfabric_la-util_eq.Tpo prov/util/src/$(DEPDIR)/src_libfabric_la-util_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_eq.c' object='prov/util/src/src_libfabric_la-util_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/src_libfabric_la-util_eq.lo `test -f 'prov/util/src/util_eq.c' || echo '$(srcdir)/'`prov/util/src/util_eq.c
+
+prov/util/src/src_libfabric_la-util_fabric.lo: prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/src_libfabric_la-util_fabric.lo -MD -MP -MF prov/util/src/$(DEPDIR)/src_libfabric_la-util_fabric.Tpo -c -o prov/util/src/src_libfabric_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/src_libfabric_la-util_fabric.Tpo prov/util/src/$(DEPDIR)/src_libfabric_la-util_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_fabric.c' object='prov/util/src/src_libfabric_la-util_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/src_libfabric_la-util_fabric.lo `test -f 'prov/util/src/util_fabric.c' || echo '$(srcdir)/'`prov/util/src/util_fabric.c
+
+prov/util/src/src_libfabric_la-util_main.lo: prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/src_libfabric_la-util_main.lo -MD -MP -MF prov/util/src/$(DEPDIR)/src_libfabric_la-util_main.Tpo -c -o prov/util/src/src_libfabric_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/src_libfabric_la-util_main.Tpo prov/util/src/$(DEPDIR)/src_libfabric_la-util_main.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_main.c' object='prov/util/src/src_libfabric_la-util_main.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/src_libfabric_la-util_main.lo `test -f 'prov/util/src/util_main.c' || echo '$(srcdir)/'`prov/util/src/util_main.c
+
+prov/util/src/src_libfabric_la-util_poll.lo: prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/src_libfabric_la-util_poll.lo -MD -MP -MF prov/util/src/$(DEPDIR)/src_libfabric_la-util_poll.Tpo -c -o prov/util/src/src_libfabric_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/src_libfabric_la-util_poll.Tpo prov/util/src/$(DEPDIR)/src_libfabric_la-util_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_poll.c' object='prov/util/src/src_libfabric_la-util_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/src_libfabric_la-util_poll.lo `test -f 'prov/util/src/util_poll.c' || echo '$(srcdir)/'`prov/util/src/util_poll.c
+
+prov/util/src/src_libfabric_la-util_wait.lo: prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/src_libfabric_la-util_wait.lo -MD -MP -MF prov/util/src/$(DEPDIR)/src_libfabric_la-util_wait.Tpo -c -o prov/util/src/src_libfabric_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/src_libfabric_la-util_wait.Tpo prov/util/src/$(DEPDIR)/src_libfabric_la-util_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_wait.c' object='prov/util/src/src_libfabric_la-util_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/src_libfabric_la-util_wait.lo `test -f 'prov/util/src/util_wait.c' || echo '$(srcdir)/'`prov/util/src/util_wait.c
+
+prov/util/src/src_libfabric_la-util_buf.lo: prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/src_libfabric_la-util_buf.lo -MD -MP -MF prov/util/src/$(DEPDIR)/src_libfabric_la-util_buf.Tpo -c -o prov/util/src/src_libfabric_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/src_libfabric_la-util_buf.Tpo prov/util/src/$(DEPDIR)/src_libfabric_la-util_buf.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_buf.c' object='prov/util/src/src_libfabric_la-util_buf.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/src_libfabric_la-util_buf.lo `test -f 'prov/util/src/util_buf.c' || echo '$(srcdir)/'`prov/util/src/util_buf.c
+
+prov/util/src/src_libfabric_la-util_mr.lo: prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/util/src/src_libfabric_la-util_mr.lo -MD -MP -MF prov/util/src/$(DEPDIR)/src_libfabric_la-util_mr.Tpo -c -o prov/util/src/src_libfabric_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/util/src/$(DEPDIR)/src_libfabric_la-util_mr.Tpo prov/util/src/$(DEPDIR)/src_libfabric_la-util_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/util/src/util_mr.c' object='prov/util/src/src_libfabric_la-util_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/util/src/src_libfabric_la-util_mr.lo `test -f 'prov/util/src/util_mr.c' || echo '$(srcdir)/'`prov/util/src/util_mr.c
+
+src/osx/src_libfabric_la-osd.lo: src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/osx/src_libfabric_la-osd.lo -MD -MP -MF src/osx/$(DEPDIR)/src_libfabric_la-osd.Tpo -c -o src/osx/src_libfabric_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/osx/$(DEPDIR)/src_libfabric_la-osd.Tpo src/osx/$(DEPDIR)/src_libfabric_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/osx/osd.c' object='src/osx/src_libfabric_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/osx/src_libfabric_la-osd.lo `test -f 'src/osx/osd.c' || echo '$(srcdir)/'`src/osx/osd.c
+
+src/unix/src_libfabric_la-osd.lo: src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT src/unix/src_libfabric_la-osd.lo -MD -MP -MF src/unix/$(DEPDIR)/src_libfabric_la-osd.Tpo -c -o src/unix/src_libfabric_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/unix/$(DEPDIR)/src_libfabric_la-osd.Tpo src/unix/$(DEPDIR)/src_libfabric_la-osd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/unix/osd.c' object='src/unix/src_libfabric_la-osd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o src/unix/src_libfabric_la-osd.lo `test -f 'src/unix/osd.c' || echo '$(srcdir)/'`src/unix/osd.c
+
+prov/sockets/src/src_libfabric_la-sock_av.lo: prov/sockets/src/sock_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_av.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_av.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_av.lo `test -f 'prov/sockets/src/sock_av.c' || echo '$(srcdir)/'`prov/sockets/src/sock_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_av.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_av.c' object='prov/sockets/src/src_libfabric_la-sock_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_av.lo `test -f 'prov/sockets/src/sock_av.c' || echo '$(srcdir)/'`prov/sockets/src/sock_av.c
+
+prov/sockets/src/src_libfabric_la-sock_dom.lo: prov/sockets/src/sock_dom.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_dom.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_dom.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_dom.lo `test -f 'prov/sockets/src/sock_dom.c' || echo '$(srcdir)/'`prov/sockets/src/sock_dom.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_dom.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_dom.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_dom.c' object='prov/sockets/src/src_libfabric_la-sock_dom.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_dom.lo `test -f 'prov/sockets/src/sock_dom.c' || echo '$(srcdir)/'`prov/sockets/src/sock_dom.c
+
+prov/sockets/src/src_libfabric_la-sock_eq.lo: prov/sockets/src/sock_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_eq.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_eq.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_eq.lo `test -f 'prov/sockets/src/sock_eq.c' || echo '$(srcdir)/'`prov/sockets/src/sock_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_eq.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_eq.c' object='prov/sockets/src/src_libfabric_la-sock_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_eq.lo `test -f 'prov/sockets/src/sock_eq.c' || echo '$(srcdir)/'`prov/sockets/src/sock_eq.c
+
+prov/sockets/src/src_libfabric_la-sock_cq.lo: prov/sockets/src/sock_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_cq.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_cq.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_cq.lo `test -f 'prov/sockets/src/sock_cq.c' || echo '$(srcdir)/'`prov/sockets/src/sock_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_cq.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_cq.c' object='prov/sockets/src/src_libfabric_la-sock_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_cq.lo `test -f 'prov/sockets/src/sock_cq.c' || echo '$(srcdir)/'`prov/sockets/src/sock_cq.c
+
+prov/sockets/src/src_libfabric_la-sock_cntr.lo: prov/sockets/src/sock_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_cntr.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_cntr.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_cntr.lo `test -f 'prov/sockets/src/sock_cntr.c' || echo '$(srcdir)/'`prov/sockets/src/sock_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_cntr.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_cntr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_cntr.c' object='prov/sockets/src/src_libfabric_la-sock_cntr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_cntr.lo `test -f 'prov/sockets/src/sock_cntr.c' || echo '$(srcdir)/'`prov/sockets/src/sock_cntr.c
+
+prov/sockets/src/src_libfabric_la-sock_poll.lo: prov/sockets/src/sock_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_poll.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_poll.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_poll.lo `test -f 'prov/sockets/src/sock_poll.c' || echo '$(srcdir)/'`prov/sockets/src/sock_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_poll.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_poll.c' object='prov/sockets/src/src_libfabric_la-sock_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_poll.lo `test -f 'prov/sockets/src/sock_poll.c' || echo '$(srcdir)/'`prov/sockets/src/sock_poll.c
+
+prov/sockets/src/src_libfabric_la-sock_wait.lo: prov/sockets/src/sock_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_wait.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_wait.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_wait.lo `test -f 'prov/sockets/src/sock_wait.c' || echo '$(srcdir)/'`prov/sockets/src/sock_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_wait.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_wait.c' object='prov/sockets/src/src_libfabric_la-sock_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_wait.lo `test -f 'prov/sockets/src/sock_wait.c' || echo '$(srcdir)/'`prov/sockets/src/sock_wait.c
+
+prov/sockets/src/src_libfabric_la-sock_ep_rdm.lo: prov/sockets/src/sock_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_ep_rdm.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep_rdm.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_ep_rdm.lo `test -f 'prov/sockets/src/sock_ep_rdm.c' || echo '$(srcdir)/'`prov/sockets/src/sock_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep_rdm.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep_rdm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_ep_rdm.c' object='prov/sockets/src/src_libfabric_la-sock_ep_rdm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_ep_rdm.lo `test -f 'prov/sockets/src/sock_ep_rdm.c' || echo '$(srcdir)/'`prov/sockets/src/sock_ep_rdm.c
+
+prov/sockets/src/src_libfabric_la-sock_ep_dgram.lo: prov/sockets/src/sock_ep_dgram.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_ep_dgram.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep_dgram.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_ep_dgram.lo `test -f 'prov/sockets/src/sock_ep_dgram.c' || echo '$(srcdir)/'`prov/sockets/src/so [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep_dgram.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep_dgram.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_ep_dgram.c' object='prov/sockets/src/src_libfabric_la-sock_ep_dgram.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_ep_dgram.lo `test -f 'prov/sockets/src/sock_ep_dgram.c' || echo '$(srcdir)/'`prov/sockets/src/sock_ep_dgram.c
+
+prov/sockets/src/src_libfabric_la-sock_ep_msg.lo: prov/sockets/src/sock_ep_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_ep_msg.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep_msg.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_ep_msg.lo `test -f 'prov/sockets/src/sock_ep_msg.c' || echo '$(srcdir)/'`prov/sockets/src/sock_ep_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep_msg.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_ep_msg.c' object='prov/sockets/src/src_libfabric_la-sock_ep_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_ep_msg.lo `test -f 'prov/sockets/src/sock_ep_msg.c' || echo '$(srcdir)/'`prov/sockets/src/sock_ep_msg.c
+
+prov/sockets/src/src_libfabric_la-sock_fabric.lo: prov/sockets/src/sock_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_fabric.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_fabric.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_fabric.lo `test -f 'prov/sockets/src/sock_fabric.c' || echo '$(srcdir)/'`prov/sockets/src/sock_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_fabric.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_fabric.c' object='prov/sockets/src/src_libfabric_la-sock_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_fabric.lo `test -f 'prov/sockets/src/sock_fabric.c' || echo '$(srcdir)/'`prov/sockets/src/sock_fabric.c
+
+prov/sockets/src/src_libfabric_la-sock_ep.lo: prov/sockets/src/sock_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_ep.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_ep.lo `test -f 'prov/sockets/src/sock_ep.c' || echo '$(srcdir)/'`prov/sockets/src/sock_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_ep.c' object='prov/sockets/src/src_libfabric_la-sock_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_ep.lo `test -f 'prov/sockets/src/sock_ep.c' || echo '$(srcdir)/'`prov/sockets/src/sock_ep.c
+
+prov/sockets/src/src_libfabric_la-sock_ctx.lo: prov/sockets/src/sock_ctx.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_ctx.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ctx.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_ctx.lo `test -f 'prov/sockets/src/sock_ctx.c' || echo '$(srcdir)/'`prov/sockets/src/sock_ctx.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ctx.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_ctx.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_ctx.c' object='prov/sockets/src/src_libfabric_la-sock_ctx.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_ctx.lo `test -f 'prov/sockets/src/sock_ctx.c' || echo '$(srcdir)/'`prov/sockets/src/sock_ctx.c
+
+prov/sockets/src/src_libfabric_la-sock_rx_entry.lo: prov/sockets/src/sock_rx_entry.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_rx_entry.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_rx_entry.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_rx_entry.lo `test -f 'prov/sockets/src/sock_rx_entry.c' || echo '$(srcdir)/'`prov/sockets/src/so [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_rx_entry.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_rx_entry.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_rx_entry.c' object='prov/sockets/src/src_libfabric_la-sock_rx_entry.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_rx_entry.lo `test -f 'prov/sockets/src/sock_rx_entry.c' || echo '$(srcdir)/'`prov/sockets/src/sock_rx_entry.c
+
+prov/sockets/src/src_libfabric_la-sock_progress.lo: prov/sockets/src/sock_progress.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_progress.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_progress.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_progress.lo `test -f 'prov/sockets/src/sock_progress.c' || echo '$(srcdir)/'`prov/sockets/src/so [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_progress.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_progress.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_progress.c' object='prov/sockets/src/src_libfabric_la-sock_progress.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_progress.lo `test -f 'prov/sockets/src/sock_progress.c' || echo '$(srcdir)/'`prov/sockets/src/sock_progress.c
+
+prov/sockets/src/src_libfabric_la-sock_comm.lo: prov/sockets/src/sock_comm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_comm.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_comm.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_comm.lo `test -f 'prov/sockets/src/sock_comm.c' || echo '$(srcdir)/'`prov/sockets/src/sock_comm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_comm.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_comm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_comm.c' object='prov/sockets/src/src_libfabric_la-sock_comm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_comm.lo `test -f 'prov/sockets/src/sock_comm.c' || echo '$(srcdir)/'`prov/sockets/src/sock_comm.c
+
+prov/sockets/src/src_libfabric_la-sock_conn.lo: prov/sockets/src/sock_conn.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_conn.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_conn.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_conn.lo `test -f 'prov/sockets/src/sock_conn.c' || echo '$(srcdir)/'`prov/sockets/src/sock_conn.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_conn.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_conn.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_conn.c' object='prov/sockets/src/src_libfabric_la-sock_conn.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_conn.lo `test -f 'prov/sockets/src/sock_conn.c' || echo '$(srcdir)/'`prov/sockets/src/sock_conn.c
+
+prov/sockets/src/src_libfabric_la-sock_msg.lo: prov/sockets/src/sock_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_msg.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_msg.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_msg.lo `test -f 'prov/sockets/src/sock_msg.c' || echo '$(srcdir)/'`prov/sockets/src/sock_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_msg.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_msg.c' object='prov/sockets/src/src_libfabric_la-sock_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_msg.lo `test -f 'prov/sockets/src/sock_msg.c' || echo '$(srcdir)/'`prov/sockets/src/sock_msg.c
+
+prov/sockets/src/src_libfabric_la-sock_rma.lo: prov/sockets/src/sock_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_rma.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_rma.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_rma.lo `test -f 'prov/sockets/src/sock_rma.c' || echo '$(srcdir)/'`prov/sockets/src/sock_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_rma.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_rma.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_rma.c' object='prov/sockets/src/src_libfabric_la-sock_rma.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_rma.lo `test -f 'prov/sockets/src/sock_rma.c' || echo '$(srcdir)/'`prov/sockets/src/sock_rma.c
+
+prov/sockets/src/src_libfabric_la-sock_atomic.lo: prov/sockets/src/sock_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_atomic.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_atomic.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_atomic.lo `test -f 'prov/sockets/src/sock_atomic.c' || echo '$(srcdir)/'`prov/sockets/src/sock_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_atomic.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_atomic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_atomic.c' object='prov/sockets/src/src_libfabric_la-sock_atomic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_atomic.lo `test -f 'prov/sockets/src/sock_atomic.c' || echo '$(srcdir)/'`prov/sockets/src/sock_atomic.c
+
+prov/sockets/src/src_libfabric_la-sock_trigger.lo: prov/sockets/src/sock_trigger.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_trigger.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_trigger.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_trigger.lo `test -f 'prov/sockets/src/sock_trigger.c' || echo '$(srcdir)/'`prov/sockets/src/sock_t [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_trigger.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_trigger.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_trigger.c' object='prov/sockets/src/src_libfabric_la-sock_trigger.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_trigger.lo `test -f 'prov/sockets/src/sock_trigger.c' || echo '$(srcdir)/'`prov/sockets/src/sock_trigger.c
+
+prov/sockets/src/src_libfabric_la-sock_epoll.lo: prov/sockets/src/sock_epoll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/sockets/src/src_libfabric_la-sock_epoll.lo -MD -MP -MF prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_epoll.Tpo -c -o prov/sockets/src/src_libfabric_la-sock_epoll.lo `test -f 'prov/sockets/src/sock_epoll.c' || echo '$(srcdir)/'`prov/sockets/src/sock_epoll.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_epoll.Tpo prov/sockets/src/$(DEPDIR)/src_libfabric_la-sock_epoll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/sockets/src/sock_epoll.c' object='prov/sockets/src/src_libfabric_la-sock_epoll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/sockets/src/src_libfabric_la-sock_epoll.lo `test -f 'prov/sockets/src/sock_epoll.c' || echo '$(srcdir)/'`prov/sockets/src/sock_epoll.c
+
+prov/udp/src/src_libfabric_la-udpx_attr.lo: prov/udp/src/udpx_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/udp/src/src_libfabric_la-udpx_attr.lo -MD -MP -MF prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_attr.Tpo -c -o prov/udp/src/src_libfabric_la-udpx_attr.lo `test -f 'prov/udp/src/udpx_attr.c' || echo '$(srcdir)/'`prov/udp/src/udpx_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_attr.Tpo prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_attr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/udp/src/udpx_attr.c' object='prov/udp/src/src_libfabric_la-udpx_attr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/udp/src/src_libfabric_la-udpx_attr.lo `test -f 'prov/udp/src/udpx_attr.c' || echo '$(srcdir)/'`prov/udp/src/udpx_attr.c
+
+prov/udp/src/src_libfabric_la-udpx_cq.lo: prov/udp/src/udpx_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/udp/src/src_libfabric_la-udpx_cq.lo -MD -MP -MF prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_cq.Tpo -c -o prov/udp/src/src_libfabric_la-udpx_cq.lo `test -f 'prov/udp/src/udpx_cq.c' || echo '$(srcdir)/'`prov/udp/src/udpx_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_cq.Tpo prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/udp/src/udpx_cq.c' object='prov/udp/src/src_libfabric_la-udpx_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/udp/src/src_libfabric_la-udpx_cq.lo `test -f 'prov/udp/src/udpx_cq.c' || echo '$(srcdir)/'`prov/udp/src/udpx_cq.c
+
+prov/udp/src/src_libfabric_la-udpx_domain.lo: prov/udp/src/udpx_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/udp/src/src_libfabric_la-udpx_domain.lo -MD -MP -MF prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_domain.Tpo -c -o prov/udp/src/src_libfabric_la-udpx_domain.lo `test -f 'prov/udp/src/udpx_domain.c' || echo '$(srcdir)/'`prov/udp/src/udpx_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_domain.Tpo prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/udp/src/udpx_domain.c' object='prov/udp/src/src_libfabric_la-udpx_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/udp/src/src_libfabric_la-udpx_domain.lo `test -f 'prov/udp/src/udpx_domain.c' || echo '$(srcdir)/'`prov/udp/src/udpx_domain.c
+
+prov/udp/src/src_libfabric_la-udpx_ep.lo: prov/udp/src/udpx_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/udp/src/src_libfabric_la-udpx_ep.lo -MD -MP -MF prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_ep.Tpo -c -o prov/udp/src/src_libfabric_la-udpx_ep.lo `test -f 'prov/udp/src/udpx_ep.c' || echo '$(srcdir)/'`prov/udp/src/udpx_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_ep.Tpo prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/udp/src/udpx_ep.c' object='prov/udp/src/src_libfabric_la-udpx_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/udp/src/src_libfabric_la-udpx_ep.lo `test -f 'prov/udp/src/udpx_ep.c' || echo '$(srcdir)/'`prov/udp/src/udpx_ep.c
+
+prov/udp/src/src_libfabric_la-udpx_fabric.lo: prov/udp/src/udpx_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/udp/src/src_libfabric_la-udpx_fabric.lo -MD -MP -MF prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_fabric.Tpo -c -o prov/udp/src/src_libfabric_la-udpx_fabric.lo `test -f 'prov/udp/src/udpx_fabric.c' || echo '$(srcdir)/'`prov/udp/src/udpx_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_fabric.Tpo prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/udp/src/udpx_fabric.c' object='prov/udp/src/src_libfabric_la-udpx_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/udp/src/src_libfabric_la-udpx_fabric.lo `test -f 'prov/udp/src/udpx_fabric.c' || echo '$(srcdir)/'`prov/udp/src/udpx_fabric.c
+
+prov/udp/src/src_libfabric_la-udpx_init.lo: prov/udp/src/udpx_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/udp/src/src_libfabric_la-udpx_init.lo -MD -MP -MF prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_init.Tpo -c -o prov/udp/src/src_libfabric_la-udpx_init.lo `test -f 'prov/udp/src/udpx_init.c' || echo '$(srcdir)/'`prov/udp/src/udpx_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_init.Tpo prov/udp/src/$(DEPDIR)/src_libfabric_la-udpx_init.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/udp/src/udpx_init.c' object='prov/udp/src/src_libfabric_la-udpx_init.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/udp/src/src_libfabric_la-udpx_init.lo `test -f 'prov/udp/src/udpx_init.c' || echo '$(srcdir)/'`prov/udp/src/udpx_init.c
+
+prov/verbs/src/src_libfabric_la-fi_verbs.lo: prov/verbs/src/fi_verbs.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/src_libfabric_la-fi_verbs.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/src_libfabric_la-fi_verbs.Tpo -c -o prov/verbs/src/src_libfabric_la-fi_verbs.lo `test -f 'prov/verbs/src/fi_verbs.c' || echo '$(srcdir)/'`prov/verbs/src/fi_verbs.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/src_libfabric_la-fi_verbs.Tpo prov/verbs/src/$(DEPDIR)/src_libfabric_la-fi_verbs.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/fi_verbs.c' object='prov/verbs/src/src_libfabric_la-fi_verbs.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/src_libfabric_la-fi_verbs.lo `test -f 'prov/verbs/src/fi_verbs.c' || echo '$(srcdir)/'`prov/verbs/src/fi_verbs.c
+
+prov/verbs/src/src_libfabric_la-verbs_atomic.lo: prov/verbs/src/verbs_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/src_libfabric_la-verbs_atomic.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_atomic.Tpo -c -o prov/verbs/src/src_libfabric_la-verbs_atomic.lo `test -f 'prov/verbs/src/verbs_atomic.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_atomic.Tpo prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_atomic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_atomic.c' object='prov/verbs/src/src_libfabric_la-verbs_atomic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/src_libfabric_la-verbs_atomic.lo `test -f 'prov/verbs/src/verbs_atomic.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_atomic.c
+
+prov/verbs/src/src_libfabric_la-verbs_av.lo: prov/verbs/src/verbs_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/src_libfabric_la-verbs_av.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_av.Tpo -c -o prov/verbs/src/src_libfabric_la-verbs_av.lo `test -f 'prov/verbs/src/verbs_av.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_av.Tpo prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_av.c' object='prov/verbs/src/src_libfabric_la-verbs_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/src_libfabric_la-verbs_av.lo `test -f 'prov/verbs/src/verbs_av.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_av.c
+
+prov/verbs/src/src_libfabric_la-verbs_cm.lo: prov/verbs/src/verbs_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/src_libfabric_la-verbs_cm.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_cm.Tpo -c -o prov/verbs/src/src_libfabric_la-verbs_cm.lo `test -f 'prov/verbs/src/verbs_cm.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_cm.Tpo prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_cm.c' object='prov/verbs/src/src_libfabric_la-verbs_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/src_libfabric_la-verbs_cm.lo `test -f 'prov/verbs/src/verbs_cm.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_cm.c
+
+prov/verbs/src/src_libfabric_la-verbs_cq.lo: prov/verbs/src/verbs_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/src_libfabric_la-verbs_cq.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_cq.Tpo -c -o prov/verbs/src/src_libfabric_la-verbs_cq.lo `test -f 'prov/verbs/src/verbs_cq.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_cq.Tpo prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_cq.c' object='prov/verbs/src/src_libfabric_la-verbs_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/src_libfabric_la-verbs_cq.lo `test -f 'prov/verbs/src/verbs_cq.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_cq.c
+
+prov/verbs/src/src_libfabric_la-verbs_srq.lo: prov/verbs/src/verbs_srq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/src_libfabric_la-verbs_srq.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_srq.Tpo -c -o prov/verbs/src/src_libfabric_la-verbs_srq.lo `test -f 'prov/verbs/src/verbs_srq.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_srq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_srq.Tpo prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_srq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_srq.c' object='prov/verbs/src/src_libfabric_la-verbs_srq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/src_libfabric_la-verbs_srq.lo `test -f 'prov/verbs/src/verbs_srq.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_srq.c
+
+prov/verbs/src/src_libfabric_la-verbs_domain.lo: prov/verbs/src/verbs_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/src_libfabric_la-verbs_domain.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_domain.Tpo -c -o prov/verbs/src/src_libfabric_la-verbs_domain.lo `test -f 'prov/verbs/src/verbs_domain.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_domain.Tpo prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_domain.c' object='prov/verbs/src/src_libfabric_la-verbs_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/src_libfabric_la-verbs_domain.lo `test -f 'prov/verbs/src/verbs_domain.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_domain.c
+
+prov/verbs/src/src_libfabric_la-verbs_eq.lo: prov/verbs/src/verbs_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/src_libfabric_la-verbs_eq.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_eq.Tpo -c -o prov/verbs/src/src_libfabric_la-verbs_eq.lo `test -f 'prov/verbs/src/verbs_eq.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_eq.Tpo prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_eq.c' object='prov/verbs/src/src_libfabric_la-verbs_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/src_libfabric_la-verbs_eq.lo `test -f 'prov/verbs/src/verbs_eq.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_eq.c
+
+prov/verbs/src/src_libfabric_la-verbs_info.lo: prov/verbs/src/verbs_info.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/src_libfabric_la-verbs_info.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_info.Tpo -c -o prov/verbs/src/src_libfabric_la-verbs_info.lo `test -f 'prov/verbs/src/verbs_info.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_info.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_info.Tpo prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_info.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_info.c' object='prov/verbs/src/src_libfabric_la-verbs_info.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/src_libfabric_la-verbs_info.lo `test -f 'prov/verbs/src/verbs_info.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_info.c
+
+prov/verbs/src/src_libfabric_la-verbs_msg.lo: prov/verbs/src/verbs_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/src_libfabric_la-verbs_msg.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_msg.Tpo -c -o prov/verbs/src/src_libfabric_la-verbs_msg.lo `test -f 'prov/verbs/src/verbs_msg.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_msg.Tpo prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_msg.c' object='prov/verbs/src/src_libfabric_la-verbs_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/src_libfabric_la-verbs_msg.lo `test -f 'prov/verbs/src/verbs_msg.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_msg.c
+
+prov/verbs/src/src_libfabric_la-verbs_msg_ep.lo: prov/verbs/src/verbs_msg_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/src_libfabric_la-verbs_msg_ep.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_msg_ep.Tpo -c -o prov/verbs/src/src_libfabric_la-verbs_msg_ep.lo `test -f 'prov/verbs/src/verbs_msg_ep.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_msg_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_msg_ep.Tpo prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_msg_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_msg_ep.c' object='prov/verbs/src/src_libfabric_la-verbs_msg_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/src_libfabric_la-verbs_msg_ep.lo `test -f 'prov/verbs/src/verbs_msg_ep.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_msg_ep.c
+
+prov/verbs/src/src_libfabric_la-verbs_rma.lo: prov/verbs/src/verbs_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/src_libfabric_la-verbs_rma.lo -MD -MP -MF prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_rma.Tpo -c -o prov/verbs/src/src_libfabric_la-verbs_rma.lo `test -f 'prov/verbs/src/verbs_rma.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_rma.Tpo prov/verbs/src/$(DEPDIR)/src_libfabric_la-verbs_rma.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/verbs_rma.c' object='prov/verbs/src/src_libfabric_la-verbs_rma.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/src_libfabric_la-verbs_rma.lo `test -f 'prov/verbs/src/verbs_rma.c' || echo '$(srcdir)/'`prov/verbs/src/verbs_rma.c
+
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_av_ep_rdm.lo: prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/src_libfabric_la-verbs_av_ep_rdm.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_av_ep_rdm.Tpo -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_av_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c' || echo '$(s [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_av_ep_rdm.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_av_ep_rdm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c' object='prov/verbs/src/ep_rdm/src_libfabric_la-verbs_av_ep_rdm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_av_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c
+
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_cq_ep_rdm.lo: prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/src_libfabric_la-verbs_cq_ep_rdm.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_cq_ep_rdm.Tpo -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_cq_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c' || echo '$(s [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_cq_ep_rdm.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_cq_ep_rdm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c' object='prov/verbs/src/ep_rdm/src_libfabric_la-verbs_cq_ep_rdm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_cq_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c
+
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_ep_rdm.lo: prov/verbs/src/ep_rdm/verbs_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/src_libfabric_la-verbs_ep_rdm.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_ep_rdm.Tpo -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_ep_rdm.c' || echo '$(srcdir)/'`pro [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_ep_rdm.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_ep_rdm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_ep_rdm.c' object='prov/verbs/src/ep_rdm/src_libfabric_la-verbs_ep_rdm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_ep_rdm.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_ep_rdm.c
+
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cm.lo: prov/verbs/src/ep_rdm/verbs_rdm_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cm.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_rdm_cm.Tpo -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_rdm_cm.c' || echo '$(srcdir)/'`pro [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_rdm_cm.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_rdm_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_rdm_cm.c' object='prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_rdm_cm.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_rdm_cm.c
+
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cntr.lo: prov/verbs/src/ep_rdm/verbs_rdm_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cntr.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_rdm_cntr.Tpo -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cntr.lo `test -f 'prov/verbs/src/ep_rdm/verbs_rdm_cntr.c' || echo '$(srcdi [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_rdm_cntr.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_rdm_cntr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_rdm_cntr.c' object='prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cntr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_cntr.lo `test -f 'prov/verbs/src/ep_rdm/verbs_rdm_cntr.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_rdm_cntr.c
+
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_msg.lo: prov/verbs/src/ep_rdm/verbs_rdm_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_msg.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_rdm_msg.Tpo -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_msg.lo `test -f 'prov/verbs/src/ep_rdm/verbs_rdm_msg.c' || echo '$(srcdir)/' [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_rdm_msg.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_rdm_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_rdm_msg.c' object='prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_rdm_msg.lo `test -f 'prov/verbs/src/ep_rdm/verbs_rdm_msg.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_rdm_msg.c
+
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm_states.lo: prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm_states.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_tagged_ep_rdm_states.Tpo -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm_states.lo `test -f 'prov/verbs/src/ep_rd [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_tagged_ep_rdm_states.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_tagged_ep_rdm_states.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c' object='prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm_states.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm_states.lo `test -f 'prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c
+
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm.lo: prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_tagged_ep_rdm.Tpo -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_tagged_ep_rdm.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_tagged_ep_rdm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c' object='prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_tagged_ep_rdm.lo `test -f 'prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c
+
+prov/verbs/src/ep_rdm/src_libfabric_la-verbs_utils.lo: prov/verbs/src/ep_rdm/verbs_utils.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/verbs/src/ep_rdm/src_libfabric_la-verbs_utils.lo -MD -MP -MF prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_utils.Tpo -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_utils.lo `test -f 'prov/verbs/src/ep_rdm/verbs_utils.c' || echo '$(srcdir)/'`prov/ve [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_utils.Tpo prov/verbs/src/ep_rdm/$(DEPDIR)/src_libfabric_la-verbs_utils.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/verbs/src/ep_rdm/verbs_utils.c' object='prov/verbs/src/ep_rdm/src_libfabric_la-verbs_utils.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/verbs/src/ep_rdm/src_libfabric_la-verbs_utils.lo `test -f 'prov/verbs/src/ep_rdm/verbs_utils.c' || echo '$(srcdir)/'`prov/verbs/src/ep_rdm/verbs_utils.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-libnl_utils_common.lo: prov/usnic/src/usnic_direct/libnl_utils_common.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-libnl_utils_common.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-libnl_utils_common.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-libnl_utils_common.lo `test -f 'prov/usnic/src/usnic_direc [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-libnl_utils_common.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-libnl_utils_common.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/libnl_utils_common.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-libnl_utils_common.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-libnl_utils_common.lo `test -f 'prov/usnic/src/usnic_direct/libnl_utils_common.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/libnl_utils_common.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_caps.lo: prov/usnic/src/usnic_direct/usd_caps.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_caps.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_caps.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_caps.lo `test -f 'prov/usnic/src/usnic_direct/usd_caps.c' || echo '$(srcdi [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_caps.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_caps.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_caps.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_caps.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_caps.lo `test -f 'prov/usnic/src/usnic_direct/usd_caps.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_caps.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_dest.lo: prov/usnic/src/usnic_direct/usd_dest.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_dest.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_dest.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_dest.lo `test -f 'prov/usnic/src/usnic_direct/usd_dest.c' || echo '$(srcdi [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_dest.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_dest.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_dest.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_dest.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_dest.lo `test -f 'prov/usnic/src/usnic_direct/usd_dest.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_dest.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_device.lo: prov/usnic/src/usnic_direct/usd_device.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_device.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_device.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_device.lo `test -f 'prov/usnic/src/usnic_direct/usd_device.c' || echo [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_device.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_device.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_device.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_device.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_device.lo `test -f 'prov/usnic/src/usnic_direct/usd_device.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_device.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_event.lo: prov/usnic/src/usnic_direct/usd_event.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_event.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_event.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_event.lo `test -f 'prov/usnic/src/usnic_direct/usd_event.c' || echo '$(s [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_event.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_event.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_event.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_event.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_event.lo `test -f 'prov/usnic/src/usnic_direct/usd_event.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_event.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_enum.lo: prov/usnic/src/usnic_direct/usd_enum.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_enum.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_enum.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_enum.lo `test -f 'prov/usnic/src/usnic_direct/usd_enum.c' || echo '$(srcdi [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_enum.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_enum.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_enum.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_enum.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_enum.lo `test -f 'prov/usnic/src/usnic_direct/usd_enum.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_enum.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_cmd.lo: prov/usnic/src/usnic_direct/usd_ib_cmd.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_cmd.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_ib_cmd.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_cmd.lo `test -f 'prov/usnic/src/usnic_direct/usd_ib_cmd.c' || echo [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_ib_cmd.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_ib_cmd.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_ib_cmd.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_cmd.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_cmd.lo `test -f 'prov/usnic/src/usnic_direct/usd_ib_cmd.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_ib_cmd.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_sysfs.lo: prov/usnic/src/usnic_direct/usd_ib_sysfs.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_sysfs.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_ib_sysfs.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_sysfs.lo `test -f 'prov/usnic/src/usnic_direct/usd_ib_sysfs.c' [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_ib_sysfs.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_ib_sysfs.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_ib_sysfs.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_sysfs.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_ib_sysfs.lo `test -f 'prov/usnic/src/usnic_direct/usd_ib_sysfs.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_ib_sysfs.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_mem.lo: prov/usnic/src/usnic_direct/usd_mem.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_mem.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_mem.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_mem.lo `test -f 'prov/usnic/src/usnic_direct/usd_mem.c' || echo '$(srcdir)/' [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_mem.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_mem.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_mem.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_mem.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_mem.lo `test -f 'prov/usnic/src/usnic_direct/usd_mem.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_mem.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_poll.lo: prov/usnic/src/usnic_direct/usd_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_poll.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_poll.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_poll.lo `test -f 'prov/usnic/src/usnic_direct/usd_poll.c' || echo '$(srcdi [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_poll.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_poll.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_poll.lo `test -f 'prov/usnic/src/usnic_direct/usd_poll.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_poll.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_post.lo: prov/usnic/src/usnic_direct/usd_post.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_post.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_post.lo `test -f 'prov/usnic/src/usnic_direct/usd_post.c' || echo '$(srcdi [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_post.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_post.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_post.lo `test -f 'prov/usnic/src/usnic_direct/usd_post.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_post.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_raw.lo: prov/usnic/src/usnic_direct/usd_post_ud_raw.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_raw.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post_ud_raw.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_raw.lo `test -f 'prov/usnic/src/usnic_direct/usd_pos [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post_ud_raw.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post_ud_raw.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_post_ud_raw.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_raw.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_raw.lo `test -f 'prov/usnic/src/usnic_direct/usd_post_ud_raw.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_post_ud_raw.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_udp.lo: prov/usnic/src/usnic_direct/usd_post_ud_udp.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_udp.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post_ud_udp.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_udp.lo `test -f 'prov/usnic/src/usnic_direct/usd_pos [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post_ud_udp.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post_ud_udp.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_post_ud_udp.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_udp.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_udp.lo `test -f 'prov/usnic/src/usnic_direct/usd_post_ud_udp.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_post_ud_udp.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_pio_udp.lo: prov/usnic/src/usnic_direct/usd_post_ud_pio_udp.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_pio_udp.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post_ud_pio_udp.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_pio_udp.lo `test -f 'prov/usnic/src/usnic_di [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post_ud_pio_udp.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_post_ud_pio_udp.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_post_ud_pio_udp.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_pio_udp.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_post_ud_pio_udp.lo `test -f 'prov/usnic/src/usnic_direct/usd_post_ud_pio_udp.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_post_ud_pio_udp.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_queues.lo: prov/usnic/src/usnic_direct/usd_queues.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_queues.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_queues.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_queues.lo `test -f 'prov/usnic/src/usnic_direct/usd_queues.c' || echo [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_queues.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_queues.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_queues.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_queues.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_queues.lo `test -f 'prov/usnic/src/usnic_direct/usd_queues.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_queues.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_socket.lo: prov/usnic/src/usnic_direct/usd_socket.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_socket.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_socket.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_socket.lo `test -f 'prov/usnic/src/usnic_direct/usd_socket.c' || echo [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_socket.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_socket.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_socket.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_socket.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_socket.lo `test -f 'prov/usnic/src/usnic_direct/usd_socket.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_socket.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usd_vnic.lo: prov/usnic/src/usnic_direct/usd_vnic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usd_vnic.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_vnic.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_vnic.lo `test -f 'prov/usnic/src/usnic_direct/usd_vnic.c' || echo '$(srcdi [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_vnic.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usd_vnic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usd_vnic.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usd_vnic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usd_vnic.lo `test -f 'prov/usnic/src/usnic_direct/usd_vnic.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usd_vnic.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-usnic_ip_utils.lo: prov/usnic/src/usnic_direct/usnic_ip_utils.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-usnic_ip_utils.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usnic_ip_utils.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usnic_ip_utils.lo `test -f 'prov/usnic/src/usnic_direct/usnic_ip_u [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usnic_ip_utils.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-usnic_ip_utils.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/usnic_ip_utils.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-usnic_ip_utils.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-usnic_ip_utils.lo `test -f 'prov/usnic/src/usnic_direct/usnic_ip_utils.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/usnic_ip_utils.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-vnic_cq.lo: prov/usnic/src/usnic_direct/vnic_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-vnic_cq.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_cq.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-vnic_cq.lo `test -f 'prov/usnic/src/usnic_direct/vnic_cq.c' || echo '$(srcdir)/' [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_cq.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/vnic_cq.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-vnic_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-vnic_cq.lo `test -f 'prov/usnic/src/usnic_direct/vnic_cq.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/vnic_cq.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-vnic_dev.lo: prov/usnic/src/usnic_direct/vnic_dev.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-vnic_dev.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_dev.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-vnic_dev.lo `test -f 'prov/usnic/src/usnic_direct/vnic_dev.c' || echo '$(srcdi [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_dev.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_dev.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/vnic_dev.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-vnic_dev.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-vnic_dev.lo `test -f 'prov/usnic/src/usnic_direct/vnic_dev.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/vnic_dev.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-vnic_rq.lo: prov/usnic/src/usnic_direct/vnic_rq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-vnic_rq.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_rq.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-vnic_rq.lo `test -f 'prov/usnic/src/usnic_direct/vnic_rq.c' || echo '$(srcdir)/' [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_rq.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_rq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/vnic_rq.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-vnic_rq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-vnic_rq.lo `test -f 'prov/usnic/src/usnic_direct/vnic_rq.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/vnic_rq.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-vnic_wq.lo: prov/usnic/src/usnic_direct/vnic_wq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-vnic_wq.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_wq.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-vnic_wq.lo `test -f 'prov/usnic/src/usnic_direct/vnic_wq.c' || echo '$(srcdir)/' [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_wq.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_wq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/vnic_wq.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-vnic_wq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-vnic_wq.lo `test -f 'prov/usnic/src/usnic_direct/vnic_wq.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/vnic_wq.c
+
+prov/usnic/src/usnic_direct/src_libfabric_la-vnic_intr.lo: prov/usnic/src/usnic_direct/vnic_intr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/usnic_direct/src_libfabric_la-vnic_intr.lo -MD -MP -MF prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_intr.Tpo -c -o prov/usnic/src/usnic_direct/src_libfabric_la-vnic_intr.lo `test -f 'prov/usnic/src/usnic_direct/vnic_intr.c' || echo '$(s [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_intr.Tpo prov/usnic/src/usnic_direct/$(DEPDIR)/src_libfabric_la-vnic_intr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usnic_direct/vnic_intr.c' object='prov/usnic/src/usnic_direct/src_libfabric_la-vnic_intr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/usnic_direct/src_libfabric_la-vnic_intr.lo `test -f 'prov/usnic/src/usnic_direct/vnic_intr.c' || echo '$(srcdir)/'`prov/usnic/src/usnic_direct/vnic_intr.c
+
+prov/usnic/src/src_libfabric_la-usdf_av.lo: prov/usnic/src/usdf_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_av.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_av.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_av.lo `test -f 'prov/usnic/src/usdf_av.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_av.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_av.c' object='prov/usnic/src/src_libfabric_la-usdf_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_av.lo `test -f 'prov/usnic/src/usdf_av.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_av.c
+
+prov/usnic/src/src_libfabric_la-usdf_cm.lo: prov/usnic/src/usdf_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_cm.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_cm.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_cm.lo `test -f 'prov/usnic/src/usdf_cm.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_cm.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_cm.c' object='prov/usnic/src/src_libfabric_la-usdf_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_cm.lo `test -f 'prov/usnic/src/usdf_cm.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_cm.c
+
+prov/usnic/src/src_libfabric_la-usdf_cq.lo: prov/usnic/src/usdf_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_cq.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_cq.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_cq.lo `test -f 'prov/usnic/src/usdf_cq.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_cq.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_cq.c' object='prov/usnic/src/src_libfabric_la-usdf_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_cq.lo `test -f 'prov/usnic/src/usdf_cq.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_cq.c
+
+prov/usnic/src/src_libfabric_la-usdf_dgram.lo: prov/usnic/src/usdf_dgram.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_dgram.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_dgram.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_dgram.lo `test -f 'prov/usnic/src/usdf_dgram.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_dgram.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_dgram.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_dgram.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_dgram.c' object='prov/usnic/src/src_libfabric_la-usdf_dgram.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_dgram.lo `test -f 'prov/usnic/src/usdf_dgram.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_dgram.c
+
+prov/usnic/src/src_libfabric_la-usdf_domain.lo: prov/usnic/src/usdf_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_domain.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_domain.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_domain.lo `test -f 'prov/usnic/src/usdf_domain.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_domain.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_domain.c' object='prov/usnic/src/src_libfabric_la-usdf_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_domain.lo `test -f 'prov/usnic/src/usdf_domain.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_domain.c
+
+prov/usnic/src/src_libfabric_la-usdf_endpoint.lo: prov/usnic/src/usdf_endpoint.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_endpoint.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_endpoint.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_endpoint.lo `test -f 'prov/usnic/src/usdf_endpoint.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_endpoint.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_endpoint.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_endpoint.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_endpoint.c' object='prov/usnic/src/src_libfabric_la-usdf_endpoint.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_endpoint.lo `test -f 'prov/usnic/src/usdf_endpoint.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_endpoint.c
+
+prov/usnic/src/src_libfabric_la-usdf_ep_dgram.lo: prov/usnic/src/usdf_ep_dgram.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_ep_dgram.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ep_dgram.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_ep_dgram.lo `test -f 'prov/usnic/src/usdf_ep_dgram.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ep_dgram.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ep_dgram.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ep_dgram.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_ep_dgram.c' object='prov/usnic/src/src_libfabric_la-usdf_ep_dgram.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_ep_dgram.lo `test -f 'prov/usnic/src/usdf_ep_dgram.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ep_dgram.c
+
+prov/usnic/src/src_libfabric_la-usdf_ep_msg.lo: prov/usnic/src/usdf_ep_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_ep_msg.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ep_msg.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_ep_msg.lo `test -f 'prov/usnic/src/usdf_ep_msg.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ep_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ep_msg.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ep_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_ep_msg.c' object='prov/usnic/src/src_libfabric_la-usdf_ep_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_ep_msg.lo `test -f 'prov/usnic/src/usdf_ep_msg.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ep_msg.c
+
+prov/usnic/src/src_libfabric_la-usdf_ep_rdm.lo: prov/usnic/src/usdf_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_ep_rdm.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ep_rdm.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_ep_rdm.lo `test -f 'prov/usnic/src/usdf_ep_rdm.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ep_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ep_rdm.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ep_rdm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_ep_rdm.c' object='prov/usnic/src/src_libfabric_la-usdf_ep_rdm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_ep_rdm.lo `test -f 'prov/usnic/src/usdf_ep_rdm.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ep_rdm.c
+
+prov/usnic/src/src_libfabric_la-usdf_eq.lo: prov/usnic/src/usdf_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_eq.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_eq.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_eq.lo `test -f 'prov/usnic/src/usdf_eq.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_eq.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_eq.c' object='prov/usnic/src/src_libfabric_la-usdf_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_eq.lo `test -f 'prov/usnic/src/usdf_eq.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_eq.c
+
+prov/usnic/src/src_libfabric_la-usdf_fabric.lo: prov/usnic/src/usdf_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_fabric.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_fabric.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_fabric.lo `test -f 'prov/usnic/src/usdf_fabric.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_fabric.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_fabric.c' object='prov/usnic/src/src_libfabric_la-usdf_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_fabric.lo `test -f 'prov/usnic/src/usdf_fabric.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_fabric.c
+
+prov/usnic/src/src_libfabric_la-usdf_mem.lo: prov/usnic/src/usdf_mem.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_mem.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_mem.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_mem.lo `test -f 'prov/usnic/src/usdf_mem.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_mem.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_mem.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_mem.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_mem.c' object='prov/usnic/src/src_libfabric_la-usdf_mem.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_mem.lo `test -f 'prov/usnic/src/usdf_mem.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_mem.c
+
+prov/usnic/src/src_libfabric_la-usdf_msg.lo: prov/usnic/src/usdf_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_msg.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_msg.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_msg.lo `test -f 'prov/usnic/src/usdf_msg.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_msg.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_msg.c' object='prov/usnic/src/src_libfabric_la-usdf_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_msg.lo `test -f 'prov/usnic/src/usdf_msg.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_msg.c
+
+prov/usnic/src/src_libfabric_la-usdf_pep.lo: prov/usnic/src/usdf_pep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_pep.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_pep.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_pep.lo `test -f 'prov/usnic/src/usdf_pep.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_pep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_pep.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_pep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_pep.c' object='prov/usnic/src/src_libfabric_la-usdf_pep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_pep.lo `test -f 'prov/usnic/src/usdf_pep.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_pep.c
+
+prov/usnic/src/src_libfabric_la-usdf_progress.lo: prov/usnic/src/usdf_progress.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_progress.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_progress.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_progress.lo `test -f 'prov/usnic/src/usdf_progress.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_progress.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_progress.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_progress.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_progress.c' object='prov/usnic/src/src_libfabric_la-usdf_progress.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_progress.lo `test -f 'prov/usnic/src/usdf_progress.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_progress.c
+
+prov/usnic/src/src_libfabric_la-usdf_rdm.lo: prov/usnic/src/usdf_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_rdm.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_rdm.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_rdm.lo `test -f 'prov/usnic/src/usdf_rdm.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_rdm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_rdm.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_rdm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_rdm.c' object='prov/usnic/src/src_libfabric_la-usdf_rdm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_rdm.lo `test -f 'prov/usnic/src/usdf_rdm.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_rdm.c
+
+prov/usnic/src/src_libfabric_la-usdf_timer.lo: prov/usnic/src/usdf_timer.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_timer.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_timer.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_timer.lo `test -f 'prov/usnic/src/usdf_timer.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_timer.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_timer.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_timer.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_timer.c' object='prov/usnic/src/src_libfabric_la-usdf_timer.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_timer.lo `test -f 'prov/usnic/src/usdf_timer.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_timer.c
+
+prov/usnic/src/src_libfabric_la-usdf_poll.lo: prov/usnic/src/usdf_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_poll.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_poll.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_poll.lo `test -f 'prov/usnic/src/usdf_poll.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_poll.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_poll.c' object='prov/usnic/src/src_libfabric_la-usdf_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_poll.lo `test -f 'prov/usnic/src/usdf_poll.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_poll.c
+
+prov/usnic/src/src_libfabric_la-usdf_ext.lo: prov/usnic/src/usdf_ext.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_ext.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ext.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_ext.lo `test -f 'prov/usnic/src/usdf_ext.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ext.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ext.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_ext.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_ext.c' object='prov/usnic/src/src_libfabric_la-usdf_ext.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_ext.lo `test -f 'prov/usnic/src/usdf_ext.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_ext.c
+
+prov/usnic/src/src_libfabric_la-usdf_wait.lo: prov/usnic/src/usdf_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_wait.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_wait.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_wait.lo `test -f 'prov/usnic/src/usdf_wait.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_wait.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_wait.c' object='prov/usnic/src/src_libfabric_la-usdf_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_wait.lo `test -f 'prov/usnic/src/usdf_wait.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_wait.c
+
+prov/usnic/src/src_libfabric_la-usdf_fake_ibv.lo: prov/usnic/src/usdf_fake_ibv.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/usnic/src/src_libfabric_la-usdf_fake_ibv.lo -MD -MP -MF prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_fake_ibv.Tpo -c -o prov/usnic/src/src_libfabric_la-usdf_fake_ibv.lo `test -f 'prov/usnic/src/usdf_fake_ibv.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_fake_ibv.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_fake_ibv.Tpo prov/usnic/src/$(DEPDIR)/src_libfabric_la-usdf_fake_ibv.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/usnic/src/usdf_fake_ibv.c' object='prov/usnic/src/src_libfabric_la-usdf_fake_ibv.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/usnic/src/src_libfabric_la-usdf_fake_ibv.lo `test -f 'prov/usnic/src/usdf_fake_ibv.c' || echo '$(srcdir)/'`prov/usnic/src/usdf_fake_ibv.c
+
+prov/mxm/src/src_libfabric_la-mlxm_init.lo: prov/mxm/src/mlxm_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/src_libfabric_la-mlxm_init.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_init.Tpo -c -o prov/mxm/src/src_libfabric_la-mlxm_init.lo `test -f 'prov/mxm/src/mlxm_init.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_init.Tpo prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_init.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_init.c' object='prov/mxm/src/src_libfabric_la-mlxm_init.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/src_libfabric_la-mlxm_init.lo `test -f 'prov/mxm/src/mlxm_init.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_init.c
+
+prov/mxm/src/src_libfabric_la-mlxm_domain.lo: prov/mxm/src/mlxm_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/src_libfabric_la-mlxm_domain.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_domain.Tpo -c -o prov/mxm/src/src_libfabric_la-mlxm_domain.lo `test -f 'prov/mxm/src/mlxm_domain.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_domain.Tpo prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_domain.c' object='prov/mxm/src/src_libfabric_la-mlxm_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/src_libfabric_la-mlxm_domain.lo `test -f 'prov/mxm/src/mlxm_domain.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_domain.c
+
+prov/mxm/src/src_libfabric_la-mlxm_cq.lo: prov/mxm/src/mlxm_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/src_libfabric_la-mlxm_cq.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_cq.Tpo -c -o prov/mxm/src/src_libfabric_la-mlxm_cq.lo `test -f 'prov/mxm/src/mlxm_cq.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_cq.Tpo prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_cq.c' object='prov/mxm/src/src_libfabric_la-mlxm_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/src_libfabric_la-mlxm_cq.lo `test -f 'prov/mxm/src/mlxm_cq.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_cq.c
+
+prov/mxm/src/src_libfabric_la-mlxm_av.lo: prov/mxm/src/mlxm_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/src_libfabric_la-mlxm_av.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_av.Tpo -c -o prov/mxm/src/src_libfabric_la-mlxm_av.lo `test -f 'prov/mxm/src/mlxm_av.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_av.Tpo prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_av.c' object='prov/mxm/src/src_libfabric_la-mlxm_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/src_libfabric_la-mlxm_av.lo `test -f 'prov/mxm/src/mlxm_av.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_av.c
+
+prov/mxm/src/src_libfabric_la-mlxm_ep.lo: prov/mxm/src/mlxm_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/src_libfabric_la-mlxm_ep.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_ep.Tpo -c -o prov/mxm/src/src_libfabric_la-mlxm_ep.lo `test -f 'prov/mxm/src/mlxm_ep.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_ep.Tpo prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_ep.c' object='prov/mxm/src/src_libfabric_la-mlxm_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/src_libfabric_la-mlxm_ep.lo `test -f 'prov/mxm/src/mlxm_ep.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_ep.c
+
+prov/mxm/src/src_libfabric_la-mlxm_cm.lo: prov/mxm/src/mlxm_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/src_libfabric_la-mlxm_cm.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_cm.Tpo -c -o prov/mxm/src/src_libfabric_la-mlxm_cm.lo `test -f 'prov/mxm/src/mlxm_cm.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_cm.Tpo prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_cm.c' object='prov/mxm/src/src_libfabric_la-mlxm_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/src_libfabric_la-mlxm_cm.lo `test -f 'prov/mxm/src/mlxm_cm.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_cm.c
+
+prov/mxm/src/src_libfabric_la-mlxm_tagged.lo: prov/mxm/src/mlxm_tagged.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/src_libfabric_la-mlxm_tagged.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_tagged.Tpo -c -o prov/mxm/src/src_libfabric_la-mlxm_tagged.lo `test -f 'prov/mxm/src/mlxm_tagged.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_tagged.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_tagged.Tpo prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_tagged.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_tagged.c' object='prov/mxm/src/src_libfabric_la-mlxm_tagged.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/src_libfabric_la-mlxm_tagged.lo `test -f 'prov/mxm/src/mlxm_tagged.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_tagged.c
+
+prov/mxm/src/src_libfabric_la-mlxm_mr.lo: prov/mxm/src/mlxm_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/mxm/src/src_libfabric_la-mlxm_mr.lo -MD -MP -MF prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_mr.Tpo -c -o prov/mxm/src/src_libfabric_la-mlxm_mr.lo `test -f 'prov/mxm/src/mlxm_mr.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_mr.Tpo prov/mxm/src/$(DEPDIR)/src_libfabric_la-mlxm_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/mxm/src/mlxm_mr.c' object='prov/mxm/src/src_libfabric_la-mlxm_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/mxm/src/src_libfabric_la-mlxm_mr.lo `test -f 'prov/mxm/src/mlxm_mr.c' || echo '$(srcdir)/'`prov/mxm/src/mlxm_mr.c
+
+prov/gni/src/src_libfabric_la-gnix_atomic.lo: prov/gni/src/gnix_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_atomic.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_atomic.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_atomic.lo `test -f 'prov/gni/src/gnix_atomic.c' || echo '$(srcdir)/'`prov/gni/src/gnix_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_atomic.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_atomic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_atomic.c' object='prov/gni/src/src_libfabric_la-gnix_atomic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_atomic.lo `test -f 'prov/gni/src/gnix_atomic.c' || echo '$(srcdir)/'`prov/gni/src/gnix_atomic.c
+
+prov/gni/src/src_libfabric_la-gnix_av.lo: prov/gni/src/gnix_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_av.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_av.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_av.lo `test -f 'prov/gni/src/gnix_av.c' || echo '$(srcdir)/'`prov/gni/src/gnix_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_av.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_av.c' object='prov/gni/src/src_libfabric_la-gnix_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_av.lo `test -f 'prov/gni/src/gnix_av.c' || echo '$(srcdir)/'`prov/gni/src/gnix_av.c
+
+prov/gni/src/src_libfabric_la-gnix_bitmap.lo: prov/gni/src/gnix_bitmap.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_bitmap.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_bitmap.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_bitmap.lo `test -f 'prov/gni/src/gnix_bitmap.c' || echo '$(srcdir)/'`prov/gni/src/gnix_bitmap.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_bitmap.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_bitmap.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_bitmap.c' object='prov/gni/src/src_libfabric_la-gnix_bitmap.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_bitmap.lo `test -f 'prov/gni/src/gnix_bitmap.c' || echo '$(srcdir)/'`prov/gni/src/gnix_bitmap.c
+
+prov/gni/src/src_libfabric_la-gnix_buddy_allocator.lo: prov/gni/src/gnix_buddy_allocator.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_buddy_allocator.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_buddy_allocator.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_buddy_allocator.lo `test -f 'prov/gni/src/gnix_buddy_allocator.c' || echo '$(srcdir)/'`prov/gn [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_buddy_allocator.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_buddy_allocator.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_buddy_allocator.c' object='prov/gni/src/src_libfabric_la-gnix_buddy_allocator.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_buddy_allocator.lo `test -f 'prov/gni/src/gnix_buddy_allocator.c' || echo '$(srcdir)/'`prov/gni/src/gnix_buddy_allocator.c
+
+prov/gni/src/src_libfabric_la-gnix_cm.lo: prov/gni/src/gnix_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_cm.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cm.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_cm.lo `test -f 'prov/gni/src/gnix_cm.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cm.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cm.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cm.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_cm.c' object='prov/gni/src/src_libfabric_la-gnix_cm.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_cm.lo `test -f 'prov/gni/src/gnix_cm.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cm.c
+
+prov/gni/src/src_libfabric_la-gnix_cm_nic.lo: prov/gni/src/gnix_cm_nic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_cm_nic.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cm_nic.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_cm_nic.lo `test -f 'prov/gni/src/gnix_cm_nic.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cm_nic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cm_nic.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cm_nic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_cm_nic.c' object='prov/gni/src/src_libfabric_la-gnix_cm_nic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_cm_nic.lo `test -f 'prov/gni/src/gnix_cm_nic.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cm_nic.c
+
+prov/gni/src/src_libfabric_la-gnix_cntr.lo: prov/gni/src/gnix_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_cntr.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cntr.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_cntr.lo `test -f 'prov/gni/src/gnix_cntr.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cntr.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cntr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_cntr.c' object='prov/gni/src/src_libfabric_la-gnix_cntr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_cntr.lo `test -f 'prov/gni/src/gnix_cntr.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cntr.c
+
+prov/gni/src/src_libfabric_la-gnix_cq.lo: prov/gni/src/gnix_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_cq.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cq.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_cq.lo `test -f 'prov/gni/src/gnix_cq.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cq.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_cq.c' object='prov/gni/src/src_libfabric_la-gnix_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_cq.lo `test -f 'prov/gni/src/gnix_cq.c' || echo '$(srcdir)/'`prov/gni/src/gnix_cq.c
+
+prov/gni/src/src_libfabric_la-gnix_datagram.lo: prov/gni/src/gnix_datagram.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_datagram.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_datagram.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_datagram.lo `test -f 'prov/gni/src/gnix_datagram.c' || echo '$(srcdir)/'`prov/gni/src/gnix_datagram.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_datagram.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_datagram.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_datagram.c' object='prov/gni/src/src_libfabric_la-gnix_datagram.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_datagram.lo `test -f 'prov/gni/src/gnix_datagram.c' || echo '$(srcdir)/'`prov/gni/src/gnix_datagram.c
+
+prov/gni/src/src_libfabric_la-gnix_dom.lo: prov/gni/src/gnix_dom.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_dom.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_dom.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_dom.lo `test -f 'prov/gni/src/gnix_dom.c' || echo '$(srcdir)/'`prov/gni/src/gnix_dom.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_dom.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_dom.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_dom.c' object='prov/gni/src/src_libfabric_la-gnix_dom.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_dom.lo `test -f 'prov/gni/src/gnix_dom.c' || echo '$(srcdir)/'`prov/gni/src/gnix_dom.c
+
+prov/gni/src/src_libfabric_la-gnix_ep.lo: prov/gni/src/gnix_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_ep.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_ep.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_ep.lo `test -f 'prov/gni/src/gnix_ep.c' || echo '$(srcdir)/'`prov/gni/src/gnix_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_ep.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_ep.c' object='prov/gni/src/src_libfabric_la-gnix_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_ep.lo `test -f 'prov/gni/src/gnix_ep.c' || echo '$(srcdir)/'`prov/gni/src/gnix_ep.c
+
+prov/gni/src/src_libfabric_la-gnix_eq.lo: prov/gni/src/gnix_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_eq.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_eq.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_eq.lo `test -f 'prov/gni/src/gnix_eq.c' || echo '$(srcdir)/'`prov/gni/src/gnix_eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_eq.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_eq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_eq.c' object='prov/gni/src/src_libfabric_la-gnix_eq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_eq.lo `test -f 'prov/gni/src/gnix_eq.c' || echo '$(srcdir)/'`prov/gni/src/gnix_eq.c
+
+prov/gni/src/src_libfabric_la-gnix_fabric.lo: prov/gni/src/gnix_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_fabric.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_fabric.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_fabric.lo `test -f 'prov/gni/src/gnix_fabric.c' || echo '$(srcdir)/'`prov/gni/src/gnix_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_fabric.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_fabric.c' object='prov/gni/src/src_libfabric_la-gnix_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_fabric.lo `test -f 'prov/gni/src/gnix_fabric.c' || echo '$(srcdir)/'`prov/gni/src/gnix_fabric.c
+
+prov/gni/src/src_libfabric_la-gnix_freelist.lo: prov/gni/src/gnix_freelist.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_freelist.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_freelist.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_freelist.lo `test -f 'prov/gni/src/gnix_freelist.c' || echo '$(srcdir)/'`prov/gni/src/gnix_freelist.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_freelist.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_freelist.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_freelist.c' object='prov/gni/src/src_libfabric_la-gnix_freelist.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_freelist.lo `test -f 'prov/gni/src/gnix_freelist.c' || echo '$(srcdir)/'`prov/gni/src/gnix_freelist.c
+
+prov/gni/src/src_libfabric_la-gnix_hashtable.lo: prov/gni/src/gnix_hashtable.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_hashtable.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_hashtable.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_hashtable.lo `test -f 'prov/gni/src/gnix_hashtable.c' || echo '$(srcdir)/'`prov/gni/src/gnix_hashtable.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_hashtable.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_hashtable.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_hashtable.c' object='prov/gni/src/src_libfabric_la-gnix_hashtable.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_hashtable.lo `test -f 'prov/gni/src/gnix_hashtable.c' || echo '$(srcdir)/'`prov/gni/src/gnix_hashtable.c
+
+prov/gni/src/src_libfabric_la-gnix_init.lo: prov/gni/src/gnix_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_init.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_init.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_init.lo `test -f 'prov/gni/src/gnix_init.c' || echo '$(srcdir)/'`prov/gni/src/gnix_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_init.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_init.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_init.c' object='prov/gni/src/src_libfabric_la-gnix_init.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_init.lo `test -f 'prov/gni/src/gnix_init.c' || echo '$(srcdir)/'`prov/gni/src/gnix_init.c
+
+prov/gni/src/src_libfabric_la-gnix_mbox_allocator.lo: prov/gni/src/gnix_mbox_allocator.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_mbox_allocator.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mbox_allocator.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_mbox_allocator.lo `test -f 'prov/gni/src/gnix_mbox_allocator.c' || echo '$(srcdir)/'`prov/gni/sr [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mbox_allocator.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mbox_allocator.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_mbox_allocator.c' object='prov/gni/src/src_libfabric_la-gnix_mbox_allocator.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_mbox_allocator.lo `test -f 'prov/gni/src/gnix_mbox_allocator.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mbox_allocator.c
+
+prov/gni/src/src_libfabric_la-gnix_mr.lo: prov/gni/src/gnix_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_mr.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mr.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_mr.lo `test -f 'prov/gni/src/gnix_mr.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mr.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_mr.c' object='prov/gni/src/src_libfabric_la-gnix_mr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_mr.lo `test -f 'prov/gni/src/gnix_mr.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mr.c
+
+prov/gni/src/src_libfabric_la-gnix_mr_cache.lo: prov/gni/src/gnix_mr_cache.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_mr_cache.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mr_cache.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_mr_cache.lo `test -f 'prov/gni/src/gnix_mr_cache.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mr_cache.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mr_cache.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mr_cache.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_mr_cache.c' object='prov/gni/src/src_libfabric_la-gnix_mr_cache.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_mr_cache.lo `test -f 'prov/gni/src/gnix_mr_cache.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mr_cache.c
+
+prov/gni/src/src_libfabric_la-gnix_mr_notifier.lo: prov/gni/src/gnix_mr_notifier.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_mr_notifier.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mr_notifier.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_mr_notifier.lo `test -f 'prov/gni/src/gnix_mr_notifier.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mr_no [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mr_notifier.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_mr_notifier.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_mr_notifier.c' object='prov/gni/src/src_libfabric_la-gnix_mr_notifier.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_mr_notifier.lo `test -f 'prov/gni/src/gnix_mr_notifier.c' || echo '$(srcdir)/'`prov/gni/src/gnix_mr_notifier.c
+
+prov/gni/src/src_libfabric_la-gnix_msg.lo: prov/gni/src/gnix_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_msg.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_msg.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_msg.lo `test -f 'prov/gni/src/gnix_msg.c' || echo '$(srcdir)/'`prov/gni/src/gnix_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_msg.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_msg.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_msg.c' object='prov/gni/src/src_libfabric_la-gnix_msg.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_msg.lo `test -f 'prov/gni/src/gnix_msg.c' || echo '$(srcdir)/'`prov/gni/src/gnix_msg.c
+
+prov/gni/src/src_libfabric_la-gnix_nameserver.lo: prov/gni/src/gnix_nameserver.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_nameserver.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_nameserver.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_nameserver.lo `test -f 'prov/gni/src/gnix_nameserver.c' || echo '$(srcdir)/'`prov/gni/src/gnix_nameserver.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_nameserver.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_nameserver.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_nameserver.c' object='prov/gni/src/src_libfabric_la-gnix_nameserver.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_nameserver.lo `test -f 'prov/gni/src/gnix_nameserver.c' || echo '$(srcdir)/'`prov/gni/src/gnix_nameserver.c
+
+prov/gni/src/src_libfabric_la-gnix_nic.lo: prov/gni/src/gnix_nic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_nic.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_nic.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_nic.lo `test -f 'prov/gni/src/gnix_nic.c' || echo '$(srcdir)/'`prov/gni/src/gnix_nic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_nic.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_nic.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_nic.c' object='prov/gni/src/src_libfabric_la-gnix_nic.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_nic.lo `test -f 'prov/gni/src/gnix_nic.c' || echo '$(srcdir)/'`prov/gni/src/gnix_nic.c
+
+prov/gni/src/src_libfabric_la-gnix_poll.lo: prov/gni/src/gnix_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_poll.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_poll.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_poll.lo `test -f 'prov/gni/src/gnix_poll.c' || echo '$(srcdir)/'`prov/gni/src/gnix_poll.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_poll.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_poll.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_poll.c' object='prov/gni/src/src_libfabric_la-gnix_poll.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_poll.lo `test -f 'prov/gni/src/gnix_poll.c' || echo '$(srcdir)/'`prov/gni/src/gnix_poll.c
+
+prov/gni/src/src_libfabric_la-gnix_queue.lo: prov/gni/src/gnix_queue.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_queue.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_queue.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_queue.lo `test -f 'prov/gni/src/gnix_queue.c' || echo '$(srcdir)/'`prov/gni/src/gnix_queue.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_queue.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_queue.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_queue.c' object='prov/gni/src/src_libfabric_la-gnix_queue.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_queue.lo `test -f 'prov/gni/src/gnix_queue.c' || echo '$(srcdir)/'`prov/gni/src/gnix_queue.c
+
+prov/gni/src/src_libfabric_la-gnix_rma.lo: prov/gni/src/gnix_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_rma.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_rma.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_rma.lo `test -f 'prov/gni/src/gnix_rma.c' || echo '$(srcdir)/'`prov/gni/src/gnix_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_rma.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_rma.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_rma.c' object='prov/gni/src/src_libfabric_la-gnix_rma.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_rma.lo `test -f 'prov/gni/src/gnix_rma.c' || echo '$(srcdir)/'`prov/gni/src/gnix_rma.c
+
+prov/gni/src/src_libfabric_la-gnix_tags.lo: prov/gni/src/gnix_tags.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_tags.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_tags.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_tags.lo `test -f 'prov/gni/src/gnix_tags.c' || echo '$(srcdir)/'`prov/gni/src/gnix_tags.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_tags.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_tags.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_tags.c' object='prov/gni/src/src_libfabric_la-gnix_tags.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_tags.lo `test -f 'prov/gni/src/gnix_tags.c' || echo '$(srcdir)/'`prov/gni/src/gnix_tags.c
+
+prov/gni/src/src_libfabric_la-gnix_trigger.lo: prov/gni/src/gnix_trigger.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_trigger.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_trigger.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_trigger.lo `test -f 'prov/gni/src/gnix_trigger.c' || echo '$(srcdir)/'`prov/gni/src/gnix_trigger.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_trigger.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_trigger.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_trigger.c' object='prov/gni/src/src_libfabric_la-gnix_trigger.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_trigger.lo `test -f 'prov/gni/src/gnix_trigger.c' || echo '$(srcdir)/'`prov/gni/src/gnix_trigger.c
+
+prov/gni/src/src_libfabric_la-gnix_util.lo: prov/gni/src/gnix_util.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_util.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_util.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_util.lo `test -f 'prov/gni/src/gnix_util.c' || echo '$(srcdir)/'`prov/gni/src/gnix_util.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_util.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_util.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_util.c' object='prov/gni/src/src_libfabric_la-gnix_util.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_util.lo `test -f 'prov/gni/src/gnix_util.c' || echo '$(srcdir)/'`prov/gni/src/gnix_util.c
+
+prov/gni/src/src_libfabric_la-gnix_vc.lo: prov/gni/src/gnix_vc.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_vc.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_vc.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_vc.lo `test -f 'prov/gni/src/gnix_vc.c' || echo '$(srcdir)/'`prov/gni/src/gnix_vc.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_vc.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_vc.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_vc.c' object='prov/gni/src/src_libfabric_la-gnix_vc.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_vc.lo `test -f 'prov/gni/src/gnix_vc.c' || echo '$(srcdir)/'`prov/gni/src/gnix_vc.c
+
+prov/gni/src/src_libfabric_la-gnix_vector.lo: prov/gni/src/gnix_vector.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_vector.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_vector.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_vector.lo `test -f 'prov/gni/src/gnix_vector.c' || echo '$(srcdir)/'`prov/gni/src/gnix_vector.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_vector.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_vector.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_vector.c' object='prov/gni/src/src_libfabric_la-gnix_vector.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_vector.lo `test -f 'prov/gni/src/gnix_vector.c' || echo '$(srcdir)/'`prov/gni/src/gnix_vector.c
+
+prov/gni/src/src_libfabric_la-gnix_xpmem.lo: prov/gni/src/gnix_xpmem.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_xpmem.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_xpmem.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_xpmem.lo `test -f 'prov/gni/src/gnix_xpmem.c' || echo '$(srcdir)/'`prov/gni/src/gnix_xpmem.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_xpmem.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_xpmem.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_xpmem.c' object='prov/gni/src/src_libfabric_la-gnix_xpmem.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_xpmem.lo `test -f 'prov/gni/src/gnix_xpmem.c' || echo '$(srcdir)/'`prov/gni/src/gnix_xpmem.c
+
+prov/gni/src/src_libfabric_la-gnix_wait.lo: prov/gni/src/gnix_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/src/src_libfabric_la-gnix_wait.lo -MD -MP -MF prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_wait.Tpo -c -o prov/gni/src/src_libfabric_la-gnix_wait.lo `test -f 'prov/gni/src/gnix_wait.c' || echo '$(srcdir)/'`prov/gni/src/gnix_wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_wait.Tpo prov/gni/src/$(DEPDIR)/src_libfabric_la-gnix_wait.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/src/gnix_wait.c' object='prov/gni/src/src_libfabric_la-gnix_wait.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/src/src_libfabric_la-gnix_wait.lo `test -f 'prov/gni/src/gnix_wait.c' || echo '$(srcdir)/'`prov/gni/src/gnix_wait.c
+
+prov/rxm/src/src_libfabric_la-rxm_attr.lo: prov/rxm/src/rxm_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxm/src/src_libfabric_la-rxm_attr.lo -MD -MP -MF prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_attr.Tpo -c -o prov/rxm/src/src_libfabric_la-rxm_attr.lo `test -f 'prov/rxm/src/rxm_attr.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_attr.Tpo prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_attr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxm/src/rxm_attr.c' object='prov/rxm/src/src_libfabric_la-rxm_attr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxm/src/src_libfabric_la-rxm_attr.lo `test -f 'prov/rxm/src/rxm_attr.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_attr.c
+
+prov/rxm/src/src_libfabric_la-rxm_init.lo: prov/rxm/src/rxm_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxm/src/src_libfabric_la-rxm_init.lo -MD -MP -MF prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_init.Tpo -c -o prov/rxm/src/src_libfabric_la-rxm_init.lo `test -f 'prov/rxm/src/rxm_init.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_init.Tpo prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_init.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxm/src/rxm_init.c' object='prov/rxm/src/src_libfabric_la-rxm_init.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxm/src/src_libfabric_la-rxm_init.lo `test -f 'prov/rxm/src/rxm_init.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_init.c
+
+prov/rxm/src/src_libfabric_la-rxm_fabric.lo: prov/rxm/src/rxm_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxm/src/src_libfabric_la-rxm_fabric.lo -MD -MP -MF prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_fabric.Tpo -c -o prov/rxm/src/src_libfabric_la-rxm_fabric.lo `test -f 'prov/rxm/src/rxm_fabric.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_fabric.Tpo prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxm/src/rxm_fabric.c' object='prov/rxm/src/src_libfabric_la-rxm_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxm/src/src_libfabric_la-rxm_fabric.lo `test -f 'prov/rxm/src/rxm_fabric.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_fabric.c
+
+prov/rxm/src/src_libfabric_la-rxm_domain.lo: prov/rxm/src/rxm_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxm/src/src_libfabric_la-rxm_domain.lo -MD -MP -MF prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_domain.Tpo -c -o prov/rxm/src/src_libfabric_la-rxm_domain.lo `test -f 'prov/rxm/src/rxm_domain.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_domain.Tpo prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxm/src/rxm_domain.c' object='prov/rxm/src/src_libfabric_la-rxm_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxm/src/src_libfabric_la-rxm_domain.lo `test -f 'prov/rxm/src/rxm_domain.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_domain.c
+
+prov/rxm/src/src_libfabric_la-rxm_conn.lo: prov/rxm/src/rxm_conn.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxm/src/src_libfabric_la-rxm_conn.lo -MD -MP -MF prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_conn.Tpo -c -o prov/rxm/src/src_libfabric_la-rxm_conn.lo `test -f 'prov/rxm/src/rxm_conn.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_conn.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_conn.Tpo prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_conn.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxm/src/rxm_conn.c' object='prov/rxm/src/src_libfabric_la-rxm_conn.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxm/src/src_libfabric_la-rxm_conn.lo `test -f 'prov/rxm/src/rxm_conn.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_conn.c
+
+prov/rxm/src/src_libfabric_la-rxm_ep.lo: prov/rxm/src/rxm_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxm/src/src_libfabric_la-rxm_ep.lo -MD -MP -MF prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_ep.Tpo -c -o prov/rxm/src/src_libfabric_la-rxm_ep.lo `test -f 'prov/rxm/src/rxm_ep.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_ep.Tpo prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxm/src/rxm_ep.c' object='prov/rxm/src/src_libfabric_la-rxm_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxm/src/src_libfabric_la-rxm_ep.lo `test -f 'prov/rxm/src/rxm_ep.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_ep.c
+
+prov/rxm/src/src_libfabric_la-rxm_cq.lo: prov/rxm/src/rxm_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxm/src/src_libfabric_la-rxm_cq.lo -MD -MP -MF prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_cq.Tpo -c -o prov/rxm/src/src_libfabric_la-rxm_cq.lo `test -f 'prov/rxm/src/rxm_cq.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_cq.Tpo prov/rxm/src/$(DEPDIR)/src_libfabric_la-rxm_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxm/src/rxm_cq.c' object='prov/rxm/src/src_libfabric_la-rxm_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxm/src/src_libfabric_la-rxm_cq.lo `test -f 'prov/rxm/src/rxm_cq.c' || echo '$(srcdir)/'`prov/rxm/src/rxm_cq.c
+
+prov/rxd/src/src_libfabric_la-rxd_attr.lo: prov/rxd/src/rxd_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxd/src/src_libfabric_la-rxd_attr.lo -MD -MP -MF prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_attr.Tpo -c -o prov/rxd/src/src_libfabric_la-rxd_attr.lo `test -f 'prov/rxd/src/rxd_attr.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_attr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_attr.Tpo prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_attr.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxd/src/rxd_attr.c' object='prov/rxd/src/src_libfabric_la-rxd_attr.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxd/src/src_libfabric_la-rxd_attr.lo `test -f 'prov/rxd/src/rxd_attr.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_attr.c
+
+prov/rxd/src/src_libfabric_la-rxd_init.lo: prov/rxd/src/rxd_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxd/src/src_libfabric_la-rxd_init.lo -MD -MP -MF prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_init.Tpo -c -o prov/rxd/src/src_libfabric_la-rxd_init.lo `test -f 'prov/rxd/src/rxd_init.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_init.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_init.Tpo prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_init.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxd/src/rxd_init.c' object='prov/rxd/src/src_libfabric_la-rxd_init.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxd/src/src_libfabric_la-rxd_init.lo `test -f 'prov/rxd/src/rxd_init.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_init.c
+
+prov/rxd/src/src_libfabric_la-rxd_fabric.lo: prov/rxd/src/rxd_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxd/src/src_libfabric_la-rxd_fabric.lo -MD -MP -MF prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_fabric.Tpo -c -o prov/rxd/src/src_libfabric_la-rxd_fabric.lo `test -f 'prov/rxd/src/rxd_fabric.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_fabric.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_fabric.Tpo prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_fabric.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxd/src/rxd_fabric.c' object='prov/rxd/src/src_libfabric_la-rxd_fabric.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxd/src/src_libfabric_la-rxd_fabric.lo `test -f 'prov/rxd/src/rxd_fabric.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_fabric.c
+
+prov/rxd/src/src_libfabric_la-rxd_domain.lo: prov/rxd/src/rxd_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxd/src/src_libfabric_la-rxd_domain.lo -MD -MP -MF prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_domain.Tpo -c -o prov/rxd/src/src_libfabric_la-rxd_domain.lo `test -f 'prov/rxd/src/rxd_domain.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_domain.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_domain.Tpo prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_domain.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxd/src/rxd_domain.c' object='prov/rxd/src/src_libfabric_la-rxd_domain.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxd/src/src_libfabric_la-rxd_domain.lo `test -f 'prov/rxd/src/rxd_domain.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_domain.c
+
+prov/rxd/src/src_libfabric_la-rxd_av.lo: prov/rxd/src/rxd_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxd/src/src_libfabric_la-rxd_av.lo -MD -MP -MF prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_av.Tpo -c -o prov/rxd/src/src_libfabric_la-rxd_av.lo `test -f 'prov/rxd/src/rxd_av.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_av.Tpo prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_av.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxd/src/rxd_av.c' object='prov/rxd/src/src_libfabric_la-rxd_av.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxd/src/src_libfabric_la-rxd_av.lo `test -f 'prov/rxd/src/rxd_av.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_av.c
+
+prov/rxd/src/src_libfabric_la-rxd_cq.lo: prov/rxd/src/rxd_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxd/src/src_libfabric_la-rxd_cq.lo -MD -MP -MF prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_cq.Tpo -c -o prov/rxd/src/src_libfabric_la-rxd_cq.lo `test -f 'prov/rxd/src/rxd_cq.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_cq.Tpo prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_cq.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxd/src/rxd_cq.c' object='prov/rxd/src/src_libfabric_la-rxd_cq.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxd/src/src_libfabric_la-rxd_cq.lo `test -f 'prov/rxd/src/rxd_cq.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_cq.c
+
+prov/rxd/src/src_libfabric_la-rxd_ep.lo: prov/rxd/src/rxd_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxd/src/src_libfabric_la-rxd_ep.lo -MD -MP -MF prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_ep.Tpo -c -o prov/rxd/src/src_libfabric_la-rxd_ep.lo `test -f 'prov/rxd/src/rxd_ep.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_ep.Tpo prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_ep.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxd/src/rxd_ep.c' object='prov/rxd/src/src_libfabric_la-rxd_ep.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxd/src/src_libfabric_la-rxd_ep.lo `test -f 'prov/rxd/src/rxd_ep.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_ep.c
+
+prov/rxd/src/src_libfabric_la-rxd_rma.lo: prov/rxd/src/rxd_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/rxd/src/src_libfabric_la-rxd_rma.lo -MD -MP -MF prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_rma.Tpo -c -o prov/rxd/src/src_libfabric_la-rxd_rma.lo `test -f 'prov/rxd/src/rxd_rma.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_rma.Tpo prov/rxd/src/$(DEPDIR)/src_libfabric_la-rxd_rma.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/rxd/src/rxd_rma.c' object='prov/rxd/src/src_libfabric_la-rxd_rma.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(src_libfabric_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/rxd/src/src_libfabric_la-rxd_rma.lo `test -f 'prov/rxd/src/rxd_rma.c' || echo '$(srcdir)/'`prov/rxd/src/rxd_rma.c
+
+prov/gni/test/prov_gni_test_gnitest-allocator.o: prov/gni/test/allocator.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-allocator.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-allocator.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-allocator.o `test -f 'prov/gni/test/allocator.c' || echo '$(srcdir)/'`prov/gni/test/allocator.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-allocator.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-allocator.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/allocator.c' object='prov/gni/test/prov_gni_test_gnitest-allocator.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-allocator.o `test -f 'prov/gni/test/allocator.c' || echo '$(srcdir)/'`prov/gni/test/allocator.c
+
+prov/gni/test/prov_gni_test_gnitest-allocator.obj: prov/gni/test/allocator.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-allocator.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-allocator.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-allocator.obj `if test -f 'prov/gni/test/allocator.c'; then $(CYGPATH_W) 'prov/gni/test/allocator.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/allocator.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-allocator.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-allocator.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/allocator.c' object='prov/gni/test/prov_gni_test_gnitest-allocator.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-allocator.obj `if test -f 'prov/gni/test/allocator.c'; then $(CYGPATH_W) 'prov/gni/test/allocator.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/allocator.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-av.o: prov/gni/test/av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-av.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-av.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-av.o `test -f 'prov/gni/test/av.c' || echo '$(srcdir)/'`prov/gni/test/av.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-av.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-av.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/av.c' object='prov/gni/test/prov_gni_test_gnitest-av.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-av.o `test -f 'prov/gni/test/av.c' || echo '$(srcdir)/'`prov/gni/test/av.c
+
+prov/gni/test/prov_gni_test_gnitest-av.obj: prov/gni/test/av.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-av.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-av.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-av.obj `if test -f 'prov/gni/test/av.c'; then $(CYGPATH_W) 'prov/gni/test/av.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/av.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-av.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-av.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/av.c' object='prov/gni/test/prov_gni_test_gnitest-av.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-av.obj `if test -f 'prov/gni/test/av.c'; then $(CYGPATH_W) 'prov/gni/test/av.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/av.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-bitmap.o: prov/gni/test/bitmap.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-bitmap.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-bitmap.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-bitmap.o `test -f 'prov/gni/test/bitmap.c' || echo '$(srcdir)/'`prov/gni/test/bitmap.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-bitmap.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-bitmap.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/bitmap.c' object='prov/gni/test/prov_gni_test_gnitest-bitmap.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-bitmap.o `test -f 'prov/gni/test/bitmap.c' || echo '$(srcdir)/'`prov/gni/test/bitmap.c
+
+prov/gni/test/prov_gni_test_gnitest-bitmap.obj: prov/gni/test/bitmap.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-bitmap.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-bitmap.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-bitmap.obj `if test -f 'prov/gni/test/bitmap.c'; then $(CYGPATH_W) 'prov/gni/test/bitmap.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/bitmap.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-bitmap.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-bitmap.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/bitmap.c' object='prov/gni/test/prov_gni_test_gnitest-bitmap.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-bitmap.obj `if test -f 'prov/gni/test/bitmap.c'; then $(CYGPATH_W) 'prov/gni/test/bitmap.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/bitmap.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-buddy_allocator.o: prov/gni/test/buddy_allocator.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-buddy_allocator.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-buddy_allocator.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-buddy_allocator.o `test -f 'prov/gni/test/buddy_allocator.c' || echo '$(srcdir)/'`prov/gni/test/buddy_allocator.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-buddy_allocator.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-buddy_allocator.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/buddy_allocator.c' object='prov/gni/test/prov_gni_test_gnitest-buddy_allocator.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-buddy_allocator.o `test -f 'prov/gni/test/buddy_allocator.c' || echo '$(srcdir)/'`prov/gni/test/buddy_allocator.c
+
+prov/gni/test/prov_gni_test_gnitest-buddy_allocator.obj: prov/gni/test/buddy_allocator.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-buddy_allocator.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-buddy_allocator.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-buddy_allocator.obj `if test -f 'prov/gni/test/buddy_allocator.c'; then $(CYGPATH_W) 'prov/gni/test/buddy_allocator.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/bud [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-buddy_allocator.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-buddy_allocator.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/buddy_allocator.c' object='prov/gni/test/prov_gni_test_gnitest-buddy_allocator.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-buddy_allocator.obj `if test -f 'prov/gni/test/buddy_allocator.c'; then $(CYGPATH_W) 'prov/gni/test/buddy_allocator.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/buddy_allocator.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-cancel.o: prov/gni/test/cancel.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-cancel.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cancel.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-cancel.o `test -f 'prov/gni/test/cancel.c' || echo '$(srcdir)/'`prov/gni/test/cancel.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cancel.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cancel.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/cancel.c' object='prov/gni/test/prov_gni_test_gnitest-cancel.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-cancel.o `test -f 'prov/gni/test/cancel.c' || echo '$(srcdir)/'`prov/gni/test/cancel.c
+
+prov/gni/test/prov_gni_test_gnitest-cancel.obj: prov/gni/test/cancel.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-cancel.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cancel.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-cancel.obj `if test -f 'prov/gni/test/cancel.c'; then $(CYGPATH_W) 'prov/gni/test/cancel.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/cancel.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cancel.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cancel.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/cancel.c' object='prov/gni/test/prov_gni_test_gnitest-cancel.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-cancel.obj `if test -f 'prov/gni/test/cancel.c'; then $(CYGPATH_W) 'prov/gni/test/cancel.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/cancel.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-cntr.o: prov/gni/test/cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-cntr.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cntr.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-cntr.o `test -f 'prov/gni/test/cntr.c' || echo '$(srcdir)/'`prov/gni/test/cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cntr.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cntr.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/cntr.c' object='prov/gni/test/prov_gni_test_gnitest-cntr.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-cntr.o `test -f 'prov/gni/test/cntr.c' || echo '$(srcdir)/'`prov/gni/test/cntr.c
+
+prov/gni/test/prov_gni_test_gnitest-cntr.obj: prov/gni/test/cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-cntr.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cntr.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-cntr.obj `if test -f 'prov/gni/test/cntr.c'; then $(CYGPATH_W) 'prov/gni/test/cntr.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/cntr.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cntr.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cntr.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/cntr.c' object='prov/gni/test/prov_gni_test_gnitest-cntr.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-cntr.obj `if test -f 'prov/gni/test/cntr.c'; then $(CYGPATH_W) 'prov/gni/test/cntr.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/cntr.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-cq.o: prov/gni/test/cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-cq.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cq.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-cq.o `test -f 'prov/gni/test/cq.c' || echo '$(srcdir)/'`prov/gni/test/cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cq.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cq.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/cq.c' object='prov/gni/test/prov_gni_test_gnitest-cq.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-cq.o `test -f 'prov/gni/test/cq.c' || echo '$(srcdir)/'`prov/gni/test/cq.c
+
+prov/gni/test/prov_gni_test_gnitest-cq.obj: prov/gni/test/cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-cq.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cq.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-cq.obj `if test -f 'prov/gni/test/cq.c'; then $(CYGPATH_W) 'prov/gni/test/cq.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/cq.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cq.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-cq.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/cq.c' object='prov/gni/test/prov_gni_test_gnitest-cq.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-cq.obj `if test -f 'prov/gni/test/cq.c'; then $(CYGPATH_W) 'prov/gni/test/cq.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/cq.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-datagram.o: prov/gni/test/datagram.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-datagram.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-datagram.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-datagram.o `test -f 'prov/gni/test/datagram.c' || echo '$(srcdir)/'`prov/gni/test/datagram.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-datagram.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-datagram.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/datagram.c' object='prov/gni/test/prov_gni_test_gnitest-datagram.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-datagram.o `test -f 'prov/gni/test/datagram.c' || echo '$(srcdir)/'`prov/gni/test/datagram.c
+
+prov/gni/test/prov_gni_test_gnitest-datagram.obj: prov/gni/test/datagram.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-datagram.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-datagram.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-datagram.obj `if test -f 'prov/gni/test/datagram.c'; then $(CYGPATH_W) 'prov/gni/test/datagram.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/datagram.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-datagram.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-datagram.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/datagram.c' object='prov/gni/test/prov_gni_test_gnitest-datagram.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-datagram.obj `if test -f 'prov/gni/test/datagram.c'; then $(CYGPATH_W) 'prov/gni/test/datagram.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/datagram.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-dlist-utils.o: prov/gni/test/dlist-utils.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-dlist-utils.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dlist-utils.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-dlist-utils.o `test -f 'prov/gni/test/dlist-utils.c' || echo '$(srcdir)/'`prov/gni/test/dlist-utils.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dlist-utils.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dlist-utils.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/dlist-utils.c' object='prov/gni/test/prov_gni_test_gnitest-dlist-utils.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-dlist-utils.o `test -f 'prov/gni/test/dlist-utils.c' || echo '$(srcdir)/'`prov/gni/test/dlist-utils.c
+
+prov/gni/test/prov_gni_test_gnitest-dlist-utils.obj: prov/gni/test/dlist-utils.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-dlist-utils.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dlist-utils.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-dlist-utils.obj `if test -f 'prov/gni/test/dlist-utils.c'; then $(CYGPATH_W) 'prov/gni/test/dlist-utils.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/dlist-utils.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dlist-utils.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dlist-utils.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/dlist-utils.c' object='prov/gni/test/prov_gni_test_gnitest-dlist-utils.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-dlist-utils.obj `if test -f 'prov/gni/test/dlist-utils.c'; then $(CYGPATH_W) 'prov/gni/test/dlist-utils.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/dlist-utils.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-dom.o: prov/gni/test/dom.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-dom.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dom.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-dom.o `test -f 'prov/gni/test/dom.c' || echo '$(srcdir)/'`prov/gni/test/dom.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dom.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dom.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/dom.c' object='prov/gni/test/prov_gni_test_gnitest-dom.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-dom.o `test -f 'prov/gni/test/dom.c' || echo '$(srcdir)/'`prov/gni/test/dom.c
+
+prov/gni/test/prov_gni_test_gnitest-dom.obj: prov/gni/test/dom.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-dom.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dom.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-dom.obj `if test -f 'prov/gni/test/dom.c'; then $(CYGPATH_W) 'prov/gni/test/dom.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/dom.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dom.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-dom.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/dom.c' object='prov/gni/test/prov_gni_test_gnitest-dom.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-dom.obj `if test -f 'prov/gni/test/dom.c'; then $(CYGPATH_W) 'prov/gni/test/dom.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/dom.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-ep.o: prov/gni/test/ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-ep.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-ep.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-ep.o `test -f 'prov/gni/test/ep.c' || echo '$(srcdir)/'`prov/gni/test/ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-ep.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-ep.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/ep.c' object='prov/gni/test/prov_gni_test_gnitest-ep.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-ep.o `test -f 'prov/gni/test/ep.c' || echo '$(srcdir)/'`prov/gni/test/ep.c
+
+prov/gni/test/prov_gni_test_gnitest-ep.obj: prov/gni/test/ep.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-ep.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-ep.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-ep.obj `if test -f 'prov/gni/test/ep.c'; then $(CYGPATH_W) 'prov/gni/test/ep.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/ep.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-ep.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-ep.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/ep.c' object='prov/gni/test/prov_gni_test_gnitest-ep.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-ep.obj `if test -f 'prov/gni/test/ep.c'; then $(CYGPATH_W) 'prov/gni/test/ep.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/ep.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-eq.o: prov/gni/test/eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-eq.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-eq.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-eq.o `test -f 'prov/gni/test/eq.c' || echo '$(srcdir)/'`prov/gni/test/eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-eq.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-eq.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/eq.c' object='prov/gni/test/prov_gni_test_gnitest-eq.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-eq.o `test -f 'prov/gni/test/eq.c' || echo '$(srcdir)/'`prov/gni/test/eq.c
+
+prov/gni/test/prov_gni_test_gnitest-eq.obj: prov/gni/test/eq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-eq.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-eq.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-eq.obj `if test -f 'prov/gni/test/eq.c'; then $(CYGPATH_W) 'prov/gni/test/eq.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/eq.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-eq.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-eq.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/eq.c' object='prov/gni/test/prov_gni_test_gnitest-eq.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-eq.obj `if test -f 'prov/gni/test/eq.c'; then $(CYGPATH_W) 'prov/gni/test/eq.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/eq.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-freelist.o: prov/gni/test/freelist.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-freelist.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-freelist.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-freelist.o `test -f 'prov/gni/test/freelist.c' || echo '$(srcdir)/'`prov/gni/test/freelist.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-freelist.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-freelist.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/freelist.c' object='prov/gni/test/prov_gni_test_gnitest-freelist.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-freelist.o `test -f 'prov/gni/test/freelist.c' || echo '$(srcdir)/'`prov/gni/test/freelist.c
+
+prov/gni/test/prov_gni_test_gnitest-freelist.obj: prov/gni/test/freelist.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-freelist.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-freelist.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-freelist.obj `if test -f 'prov/gni/test/freelist.c'; then $(CYGPATH_W) 'prov/gni/test/freelist.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/freelist.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-freelist.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-freelist.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/freelist.c' object='prov/gni/test/prov_gni_test_gnitest-freelist.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-freelist.obj `if test -f 'prov/gni/test/freelist.c'; then $(CYGPATH_W) 'prov/gni/test/freelist.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/freelist.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-hashtable.o: prov/gni/test/hashtable.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-hashtable.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-hashtable.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-hashtable.o `test -f 'prov/gni/test/hashtable.c' || echo '$(srcdir)/'`prov/gni/test/hashtable.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-hashtable.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-hashtable.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/hashtable.c' object='prov/gni/test/prov_gni_test_gnitest-hashtable.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-hashtable.o `test -f 'prov/gni/test/hashtable.c' || echo '$(srcdir)/'`prov/gni/test/hashtable.c
+
+prov/gni/test/prov_gni_test_gnitest-hashtable.obj: prov/gni/test/hashtable.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-hashtable.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-hashtable.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-hashtable.obj `if test -f 'prov/gni/test/hashtable.c'; then $(CYGPATH_W) 'prov/gni/test/hashtable.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/hashtable.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-hashtable.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-hashtable.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/hashtable.c' object='prov/gni/test/prov_gni_test_gnitest-hashtable.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-hashtable.obj `if test -f 'prov/gni/test/hashtable.c'; then $(CYGPATH_W) 'prov/gni/test/hashtable.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/hashtable.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-mr.o: prov/gni/test/mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-mr.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-mr.o `test -f 'prov/gni/test/mr.c' || echo '$(srcdir)/'`prov/gni/test/mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/mr.c' object='prov/gni/test/prov_gni_test_gnitest-mr.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-mr.o `test -f 'prov/gni/test/mr.c' || echo '$(srcdir)/'`prov/gni/test/mr.c
+
+prov/gni/test/prov_gni_test_gnitest-mr.obj: prov/gni/test/mr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-mr.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-mr.obj `if test -f 'prov/gni/test/mr.c'; then $(CYGPATH_W) 'prov/gni/test/mr.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/mr.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/mr.c' object='prov/gni/test/prov_gni_test_gnitest-mr.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-mr.obj `if test -f 'prov/gni/test/mr.c'; then $(CYGPATH_W) 'prov/gni/test/mr.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/mr.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-mr_notifier.o: prov/gni/test/mr_notifier.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-mr_notifier.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr_notifier.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-mr_notifier.o `test -f 'prov/gni/test/mr_notifier.c' || echo '$(srcdir)/'`prov/gni/test/mr_notifier.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr_notifier.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr_notifier.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/mr_notifier.c' object='prov/gni/test/prov_gni_test_gnitest-mr_notifier.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-mr_notifier.o `test -f 'prov/gni/test/mr_notifier.c' || echo '$(srcdir)/'`prov/gni/test/mr_notifier.c
+
+prov/gni/test/prov_gni_test_gnitest-mr_notifier.obj: prov/gni/test/mr_notifier.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-mr_notifier.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr_notifier.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-mr_notifier.obj `if test -f 'prov/gni/test/mr_notifier.c'; then $(CYGPATH_W) 'prov/gni/test/mr_notifier.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/mr_notifier.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr_notifier.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-mr_notifier.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/mr_notifier.c' object='prov/gni/test/prov_gni_test_gnitest-mr_notifier.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-mr_notifier.obj `if test -f 'prov/gni/test/mr_notifier.c'; then $(CYGPATH_W) 'prov/gni/test/mr_notifier.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/mr_notifier.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-nic.o: prov/gni/test/nic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-nic.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-nic.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-nic.o `test -f 'prov/gni/test/nic.c' || echo '$(srcdir)/'`prov/gni/test/nic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-nic.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-nic.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/nic.c' object='prov/gni/test/prov_gni_test_gnitest-nic.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-nic.o `test -f 'prov/gni/test/nic.c' || echo '$(srcdir)/'`prov/gni/test/nic.c
+
+prov/gni/test/prov_gni_test_gnitest-nic.obj: prov/gni/test/nic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-nic.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-nic.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-nic.obj `if test -f 'prov/gni/test/nic.c'; then $(CYGPATH_W) 'prov/gni/test/nic.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/nic.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-nic.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-nic.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/nic.c' object='prov/gni/test/prov_gni_test_gnitest-nic.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-nic.obj `if test -f 'prov/gni/test/nic.c'; then $(CYGPATH_W) 'prov/gni/test/nic.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/nic.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-pmi_utils.o: prov/gni/test/pmi_utils.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-pmi_utils.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-pmi_utils.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-pmi_utils.o `test -f 'prov/gni/test/pmi_utils.c' || echo '$(srcdir)/'`prov/gni/test/pmi_utils.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-pmi_utils.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-pmi_utils.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/pmi_utils.c' object='prov/gni/test/prov_gni_test_gnitest-pmi_utils.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-pmi_utils.o `test -f 'prov/gni/test/pmi_utils.c' || echo '$(srcdir)/'`prov/gni/test/pmi_utils.c
+
+prov/gni/test/prov_gni_test_gnitest-pmi_utils.obj: prov/gni/test/pmi_utils.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-pmi_utils.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-pmi_utils.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-pmi_utils.obj `if test -f 'prov/gni/test/pmi_utils.c'; then $(CYGPATH_W) 'prov/gni/test/pmi_utils.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/pmi_utils.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-pmi_utils.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-pmi_utils.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/pmi_utils.c' object='prov/gni/test/prov_gni_test_gnitest-pmi_utils.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-pmi_utils.obj `if test -f 'prov/gni/test/pmi_utils.c'; then $(CYGPATH_W) 'prov/gni/test/pmi_utils.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/pmi_utils.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-queue.o: prov/gni/test/queue.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-queue.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-queue.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-queue.o `test -f 'prov/gni/test/queue.c' || echo '$(srcdir)/'`prov/gni/test/queue.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-queue.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-queue.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/queue.c' object='prov/gni/test/prov_gni_test_gnitest-queue.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-queue.o `test -f 'prov/gni/test/queue.c' || echo '$(srcdir)/'`prov/gni/test/queue.c
+
+prov/gni/test/prov_gni_test_gnitest-queue.obj: prov/gni/test/queue.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-queue.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-queue.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-queue.obj `if test -f 'prov/gni/test/queue.c'; then $(CYGPATH_W) 'prov/gni/test/queue.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/queue.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-queue.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-queue.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/queue.c' object='prov/gni/test/prov_gni_test_gnitest-queue.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-queue.obj `if test -f 'prov/gni/test/queue.c'; then $(CYGPATH_W) 'prov/gni/test/queue.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/queue.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-rdm_atomic.o: prov/gni/test/rdm_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-rdm_atomic.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_atomic.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-rdm_atomic.o `test -f 'prov/gni/test/rdm_atomic.c' || echo '$(srcdir)/'`prov/gni/test/rdm_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_atomic.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_atomic.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/rdm_atomic.c' object='prov/gni/test/prov_gni_test_gnitest-rdm_atomic.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-rdm_atomic.o `test -f 'prov/gni/test/rdm_atomic.c' || echo '$(srcdir)/'`prov/gni/test/rdm_atomic.c
+
+prov/gni/test/prov_gni_test_gnitest-rdm_atomic.obj: prov/gni/test/rdm_atomic.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-rdm_atomic.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_atomic.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-rdm_atomic.obj `if test -f 'prov/gni/test/rdm_atomic.c'; then $(CYGPATH_W) 'prov/gni/test/rdm_atomic.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/rdm_atomic.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_atomic.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_atomic.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/rdm_atomic.c' object='prov/gni/test/prov_gni_test_gnitest-rdm_atomic.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-rdm_atomic.obj `if test -f 'prov/gni/test/rdm_atomic.c'; then $(CYGPATH_W) 'prov/gni/test/rdm_atomic.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/rdm_atomic.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.o: prov/gni/test/rdm_fi_pcd_trecv_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.o `test -f 'prov/gni/test/rdm_fi_pcd_trecv_msg.c' || echo '$(srcdir)/'`prov/gni/test/rdm_fi_pcd_trecv_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/rdm_fi_pcd_trecv_msg.c' object='prov/gni/test/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.o `test -f 'prov/gni/test/rdm_fi_pcd_trecv_msg.c' || echo '$(srcdir)/'`prov/gni/test/rdm_fi_pcd_trecv_msg.c
+
+prov/gni/test/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.obj: prov/gni/test/rdm_fi_pcd_trecv_msg.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.obj `if test -f 'prov/gni/test/rdm_fi_pcd_trecv_msg.c'; then $(CYGPATH_W) 'prov/gni/test/rdm_fi_pcd_trecv_msg.c'; else $(CYGPATH_W) '$( [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/rdm_fi_pcd_trecv_msg.c' object='prov/gni/test/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-rdm_fi_pcd_trecv_msg.obj `if test -f 'prov/gni/test/rdm_fi_pcd_trecv_msg.c'; then $(CYGPATH_W) 'prov/gni/test/rdm_fi_pcd_trecv_msg.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/rdm_fi_pcd_trecv_msg.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-rdm_dgram_rma.o: prov/gni/test/rdm_dgram_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-rdm_dgram_rma.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_dgram_rma.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-rdm_dgram_rma.o `test -f 'prov/gni/test/rdm_dgram_rma.c' || echo '$(srcdir)/'`prov/gni/test/rdm_dgram_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_dgram_rma.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_dgram_rma.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/rdm_dgram_rma.c' object='prov/gni/test/prov_gni_test_gnitest-rdm_dgram_rma.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-rdm_dgram_rma.o `test -f 'prov/gni/test/rdm_dgram_rma.c' || echo '$(srcdir)/'`prov/gni/test/rdm_dgram_rma.c
+
+prov/gni/test/prov_gni_test_gnitest-rdm_dgram_rma.obj: prov/gni/test/rdm_dgram_rma.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-rdm_dgram_rma.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_dgram_rma.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-rdm_dgram_rma.obj `if test -f 'prov/gni/test/rdm_dgram_rma.c'; then $(CYGPATH_W) 'prov/gni/test/rdm_dgram_rma.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/rdm_dgram_rma [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_dgram_rma.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_dgram_rma.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/rdm_dgram_rma.c' object='prov/gni/test/prov_gni_test_gnitest-rdm_dgram_rma.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-rdm_dgram_rma.obj `if test -f 'prov/gni/test/rdm_dgram_rma.c'; then $(CYGPATH_W) 'prov/gni/test/rdm_dgram_rma.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/rdm_dgram_rma.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-rdm_rx_overrun.o: prov/gni/test/rdm_rx_overrun.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-rdm_rx_overrun.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_rx_overrun.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-rdm_rx_overrun.o `test -f 'prov/gni/test/rdm_rx_overrun.c' || echo '$(srcdir)/'`prov/gni/test/rdm_rx_overrun.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_rx_overrun.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_rx_overrun.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/rdm_rx_overrun.c' object='prov/gni/test/prov_gni_test_gnitest-rdm_rx_overrun.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-rdm_rx_overrun.o `test -f 'prov/gni/test/rdm_rx_overrun.c' || echo '$(srcdir)/'`prov/gni/test/rdm_rx_overrun.c
+
+prov/gni/test/prov_gni_test_gnitest-rdm_rx_overrun.obj: prov/gni/test/rdm_rx_overrun.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-rdm_rx_overrun.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_rx_overrun.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-rdm_rx_overrun.obj `if test -f 'prov/gni/test/rdm_rx_overrun.c'; then $(CYGPATH_W) 'prov/gni/test/rdm_rx_overrun.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/rdm_rx_o [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_rx_overrun.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_rx_overrun.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/rdm_rx_overrun.c' object='prov/gni/test/prov_gni_test_gnitest-rdm_rx_overrun.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-rdm_rx_overrun.obj `if test -f 'prov/gni/test/rdm_rx_overrun.c'; then $(CYGPATH_W) 'prov/gni/test/rdm_rx_overrun.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/rdm_rx_overrun.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-rdm_sr.o: prov/gni/test/rdm_sr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-rdm_sr.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_sr.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-rdm_sr.o `test -f 'prov/gni/test/rdm_sr.c' || echo '$(srcdir)/'`prov/gni/test/rdm_sr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_sr.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_sr.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/rdm_sr.c' object='prov/gni/test/prov_gni_test_gnitest-rdm_sr.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-rdm_sr.o `test -f 'prov/gni/test/rdm_sr.c' || echo '$(srcdir)/'`prov/gni/test/rdm_sr.c
+
+prov/gni/test/prov_gni_test_gnitest-rdm_sr.obj: prov/gni/test/rdm_sr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-rdm_sr.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_sr.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-rdm_sr.obj `if test -f 'prov/gni/test/rdm_sr.c'; then $(CYGPATH_W) 'prov/gni/test/rdm_sr.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/rdm_sr.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_sr.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_sr.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/rdm_sr.c' object='prov/gni/test/prov_gni_test_gnitest-rdm_sr.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-rdm_sr.obj `if test -f 'prov/gni/test/rdm_sr.c'; then $(CYGPATH_W) 'prov/gni/test/rdm_sr.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/rdm_sr.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-rdm_tagged_sr.o: prov/gni/test/rdm_tagged_sr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-rdm_tagged_sr.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_tagged_sr.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-rdm_tagged_sr.o `test -f 'prov/gni/test/rdm_tagged_sr.c' || echo '$(srcdir)/'`prov/gni/test/rdm_tagged_sr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_tagged_sr.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_tagged_sr.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/rdm_tagged_sr.c' object='prov/gni/test/prov_gni_test_gnitest-rdm_tagged_sr.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-rdm_tagged_sr.o `test -f 'prov/gni/test/rdm_tagged_sr.c' || echo '$(srcdir)/'`prov/gni/test/rdm_tagged_sr.c
+
+prov/gni/test/prov_gni_test_gnitest-rdm_tagged_sr.obj: prov/gni/test/rdm_tagged_sr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-rdm_tagged_sr.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_tagged_sr.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-rdm_tagged_sr.obj `if test -f 'prov/gni/test/rdm_tagged_sr.c'; then $(CYGPATH_W) 'prov/gni/test/rdm_tagged_sr.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/rdm_tagged_sr [...]
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_tagged_sr.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-rdm_tagged_sr.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/rdm_tagged_sr.c' object='prov/gni/test/prov_gni_test_gnitest-rdm_tagged_sr.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-rdm_tagged_sr.obj `if test -f 'prov/gni/test/rdm_tagged_sr.c'; then $(CYGPATH_W) 'prov/gni/test/rdm_tagged_sr.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/rdm_tagged_sr.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-tags.o: prov/gni/test/tags.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-tags.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-tags.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-tags.o `test -f 'prov/gni/test/tags.c' || echo '$(srcdir)/'`prov/gni/test/tags.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-tags.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-tags.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/tags.c' object='prov/gni/test/prov_gni_test_gnitest-tags.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-tags.o `test -f 'prov/gni/test/tags.c' || echo '$(srcdir)/'`prov/gni/test/tags.c
+
+prov/gni/test/prov_gni_test_gnitest-tags.obj: prov/gni/test/tags.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-tags.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-tags.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-tags.obj `if test -f 'prov/gni/test/tags.c'; then $(CYGPATH_W) 'prov/gni/test/tags.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/tags.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-tags.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-tags.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/tags.c' object='prov/gni/test/prov_gni_test_gnitest-tags.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-tags.obj `if test -f 'prov/gni/test/tags.c'; then $(CYGPATH_W) 'prov/gni/test/tags.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/tags.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-api.o: prov/gni/test/api.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-api.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-api.o `test -f 'prov/gni/test/api.c' || echo '$(srcdir)/'`prov/gni/test/api.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/api.c' object='prov/gni/test/prov_gni_test_gnitest-api.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-api.o `test -f 'prov/gni/test/api.c' || echo '$(srcdir)/'`prov/gni/test/api.c
+
+prov/gni/test/prov_gni_test_gnitest-api.obj: prov/gni/test/api.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-api.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-api.obj `if test -f 'prov/gni/test/api.c'; then $(CYGPATH_W) 'prov/gni/test/api.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/api.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/api.c' object='prov/gni/test/prov_gni_test_gnitest-api.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-api.obj `if test -f 'prov/gni/test/api.c'; then $(CYGPATH_W) 'prov/gni/test/api.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/api.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-api_cq.o: prov/gni/test/api_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-api_cq.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cq.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-api_cq.o `test -f 'prov/gni/test/api_cq.c' || echo '$(srcdir)/'`prov/gni/test/api_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cq.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cq.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/api_cq.c' object='prov/gni/test/prov_gni_test_gnitest-api_cq.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-api_cq.o `test -f 'prov/gni/test/api_cq.c' || echo '$(srcdir)/'`prov/gni/test/api_cq.c
+
+prov/gni/test/prov_gni_test_gnitest-api_cq.obj: prov/gni/test/api_cq.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-api_cq.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cq.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-api_cq.obj `if test -f 'prov/gni/test/api_cq.c'; then $(CYGPATH_W) 'prov/gni/test/api_cq.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/api_cq.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cq.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cq.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/api_cq.c' object='prov/gni/test/prov_gni_test_gnitest-api_cq.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-api_cq.obj `if test -f 'prov/gni/test/api_cq.c'; then $(CYGPATH_W) 'prov/gni/test/api_cq.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/api_cq.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-api_cntr.o: prov/gni/test/api_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-api_cntr.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cntr.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-api_cntr.o `test -f 'prov/gni/test/api_cntr.c' || echo '$(srcdir)/'`prov/gni/test/api_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cntr.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cntr.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/api_cntr.c' object='prov/gni/test/prov_gni_test_gnitest-api_cntr.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-api_cntr.o `test -f 'prov/gni/test/api_cntr.c' || echo '$(srcdir)/'`prov/gni/test/api_cntr.c
+
+prov/gni/test/prov_gni_test_gnitest-api_cntr.obj: prov/gni/test/api_cntr.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-api_cntr.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cntr.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-api_cntr.obj `if test -f 'prov/gni/test/api_cntr.c'; then $(CYGPATH_W) 'prov/gni/test/api_cntr.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/api_cntr.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cntr.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-api_cntr.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/api_cntr.c' object='prov/gni/test/prov_gni_test_gnitest-api_cntr.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-api_cntr.obj `if test -f 'prov/gni/test/api_cntr.c'; then $(CYGPATH_W) 'prov/gni/test/api_cntr.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/api_cntr.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-utils.o: prov/gni/test/utils.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-utils.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-utils.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-utils.o `test -f 'prov/gni/test/utils.c' || echo '$(srcdir)/'`prov/gni/test/utils.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-utils.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-utils.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/utils.c' object='prov/gni/test/prov_gni_test_gnitest-utils.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-utils.o `test -f 'prov/gni/test/utils.c' || echo '$(srcdir)/'`prov/gni/test/utils.c
+
+prov/gni/test/prov_gni_test_gnitest-utils.obj: prov/gni/test/utils.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-utils.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-utils.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-utils.obj `if test -f 'prov/gni/test/utils.c'; then $(CYGPATH_W) 'prov/gni/test/utils.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/utils.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-utils.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-utils.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/utils.c' object='prov/gni/test/prov_gni_test_gnitest-utils.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-utils.obj `if test -f 'prov/gni/test/utils.c'; then $(CYGPATH_W) 'prov/gni/test/utils.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/utils.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-vc.o: prov/gni/test/vc.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-vc.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-vc.o `test -f 'prov/gni/test/vc.c' || echo '$(srcdir)/'`prov/gni/test/vc.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/vc.c' object='prov/gni/test/prov_gni_test_gnitest-vc.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-vc.o `test -f 'prov/gni/test/vc.c' || echo '$(srcdir)/'`prov/gni/test/vc.c
+
+prov/gni/test/prov_gni_test_gnitest-vc.obj: prov/gni/test/vc.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-vc.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-vc.obj `if test -f 'prov/gni/test/vc.c'; then $(CYGPATH_W) 'prov/gni/test/vc.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/vc.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/vc.c' object='prov/gni/test/prov_gni_test_gnitest-vc.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-vc.obj `if test -f 'prov/gni/test/vc.c'; then $(CYGPATH_W) 'prov/gni/test/vc.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/vc.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-vc_lookup.o: prov/gni/test/vc_lookup.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-vc_lookup.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc_lookup.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-vc_lookup.o `test -f 'prov/gni/test/vc_lookup.c' || echo '$(srcdir)/'`prov/gni/test/vc_lookup.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc_lookup.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc_lookup.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/vc_lookup.c' object='prov/gni/test/prov_gni_test_gnitest-vc_lookup.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-vc_lookup.o `test -f 'prov/gni/test/vc_lookup.c' || echo '$(srcdir)/'`prov/gni/test/vc_lookup.c
+
+prov/gni/test/prov_gni_test_gnitest-vc_lookup.obj: prov/gni/test/vc_lookup.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-vc_lookup.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc_lookup.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-vc_lookup.obj `if test -f 'prov/gni/test/vc_lookup.c'; then $(CYGPATH_W) 'prov/gni/test/vc_lookup.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/vc_lookup.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc_lookup.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vc_lookup.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/vc_lookup.c' object='prov/gni/test/prov_gni_test_gnitest-vc_lookup.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-vc_lookup.obj `if test -f 'prov/gni/test/vc_lookup.c'; then $(CYGPATH_W) 'prov/gni/test/vc_lookup.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/vc_lookup.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-vector.o: prov/gni/test/vector.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-vector.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vector.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-vector.o `test -f 'prov/gni/test/vector.c' || echo '$(srcdir)/'`prov/gni/test/vector.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vector.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vector.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/vector.c' object='prov/gni/test/prov_gni_test_gnitest-vector.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-vector.o `test -f 'prov/gni/test/vector.c' || echo '$(srcdir)/'`prov/gni/test/vector.c
+
+prov/gni/test/prov_gni_test_gnitest-vector.obj: prov/gni/test/vector.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-vector.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vector.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-vector.obj `if test -f 'prov/gni/test/vector.c'; then $(CYGPATH_W) 'prov/gni/test/vector.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/vector.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vector.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-vector.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/vector.c' object='prov/gni/test/prov_gni_test_gnitest-vector.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-vector.obj `if test -f 'prov/gni/test/vector.c'; then $(CYGPATH_W) 'prov/gni/test/vector.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/vector.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-wait.o: prov/gni/test/wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-wait.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-wait.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-wait.o `test -f 'prov/gni/test/wait.c' || echo '$(srcdir)/'`prov/gni/test/wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-wait.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-wait.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/wait.c' object='prov/gni/test/prov_gni_test_gnitest-wait.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-wait.o `test -f 'prov/gni/test/wait.c' || echo '$(srcdir)/'`prov/gni/test/wait.c
+
+prov/gni/test/prov_gni_test_gnitest-wait.obj: prov/gni/test/wait.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-wait.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-wait.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-wait.obj `if test -f 'prov/gni/test/wait.c'; then $(CYGPATH_W) 'prov/gni/test/wait.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/wait.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-wait.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-wait.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/wait.c' object='prov/gni/test/prov_gni_test_gnitest-wait.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-wait.obj `if test -f 'prov/gni/test/wait.c'; then $(CYGPATH_W) 'prov/gni/test/wait.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/wait.c'; fi`
+
+prov/gni/test/prov_gni_test_gnitest-common.o: prov/gni/test/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-common.o -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-common.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-common.o `test -f 'prov/gni/test/common.c' || echo '$(srcdir)/'`prov/gni/test/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-common.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-common.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/common.c' object='prov/gni/test/prov_gni_test_gnitest-common.o' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-common.o `test -f 'prov/gni/test/common.c' || echo '$(srcdir)/'`prov/gni/test/common.c
+
+prov/gni/test/prov_gni_test_gnitest-common.obj: prov/gni/test/common.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT prov/gni/test/prov_gni_test_gnitest-common.obj -MD -MP -MF prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-common.Tpo -c -o prov/gni/test/prov_gni_test_gnitest-common.obj `if test -f 'prov/gni/test/common.c'; then $(CYGPATH_W) 'prov/gni/test/common.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/common.c'; fi`
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-common.Tpo prov/gni/test/$(DEPDIR)/prov_gni_test_gnitest-common.Po
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/gni/test/common.c' object='prov/gni/test/prov_gni_test_gnitest-common.obj' libtool=no @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(prov_gni_test_gnitest_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o prov/gni/test/prov_gni_test_gnitest-common.obj `if test -f 'prov/gni/test/common.c'; then $(CYGPATH_W) 'prov/gni/test/common.c'; else $(CYGPATH_W) '$(srcdir)/prov/gni/test/common.c'; fi`
+
+mostlyclean-libtool:
+ -rm -f *.lo
+
+clean-libtool:
+ -rm -rf .libs _libs
+ -rm -rf prov/gni/src/.libs prov/gni/src/_libs
+ -rm -rf prov/gni/test/.libs prov/gni/test/_libs
+ -rm -rf prov/mxm/src/.libs prov/mxm/src/_libs
+ -rm -rf prov/psm/src/.libs prov/psm/src/_libs
+ -rm -rf prov/psm2/src/.libs prov/psm2/src/_libs
+ -rm -rf prov/rxd/src/.libs prov/rxd/src/_libs
+ -rm -rf prov/rxm/src/.libs prov/rxm/src/_libs
+ -rm -rf prov/sockets/src/.libs prov/sockets/src/_libs
+ -rm -rf prov/udp/src/.libs prov/udp/src/_libs
+ -rm -rf prov/usnic/src/.libs prov/usnic/src/_libs
+ -rm -rf prov/usnic/src/usnic_direct/.libs prov/usnic/src/usnic_direct/_libs
+ -rm -rf prov/util/src/.libs prov/util/src/_libs
+ -rm -rf prov/verbs/src/.libs prov/verbs/src/_libs
+ -rm -rf prov/verbs/src/ep_rdm/.libs prov/verbs/src/ep_rdm/_libs
+ -rm -rf src/.libs src/_libs
+ -rm -rf src/osx/.libs src/osx/_libs
+ -rm -rf src/unix/.libs src/unix/_libs
+ -rm -rf util/.libs util/_libs
+
+distclean-libtool:
+ -rm -f libtool config.lt
+install-man1: $(man_MANS)
+ @$(NORMAL_INSTALL)
+ test -z "$(man1dir)" || $(MKDIR_P) "$(DESTDIR)$(man1dir)"
+ @list=''; test -n "$(man1dir)" || exit 0; \
+ { for i in $$list; do echo "$$i"; done; \
+ l2='$(man_MANS)'; for i in $$l2; do echo "$$i"; done | \
+ sed -n '/\.1[a-z]*$$/p'; \
+ } | while read p; do \
+ if test -f $$p; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; echo "$$p"; \
+ done | \
+ sed -e 'n;s,.*/,,;p;h;s,.*\.,,;s,^[^1][0-9a-z]*$$,1,;x' \
+ -e 's,\.[0-9a-z]*$$,,;$(transform);G;s,\n,.,' | \
+ sed 'N;N;s,\n, ,g' | { \
+ list=; while read file base inst; do \
+ if test "$$base" = "$$inst"; then list="$$list $$file"; else \
+ echo " $(INSTALL_DATA) '$$file' '$(DESTDIR)$(man1dir)/$$inst'"; \
+ $(INSTALL_DATA) "$$file" "$(DESTDIR)$(man1dir)/$$inst" || exit $$?; \
+ fi; \
+ done; \
+ for i in $$list; do echo "$$i"; done | $(am__base_list) | \
+ while read files; do \
+ test -z "$$files" || { \
+ echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(man1dir)'"; \
+ $(INSTALL_DATA) $$files "$(DESTDIR)$(man1dir)" || exit $$?; }; \
+ done; }
+
+uninstall-man1:
+ @$(NORMAL_UNINSTALL)
+ @list=''; test -n "$(man1dir)" || exit 0; \
+ files=`{ for i in $$list; do echo "$$i"; done; \
+ l2='$(man_MANS)'; for i in $$l2; do echo "$$i"; done | \
+ sed -n '/\.1[a-z]*$$/p'; \
+ } | sed -e 's,.*/,,;h;s,.*\.,,;s,^[^1][0-9a-z]*$$,1,;x' \
+ -e 's,\.[0-9a-z]*$$,,;$(transform);G;s,\n,.,'`; \
+ test -z "$$files" || { \
+ echo " ( cd '$(DESTDIR)$(man1dir)' && rm -f" $$files ")"; \
+ cd "$(DESTDIR)$(man1dir)" && rm -f $$files; }
+install-man3: $(man_MANS)
+ @$(NORMAL_INSTALL)
+ test -z "$(man3dir)" || $(MKDIR_P) "$(DESTDIR)$(man3dir)"
+ @list=''; test -n "$(man3dir)" || exit 0; \
+ { for i in $$list; do echo "$$i"; done; \
+ l2='$(man_MANS)'; for i in $$l2; do echo "$$i"; done | \
+ sed -n '/\.3[a-z]*$$/p'; \
+ } | while read p; do \
+ if test -f $$p; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; echo "$$p"; \
+ done | \
+ sed -e 'n;s,.*/,,;p;h;s,.*\.,,;s,^[^3][0-9a-z]*$$,3,;x' \
+ -e 's,\.[0-9a-z]*$$,,;$(transform);G;s,\n,.,' | \
+ sed 'N;N;s,\n, ,g' | { \
+ list=; while read file base inst; do \
+ if test "$$base" = "$$inst"; then list="$$list $$file"; else \
+ echo " $(INSTALL_DATA) '$$file' '$(DESTDIR)$(man3dir)/$$inst'"; \
+ $(INSTALL_DATA) "$$file" "$(DESTDIR)$(man3dir)/$$inst" || exit $$?; \
+ fi; \
+ done; \
+ for i in $$list; do echo "$$i"; done | $(am__base_list) | \
+ while read files; do \
+ test -z "$$files" || { \
+ echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(man3dir)'"; \
+ $(INSTALL_DATA) $$files "$(DESTDIR)$(man3dir)" || exit $$?; }; \
+ done; }
+
+uninstall-man3:
+ @$(NORMAL_UNINSTALL)
+ @list=''; test -n "$(man3dir)" || exit 0; \
+ files=`{ for i in $$list; do echo "$$i"; done; \
+ l2='$(man_MANS)'; for i in $$l2; do echo "$$i"; done | \
+ sed -n '/\.3[a-z]*$$/p'; \
+ } | sed -e 's,.*/,,;h;s,.*\.,,;s,^[^3][0-9a-z]*$$,3,;x' \
+ -e 's,\.[0-9a-z]*$$,,;$(transform);G;s,\n,.,'`; \
+ test -z "$$files" || { \
+ echo " ( cd '$(DESTDIR)$(man3dir)' && rm -f" $$files ")"; \
+ cd "$(DESTDIR)$(man3dir)" && rm -f $$files; }
+install-man7: $(man_MANS)
+ @$(NORMAL_INSTALL)
+ test -z "$(man7dir)" || $(MKDIR_P) "$(DESTDIR)$(man7dir)"
+ @list=''; test -n "$(man7dir)" || exit 0; \
+ { for i in $$list; do echo "$$i"; done; \
+ l2='$(man_MANS)'; for i in $$l2; do echo "$$i"; done | \
+ sed -n '/\.7[a-z]*$$/p'; \
+ } | while read p; do \
+ if test -f $$p; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; echo "$$p"; \
+ done | \
+ sed -e 'n;s,.*/,,;p;h;s,.*\.,,;s,^[^7][0-9a-z]*$$,7,;x' \
+ -e 's,\.[0-9a-z]*$$,,;$(transform);G;s,\n,.,' | \
+ sed 'N;N;s,\n, ,g' | { \
+ list=; while read file base inst; do \
+ if test "$$base" = "$$inst"; then list="$$list $$file"; else \
+ echo " $(INSTALL_DATA) '$$file' '$(DESTDIR)$(man7dir)/$$inst'"; \
+ $(INSTALL_DATA) "$$file" "$(DESTDIR)$(man7dir)/$$inst" || exit $$?; \
+ fi; \
+ done; \
+ for i in $$list; do echo "$$i"; done | $(am__base_list) | \
+ while read files; do \
+ test -z "$$files" || { \
+ echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(man7dir)'"; \
+ $(INSTALL_DATA) $$files "$(DESTDIR)$(man7dir)" || exit $$?; }; \
+ done; }
+
+uninstall-man7:
+ @$(NORMAL_UNINSTALL)
+ @list=''; test -n "$(man7dir)" || exit 0; \
+ files=`{ for i in $$list; do echo "$$i"; done; \
+ l2='$(man_MANS)'; for i in $$l2; do echo "$$i"; done | \
+ sed -n '/\.7[a-z]*$$/p'; \
+ } | sed -e 's,.*/,,;h;s,.*\.,,;s,^[^7][0-9a-z]*$$,7,;x' \
+ -e 's,\.[0-9a-z]*$$,,;$(transform);G;s,\n,.,'`; \
+ test -z "$$files" || { \
+ echo " ( cd '$(DESTDIR)$(man7dir)' && rm -f" $$files ")"; \
+ cd "$(DESTDIR)$(man7dir)" && rm -f $$files; }
+install-pkgconfigDATA: $(pkgconfig_DATA)
+ @$(NORMAL_INSTALL)
+ test -z "$(pkgconfigdir)" || $(MKDIR_P) "$(DESTDIR)$(pkgconfigdir)"
+ @list='$(pkgconfig_DATA)'; test -n "$(pkgconfigdir)" || list=; \
+ for p in $$list; do \
+ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; \
+ done | $(am__base_list) | \
+ while read files; do \
+ echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(pkgconfigdir)'"; \
+ $(INSTALL_DATA) $$files "$(DESTDIR)$(pkgconfigdir)" || exit $$?; \
+ done
+
+uninstall-pkgconfigDATA:
+ @$(NORMAL_UNINSTALL)
+ @list='$(pkgconfig_DATA)'; test -n "$(pkgconfigdir)" || list=; \
+ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+ test -n "$$files" || exit 0; \
+ echo " ( cd '$(DESTDIR)$(pkgconfigdir)' && rm -f" $$files ")"; \
+ cd "$(DESTDIR)$(pkgconfigdir)" && rm -f $$files
+install-nodist_rdmaincludeHEADERS: $(nodist_rdmainclude_HEADERS)
+ @$(NORMAL_INSTALL)
+ test -z "$(rdmaincludedir)" || $(MKDIR_P) "$(DESTDIR)$(rdmaincludedir)"
+ @list='$(nodist_rdmainclude_HEADERS)'; test -n "$(rdmaincludedir)" || list=; \
+ for p in $$list; do \
+ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; \
+ done | $(am__base_list) | \
+ while read files; do \
+ echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(rdmaincludedir)'"; \
+ $(INSTALL_HEADER) $$files "$(DESTDIR)$(rdmaincludedir)" || exit $$?; \
+ done
+
+uninstall-nodist_rdmaincludeHEADERS:
+ @$(NORMAL_UNINSTALL)
+ @list='$(nodist_rdmainclude_HEADERS)'; test -n "$(rdmaincludedir)" || list=; \
+ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+ test -n "$$files" || exit 0; \
+ echo " ( cd '$(DESTDIR)$(rdmaincludedir)' && rm -f" $$files ")"; \
+ cd "$(DESTDIR)$(rdmaincludedir)" && rm -f $$files
+install-rdmaincludeHEADERS: $(rdmainclude_HEADERS)
+ @$(NORMAL_INSTALL)
+ test -z "$(rdmaincludedir)" || $(MKDIR_P) "$(DESTDIR)$(rdmaincludedir)"
+ @list='$(rdmainclude_HEADERS)'; test -n "$(rdmaincludedir)" || list=; \
+ for p in $$list; do \
+ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; \
+ done | $(am__base_list) | \
+ while read files; do \
+ echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(rdmaincludedir)'"; \
+ $(INSTALL_HEADER) $$files "$(DESTDIR)$(rdmaincludedir)" || exit $$?; \
+ done
+
+uninstall-rdmaincludeHEADERS:
+ @$(NORMAL_UNINSTALL)
+ @list='$(rdmainclude_HEADERS)'; test -n "$(rdmaincludedir)" || list=; \
+ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+ test -n "$$files" || exit 0; \
+ echo " ( cd '$(DESTDIR)$(rdmaincludedir)' && rm -f" $$files ")"; \
+ cd "$(DESTDIR)$(rdmaincludedir)" && rm -f $$files
+
+ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
+ list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
+ unique=`for i in $$list; do \
+ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+ done | \
+ $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
+ END { if (nonempty) { for (i in files) print i; }; }'`; \
+ mkid -fID $$unique
+tags: TAGS
+
+TAGS: $(HEADERS) $(SOURCES) config.h.in $(TAGS_DEPENDENCIES) \
+ $(TAGS_FILES) $(LISP)
+ set x; \
+ here=`pwd`; \
+ list='$(SOURCES) $(HEADERS) config.h.in $(LISP) $(TAGS_FILES)'; \
+ unique=`for i in $$list; do \
+ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+ done | \
+ $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
+ END { if (nonempty) { for (i in files) print i; }; }'`; \
+ shift; \
+ if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+ test -n "$$unique" || unique=$$empty_fix; \
+ if test $$# -gt 0; then \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ "$$@" $$unique; \
+ else \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ $$unique; \
+ fi; \
+ fi
+ctags: CTAGS
+CTAGS: $(HEADERS) $(SOURCES) config.h.in $(TAGS_DEPENDENCIES) \
+ $(TAGS_FILES) $(LISP)
+ list='$(SOURCES) $(HEADERS) config.h.in $(LISP) $(TAGS_FILES)'; \
+ unique=`for i in $$list; do \
+ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+ done | \
+ $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
+ END { if (nonempty) { for (i in files) print i; }; }'`; \
+ test -z "$(CTAGS_ARGS)$$unique" \
+ || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+ $$unique
+
+GTAGS:
+ here=`$(am__cd) $(top_builddir) && pwd` \
+ && $(am__cd) $(top_srcdir) \
+ && gtags -i $(GTAGS_ARGS) "$$here"
+
+distclean-tags:
+ -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+# To be appended to the command running the test. Handle the stdout
+# and stderr redirection, and catch the exit status.
+am__check_post = \
+>$@-t 2>&1; \
+estatus=$$?; \
+if test -n '$(DISABLE_HARD_ERRORS)' \
+ && test $$estatus -eq 99; then \
+ estatus=1; \
+fi; \
+TERM=$$__SAVED_TERM; export TERM; \
+$(am__tty_colors); \
+xfailed=PASS; \
+case " $(XFAIL_TESTS) " in \
+ *[\ \ ]$$f[\ \ ]* | *[\ \ ]$$dir$$f[\ \ ]*) \
+ xfailed=XFAIL;; \
+esac; \
+case $$estatus:$$xfailed in \
+ 0:XFAIL) col=$$red; res=XPASS;; \
+ 0:*) col=$$grn; res=PASS ;; \
+ 77:*) col=$$blu; res=SKIP ;; \
+ 99:*) col=$$red; res=FAIL ;; \
+ *:XFAIL) col=$$lgn; res=XFAIL;; \
+ *:*) col=$$red; res=FAIL ;; \
+esac; \
+echo "$${col}$$res$${std}: $$f"; \
+echo "$$res: $$f (exit: $$estatus)" | \
+ $(am__rst_section) >$@; \
+cat $@-t >>$@; \
+rm -f $@-t
+
+$(TEST_SUITE_LOG): $(TEST_LOGS)
+ @$(am__sh_e_setup); \
+ list='$(TEST_LOGS)'; \
+ results=`for f in $$list; do \
+ read line < $$f && echo "$$line" || echo FAIL; \
+ done`; \
+ all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[ ]*//'`; \
+ fail=`echo "$$results" | grep -c '^FAIL'`; \
+ pass=`echo "$$results" | grep -c '^PASS'`; \
+ skip=`echo "$$results" | grep -c '^SKIP'`; \
+ xfail=`echo "$$results" | grep -c '^XFAIL'`; \
+ xpass=`echo "$$results" | grep -c '^XPASS'`; \
+ failures=`expr $$fail + $$xpass`; \
+ all=`expr $$all - $$skip`; \
+ if test "$$all" -eq 1; then tests=test; All=; \
+ else tests=tests; All="All "; fi; \
+ case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \
+ fail=0:xpass=0:xfail=0) \
+ msg="$$All$$all $$tests passed. "; \
+ exit=true;; \
+ fail=0:xpass=0:xfail=*) \
+ msg="$$All$$all $$tests behaved as expected"; \
+ if test "$$xfail" -eq 1; then xfailures=failure; \
+ else xfailures=failures; fi; \
+ msg="$$msg ($$xfail expected $$xfailures). "; \
+ exit=true;; \
+ fail=*:xpass=0:xfail=*) \
+ msg="$$fail of $$all $$tests failed. "; \
+ exit=false;; \
+ fail=*:xpass=*:xfail=*) \
+ msg="$$failures of $$all $$tests did not behave as expected"; \
+ if test "$$xpass" -eq 1; then xpasses=pass; \
+ else xpasses=passes; fi; \
+ msg="$$msg ($$xpass unexpected $$xpasses). "; \
+ exit=false;; \
+ *) \
+ echo >&2 "incorrect case"; exit 4;; \
+ esac; \
+ if test "$$skip" -ne 0; then \
+ if test "$$skip" -eq 1; then \
+ msg="$$msg($$skip test was not run). "; \
+ else \
+ msg="$$msg($$skip tests were not run). "; \
+ fi; \
+ fi; \
+ { \
+ echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
+ $(am__rst_title); \
+ echo "$$msg"; \
+ echo; \
+ echo ".. contents:: :depth: 2"; \
+ echo; \
+ for f in $$list; do \
+ read line < $$f; \
+ case $$line in \
+ PASS:*|XFAIL:*);; \
+ *) echo; cat $$f;; \
+ esac; \
+ done; \
+ } >$(TEST_SUITE_LOG).tmp; \
+ mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
+ if test "$$failures" -ne 0; then \
+ msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \
+ if test -n "$(PACKAGE_BUGREPORT)"; then \
+ msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \
+ fi; \
+ fi; \
+ test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \
+ $(am__tty_colors); \
+ if $$exit; then \
+ echo $(ECHO_N) "$$grn$(ECHO_C)"; \
+ else \
+ echo $(ECHO_N) "$$red$(ECHO_C)"; \
+ fi; \
+ echo "$$msg" | $(am__text_box); \
+ echo $(ECHO_N) "$$std$(ECHO_C)"; \
+ $$exit
+
+# Run all the tests.
+check-TESTS:
+ @list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list
+ @test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
+ @list='$(TEST_LOGS)'; \
+ list=`for f in $$list; do \
+ test .log = $$f || echo $$f; \
+ done | tr '\012\015' ' '`; \
+ $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) TEST_LOGS="$$list"
+
+.log.html:
+ @list='$(RST2HTML) $$RST2HTML rst2html rst2html.py'; \
+ for r2h in $$list; do \
+ if ($$r2h --version) >/dev/null 2>&1; then \
+ R2H=$$r2h; \
+ fi; \
+ done; \
+ if test -z "$$R2H"; then \
+ echo >&2 "cannot find rst2html, cannot create $@"; \
+ exit 2; \
+ fi; \
+ $$R2H $< >$@.tmp
+ @mv $@.tmp $@
+
+# Be sure to run check first, and then to convert the result.
+# Beware of concurrent executions. Run "check" not "check-TESTS", as
+# check-SCRIPTS and other dependencies are rebuilt by the former only.
+# And expect check to fail.
+check-html:
+ @if $(MAKE) $(AM_MAKEFLAGS) check; then \
+ rv=0; else rv=$$?; \
+ fi; \
+ $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_HTML) || exit 4; \
+ exit $$rv
+recheck recheck-html:
+ @target=`echo $@ | sed 's,^re,,'`; \
+ list='$(TEST_LOGS)'; \
+ list=`for f in $$list; do \
+ test -f $$f || continue; \
+ if read line < $$f; then \
+ case $$line in FAIL*|XPASS*) echo $$f;; esac; \
+ else echo $$f; fi; \
+ done | tr '\012\015' ' '`; \
+ $(MAKE) $(AM_MAKEFLAGS) $$target AM_MAKEFLAGS='$(AM_MAKEFLAGS) TEST_LOGS="'"$$list"'"'
+util/fi_info.log: util/fi_info$(EXEEXT)
+ @p='util/fi_info$(EXEEXT)'; $(am__check_pre) $(LOG_COMPILE) "$$tst" $(am__check_post)
+.test.log:
+ @p='$<'; $(am__check_pre) $(TEST_LOG_COMPILE) "$$tst" $(am__check_post)
+ at am__EXEEXT_TRUE@.test$(EXEEXT).log:
+ at am__EXEEXT_TRUE@ @p='$<'; $(am__check_pre) $(TEST_LOG_COMPILE) "$$tst" $(am__check_post)
+
+distdir: $(DISTFILES)
+ @list='$(MANS)'; if test -n "$$list"; then \
+ list=`for p in $$list; do \
+ if test -f $$p; then d=; else d="$(srcdir)/"; fi; \
+ if test -f "$$d$$p"; then echo "$$d$$p"; else :; fi; done`; \
+ if test -n "$$list" && \
+ grep 'ab help2man is required to generate this page' $$list >/dev/null; then \
+ echo "error: found man pages containing the \`missing help2man' replacement text:" >&2; \
+ grep -l 'ab help2man is required to generate this page' $$list | sed 's/^/ /' >&2; \
+ echo " to fix them, install help2man, remove and regenerate the man pages;" >&2; \
+ echo " typically \`make maintainer-clean' will remove them" >&2; \
+ exit 1; \
+ else :; fi; \
+ else :; fi
+ $(am__remove_distdir)
+ test -d "$(distdir)" || mkdir "$(distdir)"
+ @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ list='$(DISTFILES)'; \
+ dist_files=`for file in $$list; do echo $$file; done | \
+ sed -e "s|^$$srcdirstrip/||;t" \
+ -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+ case $$dist_files in \
+ */*) $(MKDIR_P) `echo "$$dist_files" | \
+ sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+ sort -u` ;; \
+ esac; \
+ for file in $$dist_files; do \
+ if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+ if test -d $$d/$$file; then \
+ dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+ if test -d "$(distdir)/$$file"; then \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+ cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+ else \
+ test -f "$(distdir)/$$file" \
+ || cp -p $$d/$$file "$(distdir)/$$file" \
+ || exit 1; \
+ fi; \
+ done
+ $(MAKE) $(AM_MAKEFLAGS) \
+ top_distdir="$(top_distdir)" distdir="$(distdir)" \
+ dist-hook
+ -test -n "$(am__skip_mode_fix)" \
+ || find "$(distdir)" -type d ! -perm -755 \
+ -exec chmod u+rwx,go+rx {} \; -o \
+ ! -type d ! -perm -444 -links 1 -exec chmod a+r {} \; -o \
+ ! -type d ! -perm -400 -exec chmod a+r {} \; -o \
+ ! -type d ! -perm -444 -exec $(install_sh) -c -m a+r {} {} \; \
+ || chmod -R a+r "$(distdir)"
+dist-gzip: distdir
+ tardir=$(distdir) && $(am__tar) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).tar.gz
+ $(am__remove_distdir)
+dist-bzip2: distdir
+ tardir=$(distdir) && $(am__tar) | bzip2 -9 -c >$(distdir).tar.bz2
+ $(am__remove_distdir)
+
+dist-lzma: distdir
+ tardir=$(distdir) && $(am__tar) | lzma -9 -c >$(distdir).tar.lzma
+ $(am__remove_distdir)
+
+dist-xz: distdir
+ tardir=$(distdir) && $(am__tar) | xz -c >$(distdir).tar.xz
+ $(am__remove_distdir)
+
+dist-tarZ: distdir
+ tardir=$(distdir) && $(am__tar) | compress -c >$(distdir).tar.Z
+ $(am__remove_distdir)
+
+dist-shar: distdir
+ shar $(distdir) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).shar.gz
+ $(am__remove_distdir)
+
+dist-zip: distdir
+ -rm -f $(distdir).zip
+ zip -rq $(distdir).zip $(distdir)
+ $(am__remove_distdir)
+
+dist dist-all: distdir
+ tardir=$(distdir) && $(am__tar) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).tar.gz
+ tardir=$(distdir) && $(am__tar) | bzip2 -9 -c >$(distdir).tar.bz2
+ $(am__remove_distdir)
+
+# This target untars the dist file and tries a VPATH configuration. Then
+# it guarantees that the distribution is self-contained by making another
+# tarfile.
+distcheck: dist
+ case '$(DIST_ARCHIVES)' in \
+ *.tar.gz*) \
+ GZIP=$(GZIP_ENV) gzip -dc $(distdir).tar.gz | $(am__untar) ;;\
+ *.tar.bz2*) \
+ bzip2 -dc $(distdir).tar.bz2 | $(am__untar) ;;\
+ *.tar.lzma*) \
+ lzma -dc $(distdir).tar.lzma | $(am__untar) ;;\
+ *.tar.xz*) \
+ xz -dc $(distdir).tar.xz | $(am__untar) ;;\
+ *.tar.Z*) \
+ uncompress -c $(distdir).tar.Z | $(am__untar) ;;\
+ *.shar.gz*) \
+ GZIP=$(GZIP_ENV) gzip -dc $(distdir).shar.gz | unshar ;;\
+ *.zip*) \
+ unzip $(distdir).zip ;;\
+ esac
+ chmod -R a-w $(distdir); chmod a+w $(distdir)
+ mkdir $(distdir)/_build
+ mkdir $(distdir)/_inst
+ chmod a-w $(distdir)
+ test -d $(distdir)/_build || exit 0; \
+ dc_install_base=`$(am__cd) $(distdir)/_inst && pwd | sed -e 's,^[^:\\/]:[\\/],/,'` \
+ && dc_destdir="$${TMPDIR-/tmp}/am-dc-$$$$/" \
+ && am__cwd=`pwd` \
+ && $(am__cd) $(distdir)/_build \
+ && ../configure --srcdir=.. --prefix="$$dc_install_base" \
+ $(DISTCHECK_CONFIGURE_FLAGS) \
+ && $(MAKE) $(AM_MAKEFLAGS) \
+ && $(MAKE) $(AM_MAKEFLAGS) dvi \
+ && $(MAKE) $(AM_MAKEFLAGS) check \
+ && $(MAKE) $(AM_MAKEFLAGS) install \
+ && $(MAKE) $(AM_MAKEFLAGS) installcheck \
+ && $(MAKE) $(AM_MAKEFLAGS) uninstall \
+ && $(MAKE) $(AM_MAKEFLAGS) distuninstallcheck_dir="$$dc_install_base" \
+ distuninstallcheck \
+ && chmod -R a-w "$$dc_install_base" \
+ && ({ \
+ (cd ../.. && umask 077 && mkdir "$$dc_destdir") \
+ && $(MAKE) $(AM_MAKEFLAGS) DESTDIR="$$dc_destdir" install \
+ && $(MAKE) $(AM_MAKEFLAGS) DESTDIR="$$dc_destdir" uninstall \
+ && $(MAKE) $(AM_MAKEFLAGS) DESTDIR="$$dc_destdir" \
+ distuninstallcheck_dir="$$dc_destdir" distuninstallcheck; \
+ } || { rm -rf "$$dc_destdir"; exit 1; }) \
+ && rm -rf "$$dc_destdir" \
+ && $(MAKE) $(AM_MAKEFLAGS) dist \
+ && rm -rf $(DIST_ARCHIVES) \
+ && $(MAKE) $(AM_MAKEFLAGS) distcleancheck \
+ && cd "$$am__cwd" \
+ || exit 1
+ $(am__remove_distdir)
+ @(echo "$(distdir) archives ready for distribution: "; \
+ list='$(DIST_ARCHIVES)'; for i in $$list; do echo $$i; done) | \
+ sed -e 1h -e 1s/./=/g -e 1p -e 1x -e '$$p' -e '$$x'
+distuninstallcheck:
+ @$(am__cd) '$(distuninstallcheck_dir)' \
+ && test `$(distuninstallcheck_listfiles) | wc -l` -le 1 \
+ || { echo "ERROR: files left after uninstall:" ; \
+ if test -n "$(DESTDIR)"; then \
+ echo " (check DESTDIR support)"; \
+ fi ; \
+ $(distuninstallcheck_listfiles) ; \
+ exit 1; } >&2
+distcleancheck: distclean
+ @if test '$(srcdir)' = . ; then \
+ echo "ERROR: distcleancheck can only run from a VPATH build" ; \
+ exit 1 ; \
+ fi
+ @test `$(distcleancheck_listfiles) | wc -l` -eq 0 \
+ || { echo "ERROR: files left in build directory after distclean:" ; \
+ $(distcleancheck_listfiles) ; \
+ exit 1; } >&2
+check-am: all-am
+ $(MAKE) $(AM_MAKEFLAGS) check-TESTS
+check: check-am
+all-am: Makefile $(LTLIBRARIES) $(PROGRAMS) $(SCRIPTS) $(MANS) $(DATA) \
+ $(HEADERS) config.h
+install-binPROGRAMS: install-libLTLIBRARIES
+
+installdirs:
+ for dir in "$(DESTDIR)$(libdir)" "$(DESTDIR)$(pkglibdir)" "$(DESTDIR)$(bindir)" "$(DESTDIR)$(bindir)" "$(DESTDIR)$(man1dir)" "$(DESTDIR)$(man3dir)" "$(DESTDIR)$(man7dir)" "$(DESTDIR)$(pkgconfigdir)" "$(DESTDIR)$(rdmaincludedir)" "$(DESTDIR)$(rdmaincludedir)"; do \
+ test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+ done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+ @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ `test -z '$(STRIP)' || \
+ echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install
+mostlyclean-generic:
+ -test -z "$(TEST_LOGS)" || rm -f $(TEST_LOGS)
+ -test -z "$(TEST_LOGS_TMP)" || rm -f $(TEST_LOGS_TMP)
+ -test -z "$(TEST_SUITE_HTML)" || rm -f $(TEST_SUITE_HTML)
+ -test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
+
+clean-generic:
+
+distclean-generic:
+ -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+ -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+ -rm -f prov/gni/src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/gni/src/$(am__dirstamp)
+ -rm -f prov/gni/test/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/gni/test/$(am__dirstamp)
+ -rm -f prov/mxm/src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/mxm/src/$(am__dirstamp)
+ -rm -f prov/psm/src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/psm/src/$(am__dirstamp)
+ -rm -f prov/psm2/src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/psm2/src/$(am__dirstamp)
+ -rm -f prov/rxd/src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/rxd/src/$(am__dirstamp)
+ -rm -f prov/rxm/src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/rxm/src/$(am__dirstamp)
+ -rm -f prov/sockets/src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/sockets/src/$(am__dirstamp)
+ -rm -f prov/udp/src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/udp/src/$(am__dirstamp)
+ -rm -f prov/usnic/src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/usnic/src/$(am__dirstamp)
+ -rm -f prov/usnic/src/usnic_direct/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/usnic/src/usnic_direct/$(am__dirstamp)
+ -rm -f prov/util/src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/util/src/$(am__dirstamp)
+ -rm -f prov/verbs/src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/verbs/src/$(am__dirstamp)
+ -rm -f prov/verbs/src/ep_rdm/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/verbs/src/ep_rdm/$(am__dirstamp)
+ -rm -f src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f src/$(am__dirstamp)
+ -rm -f src/osx/$(DEPDIR)/$(am__dirstamp)
+ -rm -f src/osx/$(am__dirstamp)
+ -rm -f src/unix/$(DEPDIR)/$(am__dirstamp)
+ -rm -f src/unix/$(am__dirstamp)
+ -rm -f util/$(DEPDIR)/$(am__dirstamp)
+ -rm -f util/$(am__dirstamp)
+
+maintainer-clean-generic:
+ @echo "This command is intended for maintainers to use"
+ @echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-binPROGRAMS clean-generic clean-libLTLIBRARIES \
+ clean-libtool clean-noinstLTLIBRARIES clean-pkglibLTLIBRARIES \
+ mostlyclean-am
+
+distclean: distclean-am
+ -rm -f $(am__CONFIG_DISTCLEAN_FILES)
+ -rm -rf prov/gni/src/$(DEPDIR) prov/gni/test/$(DEPDIR) prov/mxm/src/$(DEPDIR) prov/psm/src/$(DEPDIR) prov/psm2/src/$(DEPDIR) prov/rxd/src/$(DEPDIR) prov/rxm/src/$(DEPDIR) prov/sockets/src/$(DEPDIR) prov/udp/src/$(DEPDIR) prov/usnic/src/$(DEPDIR) prov/usnic/src/usnic_direct/$(DEPDIR) prov/util/src/$(DEPDIR) prov/verbs/src/$(DEPDIR) prov/verbs/src/ep_rdm/$(DEPDIR) src/$(DEPDIR) src/osx/$(DEPDIR) src/unix/$(DEPDIR) util/$(DEPDIR)
+ -rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+ distclean-hdr distclean-libtool distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-man install-nodist_rdmaincludeHEADERS \
+ install-pkgconfigDATA install-rdmaincludeHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am: install-binPROGRAMS install-binSCRIPTS \
+ install-libLTLIBRARIES install-pkglibLTLIBRARIES
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man: install-man1 install-man3 install-man7
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+ -rm -f $(am__CONFIG_DISTCLEAN_FILES)
+ -rm -rf $(top_srcdir)/autom4te.cache
+ -rm -rf prov/gni/src/$(DEPDIR) prov/gni/test/$(DEPDIR) prov/mxm/src/$(DEPDIR) prov/psm/src/$(DEPDIR) prov/psm2/src/$(DEPDIR) prov/rxd/src/$(DEPDIR) prov/rxm/src/$(DEPDIR) prov/sockets/src/$(DEPDIR) prov/udp/src/$(DEPDIR) prov/usnic/src/$(DEPDIR) prov/usnic/src/usnic_direct/$(DEPDIR) prov/util/src/$(DEPDIR) prov/verbs/src/$(DEPDIR) prov/verbs/src/ep_rdm/$(DEPDIR) src/$(DEPDIR) src/osx/$(DEPDIR) src/unix/$(DEPDIR) util/$(DEPDIR)
+ -rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+ mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-binPROGRAMS uninstall-binSCRIPTS \
+ uninstall-libLTLIBRARIES uninstall-man \
+ uninstall-nodist_rdmaincludeHEADERS uninstall-pkgconfigDATA \
+ uninstall-pkglibLTLIBRARIES uninstall-rdmaincludeHEADERS
+
+uninstall-man: uninstall-man1 uninstall-man3 uninstall-man7
+
+.MAKE: all check-am check-html install-am install-strip recheck \
+ recheck-html
+
+.PHONY: CTAGS GTAGS all all-am am--refresh check check-TESTS check-am \
+ check-html clean clean-binPROGRAMS clean-generic \
+ clean-libLTLIBRARIES clean-libtool clean-noinstLTLIBRARIES \
+ clean-pkglibLTLIBRARIES ctags dist dist-all dist-bzip2 \
+ dist-gzip dist-hook dist-lzma dist-shar dist-tarZ dist-xz \
+ dist-zip distcheck distclean distclean-compile \
+ distclean-generic distclean-hdr distclean-libtool \
+ distclean-tags distcleancheck distdir distuninstallcheck dvi \
+ dvi-am html html-am info info-am install install-am \
+ install-binPROGRAMS install-binSCRIPTS install-data \
+ install-data-am install-dvi install-dvi-am install-exec \
+ install-exec-am install-html install-html-am install-info \
+ install-info-am install-libLTLIBRARIES install-man \
+ install-man1 install-man3 install-man7 \
+ install-nodist_rdmaincludeHEADERS install-pdf install-pdf-am \
+ install-pkgconfigDATA install-pkglibLTLIBRARIES install-ps \
+ install-ps-am install-rdmaincludeHEADERS install-strip \
+ installcheck installcheck-am installdirs maintainer-clean \
+ maintainer-clean-generic mostlyclean mostlyclean-compile \
+ mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \
+ recheck recheck-html tags uninstall uninstall-am \
+ uninstall-binPROGRAMS uninstall-binSCRIPTS \
+ uninstall-libLTLIBRARIES uninstall-man uninstall-man1 \
+ uninstall-man3 uninstall-man7 \
+ uninstall-nodist_rdmaincludeHEADERS uninstall-pkgconfigDATA \
+ uninstall-pkglibLTLIBRARIES uninstall-rdmaincludeHEADERS
+
+
+nroff:
+ @for file in $(real_man_pages) $(prov_install_man_pages); do \
+ source=`echo $$file | sed -e 's@/man[0-9]@@'`; \
+ config/md2nroff.pl --source=$$source.md; \
+ done
+
+dist-hook: libfabric.spec
+ cp libfabric.spec $(distdir)
+ "$(top_srcdir)/config/distscript.pl" "$(distdir)" "$(PACKAGE_VERSION)"
+
+test:
+ ./util/fi_info
+
+rpm: dist
+ LDFLAGS=-Wl,--build-id rpmbuild -ta libfabric-$(PACKAGE_VERSION).tar.bz2
+
+#prov_install_man_pages += man/man7/fi_rxd.7
+
+#prov_dist_man_pages += man/man7/fi_rxd.7
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/NEWS.md b/NEWS.md
index 54c32b2..aa22a2b 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -5,6 +5,151 @@ This file contains the main features as well as overviews of specific
bug fixes (and other actions) for each version of Libfabric since
version 1.0.
+v1.4.0rc2, Mon Oct 17, 2016
+===========
+
+- Add new options, `-f` and `-d`, to fi_info that can be used to specify hints
+ about the fabric and domain name. Change port to `-P` and provider to `-p` to
+ be more in line with fi_pingpong.
+
+## GNI provider notes
+
+- General bug fixes, plugged memory leaks, performance improvements,
+ improved error handling and warning messages, etc.
+- Additional API support:
+ - FI_THREAD_COMPLETION
+ - FI_RMA_EVENT
+ - iov length up to 8 for messaging data transfers
+- Provider-specific API support:
+ - Aries native AXOR atomic operation
+ - Memory registation cache flush operation
+- Memory registration cache improvements:
+ - IOMMU notifier support
+ - Alternatives to the internal cache
+ - Additional tuning knobs
+- On-node optimization for rendezvous message communication via XPMEM
+- Internal fixes to support accelerators and KNL processors
+- Better support for running in CCM mode (in support for fabtests)
+
+## MXM provider
+
+- The mxm provider has been deprecated and will be replaced in a future release.
+
+## PSM provider notes
+
+- General bug fixes
+- Use utility provider for EQ, wait object, and poll set
+- Allow multi-recv to post buffer larger than message size limit
+
+## PSM2 provider notes
+
+- General bug fixes
+- Add support for multi-iov RMA read and aromic operations
+- Allow multi-recv to post buffer larger than message size limit
+
+## Sockets provider notes
+
+- General code cleanup and bug fixes
+- Set tx/rx op_flags correctly to be consistent with manpage
+- Restructure struct sock_ep to support alias ep
+- Refactor CQ/Cntr bindings, CQ completion generation, and counter increments
+- Copy compare data to internal buffer when FI_INJECT is set in
+ fi_compare_atomic
+- Correctly handle triggered operation when FI_INJECT is set or
+ triggered op is enqueued or counter is incremented. Initialize counter
+ threshold to INT_MAX
+- Refactor and cleanup connection management code, add locks to avoid
+ race between main thread and progress thread, add logic to correctly handle
+ FI_SHUTDOWN and FI_REJECT
+- Set fabric name as network address in the format of a.b.c.d/e and
+ domain name as network interface name
+- Remove sock_compare_addr and add two utility functions ofi_equals_ipaddr
+ and ofi_equals_sockaddr in fi.h
+- Refactor fi_getinfo to handle corner cases and add logic if a given
+ src_addr matches to any local interface addr
+- Restructure acquiring/releasing the list_lock in progress thread so that
+ it is only acquired once per iteration
+- Refactor connection management of MSG ep so that it uses TCP instead of
+ UDP for connection management msg and new port for every MSG endpoint
+- Add sock_cq_sanitize_flags function to make sure only flags returned in
+ CQ events are the ones that are listed on the manpage
+- Update fi_poll semantics for counters so that it returns success if the
+ counter value is different from the last-read-value
+- Allow multiple threads to wait on one counter
+- Update code to use ofi_util_mr - the new MR structure added to util code
+- Fix fi_av_insert not to report error when the number of inserted addr
+ exceeds the count attribute in fi_av_attr
+- Add garbage collection of AV indices after fi_av_remove, add ep list in AV
+ and cleanup conn map during fi_av_remove
+- Use correct fi_tx_attr/fi_rx_attr for scalable ep
+
+## UDP provider notes
+
+- Enhance parameter checks for several function calls.
+- Fix memory leak freeing CQ structure.
+- Bind to a source address when enabling endpoint.
+- Reduce reported resource limits (domain attributes).
+
+## usNIC provider notes
+- Fix handling of EP_MSG peers on different IP subnets [PR #1988]
+- Fix handling of CM data. Fixes a bug where data received would overwrite
+ parts of the connection management structure [PR #1991]
+- Fix bug in CM connect/accept handling that would cause a seg fault if data
+ was sent as part of a connection request [PR #1991]
+- Fix invalid completion lengths in the MSG and RDM endpoint implementations of
+ fi_recvv and fi_recvmsg [PR #2026]
+- Implement the FI_CM_DATA_SIZE option for fi_getopt on passive endpoints
+ [PR #2033]
+- Add fi_reject implementation that supports data exchange [PR #2038]
+- Fix fi_av_straddr bug that reported port in network order [PR #2244]
+- Report -FI_EOPBADSTATE if the size left functions are used on an endpoint
+ which has not been enabled [PR #2266]
+- Change the domain/fabric naming. The fabric is now represented as the network
+ address in the form of a.b.c.d/e and the domain name is the usNIC device
+ name. For more information see fi_usnic(7) [PR #2287]
+- Fix the domain name matching in fi_getinfo/fi_domain [PR #2298]
+- Fix issue with AV where it is fully closed before pending asynchronous
+ inserts can finish leading to invalid data accesses [PR #2397]
+- Free all data associated with AV when fi_av_close is called [PR #2397]
+- Fail with -FI_EINVAL if a value of FI_ADDR_NOTAVAIL is given to fi_av_lookup.
+ [PR #2397]
+- Verify AV attributes and return an error if anything that is unsupported is
+ requested (FI_AV_TABLE, named AVs, FI_READ, etc.) [PR #2397]
+
+## Verbs provider notes
+
+- Add fork support. It is enabled by default and can be turned off by setting the
+ FI_FORK_UNSAFE variable to "yes". This can improve performance of memory registrations
+ but also makes fork unsafe. The following are the limitations of fork support:
+ - Fabric resources like endpoint, CQ, EQ, etc. should not be used in the
+ forked process.
+ - The memory registered using fi_mr_reg has to be page aligned since ibv_reg_mr
+ marks the entire page that a memory region belongs to as not to be re-mapped
+ when the process is forked (MADV_DONTFORK).
+- Fix a bug where source address info was not being returned in fi_info when
+ destination node is specified.
+
+- verbs/MSG
+ - Add fi_getopt for passive endpoints.
+ - Add support for shared RX contexts.
+- verbs/RDM
+ - General bug fixes
+ - Add FI_MSG capability
+ - Add FI_PEEK and FI_CLAIM flags support
+ - Add completion flags support
+ - Add selective completion support
+ - Add fi_cq_readerr support
+ - Add possibility to set IPoIB network interface via FI_VERBS_IFACE
+ environment variable
+ - Add large data transfer support (> 1 GB)
+ - Add FI_AV_TABLE support
+ - Add fi_cntr support
+ - Add environment variables for the provider tuning: FI_VERBS_RDM_BUFFER_NUM,
+ FI_VERBS_RDM_BUFFER_SIZE, FI_VERBS_RDM_RNDV_SEG_SIZE,
+ FI_VERBS_RDM_CQREAD_BUNCH_SIZE, FI_VERBS_RDM_THREAD_TIMEOUT,
+ FI_VERBS_RDM_EAGER_SEND_OPCODE
+ - Add iWarp support
+
v1.3.0, Mon Apr 11, 2016
========================
diff --git a/README b/README
index b4111ed..f853783 100644
--- a/README
+++ b/README
@@ -1,5 +1,5 @@
-Version @VERSION@
-Released on @DATE@
+Version Libfabric v1.4.0rc2
+Released on 2016-10-17
Introduction
============
@@ -76,3 +76,8 @@ To install from a libfabric source package run the following commands:
If building directly from the libfabric git tree, run './autogen.sh'
before the configure step.
+
+Windows
+=======
+Please find Windows specific instructions in docs/windows.txt file
+
diff --git a/README.md b/README.md
deleted file mode 100644
index adeb5bb..0000000
--- a/README.md
+++ /dev/null
@@ -1,252 +0,0 @@
-[<img alt="libfabric master branch Travis CI status" src="https://travis-ci.org/ofiwg/libfabric.svg?branch=master"/>](https://travis-ci.org/ofiwg/libfabric)
-[<img alt="libfabric Coverity scan suild status" src="https://scan.coverity.com/projects/4274/badge.svg"/>](https://scan.coverity.com/projects/4274)
-[](https://github.com/ofiwg/libfabric/releases/latest)
-
-# libfabric
-
-The Open Fabrics Interfaces (OFI) is a framework focused on exporting fabric
-communication services to applications.
-
-See [the OFI web site](http://libfabric.org) for more details, including a
-description and overview of the project, and detailed documentation of the
-Libfabric APIs.
-
-## Installing pre-built Libfabric packages
-
-On OS X, the latest release of Libfabric can be installed using the
-[Homebrew](https://github.com/Homebrew/homebrew) package manager using the
-following command:
-
-```
-$ brew install libfabric
-```
-
-Libfabric pre-built binaries may be available from other sources, such as Linux
-distributions.
-
-## Building and installing Libfabric from source
-
-Distribution tarballs are available from the Github
-[releases](https://github.com/ofiwg/libfabric/releases) tab.
-
-If you are building Libfabric from a developer Git clone, you must first run
-the `autogen.sh` script. This will invoke the GNU Autotools to bootstrap
-Libfabric's configuration and build mechanisms. If you are building Libfabric
-from an official distribution tarball, there is no need to run `autogen.sh`;
-Libfabric distribution tarballs are already bootstrapped for you.
-
-Libfabric currently supports GNU/Linux, Free BSD, and OS X.
-
-### Configure options
-
-The `configure` script has many built in options (see `./configure --help`).
-Some useful options are:
-
-```
---prefix=<directory>
-```
-
-By default `make install` will place the files in the `/usr` tree.
-The `--prefix` option specifies that Libfabric files should be installed into
-the tree specified by named `<directory>`. The executables will be located at
-`<directory>/bin`.
-
-```
---with-valgrind=<directory>
-```
-
-Directory where valgrind is installed. If valgrind is found, then valgrind
-annotations are enabled. This may incur a performance penalty.
-
-```
---enable-debug
-```
-
-Enable debug code paths. This enables various extra checks and allows for using
-the highest verbosity logging output that is normally compiled out in
-production builds.
-
-```
---enable-<provider>=[yes|no|auto|dl|<directory>]
---disable-<provider>
-```
-
-This enables or disables the provider named `<provider>`. Valid options are:
-- auto (This is the default if the `--enable-<provider>` option isn't specified)
-
- The provider will be enabled if all of its requirements are satisfied. If one
- of the requirements cannot be satisfied, then the provider is disabled.
-- yes (This is the default if the `--enable-<provider>` option is specified)
-
- The configure script will abort if the provider cannot be enabled (e.g., due
- to some of its requirements not being available.
-- no
-
- Disable the provider. This is synonymous with `--disable-<provider>`.
-- dl
-
- Enable the provider and build it as a loadable library.
-- \<directory\>
-
- Enable the provider and use the installation given in `<directory>`.
-
-### Examples
-
-Consider the following example:
-
-```
-$ ./configure --prefix=/opt/libfabric --disable-sockets && make -j 32 && sudo make install
-```
-This will tell Libfabric to disable the `sockets` provider, and install
-Libfabric in the `/opt/libfabric` tree. All other providers will be enabled if
-possible and all debug features will be disabled.
-
-Alternatively:
-
-```
-$ ./configure --prefix=/opt/libfabric --enable-debug --enable-psm=dl && make -j 32 && sudo make install
-```
-
-This will tell Libfabric to enable the `psm` provider as a loadable library,
-enable all debug code paths, and install Libfabric to the `/opt/libfabric`
-tree. All other providers will be enabled if possible.
-
-
-## Providers
-
-### gni
-
-***
-
-The `gni` provider runs on Cray XC (TM) systems utilizing the user-space
-Generic Network Interface (`uGNI`) which provides low-level access to
-the Aries interconnect. The Aries interconnect is designed for
-low-latency one-sided messaging and also includes direct hardware
-support for common atomic operations and optimized collectives.
-
-See the `fi_gni(7)` man page for more details.
-
-#### Dependencies
-
-- The `gni` provider requires `gcc` version 4.9 or higher.
-
-### mxm
-
-***
-
-The `mxm` provider runs over the MXM (Mellanox messaging) interface that is
-currently supported by the Mellanox InfiniBand fabrics. The `mxm` provider
-makes use of MXM tag matching API in order to implement a limited set of the
-Libfabric data transfer APIs, namely, tagged message queue.
-
-See the `fi_mxm(7)` man page for more details.
-
-### psm
-
-***
-
-The `psm` provider runs over the PSM 1.x interface that is currently supported
-by the Intel TrueScale Fabric. PSM provides tag-matching message queue
-functions that are optimized for MPI implementations. PSM also has limited
-Active Message support, which is not officially published but is quite stable
-and well documented in the source code (part of the OFED release). The `psm`
-provider makes use of both the tag-matching message queue functions and the
-Active Message functions to support a variety of Libfabric data transfer APIs,
-including tagged message queue, message queue, RMA, and atomic
-operations.
-
-The `psm` provider can work with the `psm2-compat` library, which exposes
-a PSM 1.x interface over the Intel Omni-Path Fabric.
-
-See the `fi_psm(7)` man page for more details.
-
-### psm2
-
-***
-
-The `psm2` provider runs over the PSM 2.x interface that is supported
-by the Intel Omni-Path Fabric. PSM 2.x has all the PSM 1.x features plus a set
-of new functions with enhanced capabilities. Since PSM 1.x and PSM 2.x are not
-ABI compatible, the `psm2` provider only works with PSM 2.x and doesn't support
-Intel TrueScale Fabric.
-
-See the `fi_psm2(7)` man page for more details.
-
-### sockets
-
-***
-
-The `sockets` provider is a general purpose provider that can be used on any
-system that supports TCP sockets. The provider is not intended to provide
-performance improvements over regular TCP sockets, but rather to allow
-developers to write, test, and debug application code even on platforms
-that do not have high-performance fabric hardware. The sockets provider
-supports all Libfabric provider requirements and interfaces.
-
-See the `fi_sockets(7)` man page for more details.
-
-
-### udp
-
-***
-
-The `udp` provider is a basic provider that can be used on any system that
-supports UDP sockets. The provider is not intended to provide performance
-improvements over regular UDP sockets, but rather to allow application and
-provider developers to write, test, and debug their code. The `udp` provider
-forms the foundation of a utility provider that enables the implementation of
-Libfabric features over any hardware.
-
-See the `fi_udp(7)` man page for more details.
-
-### usnic
-
-***
-
-The `usnic` provider is designed to run over the Cisco VIC (virtualized NIC)
-hardware on Cisco UCS servers. It utilizes the Cisco usnic (userspace NIC)
-capabilities of the VIC to enable ultra low latency and other offload
-capabilities on Ethernet networks.
-
-See the `fi_usnic(7)` man page for more details.
-
-#### Dependencies
-
-- The `usnic` provider depends on library files from either `libnl` version 1
- (sometimes known as `libnl` or `libnl1`) or version 3 (sometimes known as
- `libnl3`). If you are compiling Libfabric from source and want to enable
- usNIC support, you will also need the matching `libnl` header files (e.g.,
- if you are building with `libnl` version 3, you need both the header and
- library files from version 3).
-
-#### Configure options
-
-```
---with-libnl=<directory>
-```
-
-If specified, look for libnl support. If it is not found then the `usnic`
-provider will not be built. If `<directory>` is specified, then check in the
-directory and check for `libnl` version 3. If version 3 is not found, then
-check for version 1. If no `<directory>` argument is specified, then this
-option is redundant with `--with-usnic`.
-
-### verbs
-
-***
-
-The verbs provider enables applications using OFI to be run over any verbs
-hardware (Infiniband, iWarp, etc). It uses the Linux Verbs API for network
-transport and provides a translation of OFI calls to appropriate verbs API calls.
-It uses librdmacm for communication management and libibverbs for other control
-and data transfer operations.
-
-See the `fi_verbs(7)` man page for more details.
-
-#### Dependencies
-
-- The verbs provider requires libibverbs (v1.1.8 or newer) and librdmacm (v1.0.16
- or newer). If you are compiling Libfabric from source and want to enable verbs
- support, you will also need the matching header files for the above two libraries.
- If the libraries and header files are not in default paths, specify them in CFLAGS,
- LDFLAGS and LD_LIBRARY_PATH environment variables.
diff --git a/aclocal.m4 b/aclocal.m4
new file mode 100644
index 0000000..cb368a8
--- /dev/null
+++ b/aclocal.m4
@@ -0,0 +1,1036 @@
+# generated automatically by aclocal 1.11.1 -*- Autoconf -*-
+
+# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
+# 2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc.
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+m4_ifndef([AC_AUTOCONF_VERSION],
+ [m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl
+m4_if(m4_defn([AC_AUTOCONF_VERSION]), [2.67],,
+[m4_warning([this file was generated for autoconf 2.67.
+You have another version of autoconf. It may work, but is not guaranteed to.
+If you have problems, you may need to regenerate the build system entirely.
+To do so, use the procedure documented by the package, typically `autoreconf'.])])
+
+# Copyright (C) 2002, 2003, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# AM_AUTOMAKE_VERSION(VERSION)
+# ----------------------------
+# Automake X.Y traces this macro to ensure aclocal.m4 has been
+# generated from the m4 files accompanying Automake X.Y.
+# (This private macro should not be called outside this file.)
+AC_DEFUN([AM_AUTOMAKE_VERSION],
+[am__api_version='1.11'
+dnl Some users find AM_AUTOMAKE_VERSION and mistake it for a way to
+dnl require some minimum version. Point them to the right macro.
+m4_if([$1], [1.11.1], [],
+ [AC_FATAL([Do not call $0, use AM_INIT_AUTOMAKE([$1]).])])dnl
+])
+
+# _AM_AUTOCONF_VERSION(VERSION)
+# -----------------------------
+# aclocal traces this macro to find the Autoconf version.
+# This is a private macro too. Using m4_define simplifies
+# the logic in aclocal, which can simply ignore this definition.
+m4_define([_AM_AUTOCONF_VERSION], [])
+
+# AM_SET_CURRENT_AUTOMAKE_VERSION
+# -------------------------------
+# Call AM_AUTOMAKE_VERSION and AM_AUTOMAKE_VERSION so they can be traced.
+# This function is AC_REQUIREd by AM_INIT_AUTOMAKE.
+AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION],
+[AM_AUTOMAKE_VERSION([1.11.1])dnl
+m4_ifndef([AC_AUTOCONF_VERSION],
+ [m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl
+_AM_AUTOCONF_VERSION(m4_defn([AC_AUTOCONF_VERSION]))])
+
+# AM_AUX_DIR_EXPAND -*- Autoconf -*-
+
+# Copyright (C) 2001, 2003, 2005 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# For projects using AC_CONFIG_AUX_DIR([foo]), Autoconf sets
+# $ac_aux_dir to `$srcdir/foo'. In other projects, it is set to
+# `$srcdir', `$srcdir/..', or `$srcdir/../..'.
+#
+# Of course, Automake must honor this variable whenever it calls a
+# tool from the auxiliary directory. The problem is that $srcdir (and
+# therefore $ac_aux_dir as well) can be either absolute or relative,
+# depending on how configure is run. This is pretty annoying, since
+# it makes $ac_aux_dir quite unusable in subdirectories: in the top
+# source directory, any form will work fine, but in subdirectories a
+# relative path needs to be adjusted first.
+#
+# $ac_aux_dir/missing
+# fails when called from a subdirectory if $ac_aux_dir is relative
+# $top_srcdir/$ac_aux_dir/missing
+# fails if $ac_aux_dir is absolute,
+# fails when called from a subdirectory in a VPATH build with
+# a relative $ac_aux_dir
+#
+# The reason of the latter failure is that $top_srcdir and $ac_aux_dir
+# are both prefixed by $srcdir. In an in-source build this is usually
+# harmless because $srcdir is `.', but things will broke when you
+# start a VPATH build or use an absolute $srcdir.
+#
+# So we could use something similar to $top_srcdir/$ac_aux_dir/missing,
+# iff we strip the leading $srcdir from $ac_aux_dir. That would be:
+# am_aux_dir='\$(top_srcdir)/'`expr "$ac_aux_dir" : "$srcdir//*\(.*\)"`
+# and then we would define $MISSING as
+# MISSING="\${SHELL} $am_aux_dir/missing"
+# This will work as long as MISSING is not called from configure, because
+# unfortunately $(top_srcdir) has no meaning in configure.
+# However there are other variables, like CC, which are often used in
+# configure, and could therefore not use this "fixed" $ac_aux_dir.
+#
+# Another solution, used here, is to always expand $ac_aux_dir to an
+# absolute PATH. The drawback is that using absolute paths prevent a
+# configured tree to be moved without reconfiguration.
+
+AC_DEFUN([AM_AUX_DIR_EXPAND],
+[dnl Rely on autoconf to set up CDPATH properly.
+AC_PREREQ([2.50])dnl
+# expand $ac_aux_dir to an absolute path
+am_aux_dir=`cd $ac_aux_dir && pwd`
+])
+
+# AM_CONDITIONAL -*- Autoconf -*-
+
+# Copyright (C) 1997, 2000, 2001, 2003, 2004, 2005, 2006, 2008
+# Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# serial 9
+
+# AM_CONDITIONAL(NAME, SHELL-CONDITION)
+# -------------------------------------
+# Define a conditional.
+AC_DEFUN([AM_CONDITIONAL],
+[AC_PREREQ(2.52)dnl
+ ifelse([$1], [TRUE], [AC_FATAL([$0: invalid condition: $1])],
+ [$1], [FALSE], [AC_FATAL([$0: invalid condition: $1])])dnl
+AC_SUBST([$1_TRUE])dnl
+AC_SUBST([$1_FALSE])dnl
+_AM_SUBST_NOTMAKE([$1_TRUE])dnl
+_AM_SUBST_NOTMAKE([$1_FALSE])dnl
+m4_define([_AM_COND_VALUE_$1], [$2])dnl
+if $2; then
+ $1_TRUE=
+ $1_FALSE='#'
+else
+ $1_TRUE='#'
+ $1_FALSE=
+fi
+AC_CONFIG_COMMANDS_PRE(
+[if test -z "${$1_TRUE}" && test -z "${$1_FALSE}"; then
+ AC_MSG_ERROR([[conditional "$1" was never defined.
+Usually this means the macro was only invoked conditionally.]])
+fi])])
+
+# Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2009
+# Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# serial 10
+
+# There are a few dirty hacks below to avoid letting `AC_PROG_CC' be
+# written in clear, in which case automake, when reading aclocal.m4,
+# will think it sees a *use*, and therefore will trigger all it's
+# C support machinery. Also note that it means that autoscan, seeing
+# CC etc. in the Makefile, will ask for an AC_PROG_CC use...
+
+
+# _AM_DEPENDENCIES(NAME)
+# ----------------------
+# See how the compiler implements dependency checking.
+# NAME is "CC", "CXX", "GCJ", or "OBJC".
+# We try a few techniques and use that to set a single cache variable.
+#
+# We don't AC_REQUIRE the corresponding AC_PROG_CC since the latter was
+# modified to invoke _AM_DEPENDENCIES(CC); we would have a circular
+# dependency, and given that the user is not expected to run this macro,
+# just rely on AC_PROG_CC.
+AC_DEFUN([_AM_DEPENDENCIES],
+[AC_REQUIRE([AM_SET_DEPDIR])dnl
+AC_REQUIRE([AM_OUTPUT_DEPENDENCY_COMMANDS])dnl
+AC_REQUIRE([AM_MAKE_INCLUDE])dnl
+AC_REQUIRE([AM_DEP_TRACK])dnl
+
+ifelse([$1], CC, [depcc="$CC" am_compiler_list=],
+ [$1], CXX, [depcc="$CXX" am_compiler_list=],
+ [$1], OBJC, [depcc="$OBJC" am_compiler_list='gcc3 gcc'],
+ [$1], UPC, [depcc="$UPC" am_compiler_list=],
+ [$1], GCJ, [depcc="$GCJ" am_compiler_list='gcc3 gcc'],
+ [depcc="$$1" am_compiler_list=])
+
+AC_CACHE_CHECK([dependency style of $depcc],
+ [am_cv_$1_dependencies_compiler_type],
+[if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then
+ # We make a subdir and do the tests there. Otherwise we can end up
+ # making bogus files that we don't know about and never remove. For
+ # instance it was reported that on HP-UX the gcc test will end up
+ # making a dummy file named `D' -- because `-MD' means `put the output
+ # in D'.
+ mkdir conftest.dir
+ # Copy depcomp to subdir because otherwise we won't find it if we're
+ # using a relative directory.
+ cp "$am_depcomp" conftest.dir
+ cd conftest.dir
+ # We will build objects and dependencies in a subdirectory because
+ # it helps to detect inapplicable dependency modes. For instance
+ # both Tru64's cc and ICC support -MD to output dependencies as a
+ # side effect of compilation, but ICC will put the dependencies in
+ # the current directory while Tru64 will put them in the object
+ # directory.
+ mkdir sub
+
+ am_cv_$1_dependencies_compiler_type=none
+ if test "$am_compiler_list" = ""; then
+ am_compiler_list=`sed -n ['s/^#*\([a-zA-Z0-9]*\))$/\1/p'] < ./depcomp`
+ fi
+ am__universal=false
+ m4_case([$1], [CC],
+ [case " $depcc " in #(
+ *\ -arch\ *\ -arch\ *) am__universal=true ;;
+ esac],
+ [CXX],
+ [case " $depcc " in #(
+ *\ -arch\ *\ -arch\ *) am__universal=true ;;
+ esac])
+
+ for depmode in $am_compiler_list; do
+ # Setup a source with many dependencies, because some compilers
+ # like to wrap large dependency lists on column 80 (with \), and
+ # we should not choose a depcomp mode which is confused by this.
+ #
+ # We need to recreate these files for each test, as the compiler may
+ # overwrite some of them when testing with obscure command lines.
+ # This happens at least with the AIX C compiler.
+ : > sub/conftest.c
+ for i in 1 2 3 4 5 6; do
+ echo '#include "conftst'$i'.h"' >> sub/conftest.c
+ # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with
+ # Solaris 8's {/usr,}/bin/sh.
+ touch sub/conftst$i.h
+ done
+ echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf
+
+ # We check with `-c' and `-o' for the sake of the "dashmstdout"
+ # mode. It turns out that the SunPro C++ compiler does not properly
+ # handle `-M -o', and we need to detect this. Also, some Intel
+ # versions had trouble with output in subdirs
+ am__obj=sub/conftest.${OBJEXT-o}
+ am__minus_obj="-o $am__obj"
+ case $depmode in
+ gcc)
+ # This depmode causes a compiler race in universal mode.
+ test "$am__universal" = false || continue
+ ;;
+ nosideeffect)
+ # after this tag, mechanisms are not by side-effect, so they'll
+ # only be used when explicitly requested
+ if test "x$enable_dependency_tracking" = xyes; then
+ continue
+ else
+ break
+ fi
+ ;;
+ msvisualcpp | msvcmsys)
+ # This compiler won't grok `-c -o', but also, the minuso test has
+ # not run yet. These depmodes are late enough in the game, and
+ # so weak that their functioning should not be impacted.
+ am__obj=conftest.${OBJEXT-o}
+ am__minus_obj=
+ ;;
+ none) break ;;
+ esac
+ if depmode=$depmode \
+ source=sub/conftest.c object=$am__obj \
+ depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \
+ $SHELL ./depcomp $depcc -c $am__minus_obj sub/conftest.c \
+ >/dev/null 2>conftest.err &&
+ grep sub/conftst1.h sub/conftest.Po > /dev/null 2>&1 &&
+ grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 &&
+ grep $am__obj sub/conftest.Po > /dev/null 2>&1 &&
+ ${MAKE-make} -s -f confmf > /dev/null 2>&1; then
+ # icc doesn't choke on unknown options, it will just issue warnings
+ # or remarks (even with -Werror). So we grep stderr for any message
+ # that says an option was ignored or not supported.
+ # When given -MP, icc 7.0 and 7.1 complain thusly:
+ # icc: Command line warning: ignoring option '-M'; no argument required
+ # The diagnosis changed in icc 8.0:
+ # icc: Command line remark: option '-MP' not supported
+ if (grep 'ignoring option' conftest.err ||
+ grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else
+ am_cv_$1_dependencies_compiler_type=$depmode
+ break
+ fi
+ fi
+ done
+
+ cd ..
+ rm -rf conftest.dir
+else
+ am_cv_$1_dependencies_compiler_type=none
+fi
+])
+AC_SUBST([$1DEPMODE], [depmode=$am_cv_$1_dependencies_compiler_type])
+AM_CONDITIONAL([am__fastdep$1], [
+ test "x$enable_dependency_tracking" != xno \
+ && test "$am_cv_$1_dependencies_compiler_type" = gcc3])
+])
+
+
+# AM_SET_DEPDIR
+# -------------
+# Choose a directory name for dependency files.
+# This macro is AC_REQUIREd in _AM_DEPENDENCIES
+AC_DEFUN([AM_SET_DEPDIR],
+[AC_REQUIRE([AM_SET_LEADING_DOT])dnl
+AC_SUBST([DEPDIR], ["${am__leading_dot}deps"])dnl
+])
+
+
+# AM_DEP_TRACK
+# ------------
+AC_DEFUN([AM_DEP_TRACK],
+[AC_ARG_ENABLE(dependency-tracking,
+[ --disable-dependency-tracking speeds up one-time build
+ --enable-dependency-tracking do not reject slow dependency extractors])
+if test "x$enable_dependency_tracking" != xno; then
+ am_depcomp="$ac_aux_dir/depcomp"
+ AMDEPBACKSLASH='\'
+fi
+AM_CONDITIONAL([AMDEP], [test "x$enable_dependency_tracking" != xno])
+AC_SUBST([AMDEPBACKSLASH])dnl
+_AM_SUBST_NOTMAKE([AMDEPBACKSLASH])dnl
+])
+
+# Generate code to set up dependency tracking. -*- Autoconf -*-
+
+# Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2008
+# Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+#serial 5
+
+# _AM_OUTPUT_DEPENDENCY_COMMANDS
+# ------------------------------
+AC_DEFUN([_AM_OUTPUT_DEPENDENCY_COMMANDS],
+[{
+ # Autoconf 2.62 quotes --file arguments for eval, but not when files
+ # are listed without --file. Let's play safe and only enable the eval
+ # if we detect the quoting.
+ case $CONFIG_FILES in
+ *\'*) eval set x "$CONFIG_FILES" ;;
+ *) set x $CONFIG_FILES ;;
+ esac
+ shift
+ for mf
+ do
+ # Strip MF so we end up with the name of the file.
+ mf=`echo "$mf" | sed -e 's/:.*$//'`
+ # Check whether this is an Automake generated Makefile or not.
+ # We used to match only the files named `Makefile.in', but
+ # some people rename them; so instead we look at the file content.
+ # Grep'ing the first line is not enough: some people post-process
+ # each Makefile.in and add a new line on top of each file to say so.
+ # Grep'ing the whole file is not good either: AIX grep has a line
+ # limit of 2048, but all sed's we know have understand at least 4000.
+ if sed -n 's,^#.*generated by automake.*,X,p' "$mf" | grep X >/dev/null 2>&1; then
+ dirpart=`AS_DIRNAME("$mf")`
+ else
+ continue
+ fi
+ # Extract the definition of DEPDIR, am__include, and am__quote
+ # from the Makefile without running `make'.
+ DEPDIR=`sed -n 's/^DEPDIR = //p' < "$mf"`
+ test -z "$DEPDIR" && continue
+ am__include=`sed -n 's/^am__include = //p' < "$mf"`
+ test -z "am__include" && continue
+ am__quote=`sed -n 's/^am__quote = //p' < "$mf"`
+ # When using ansi2knr, U may be empty or an underscore; expand it
+ U=`sed -n 's/^U = //p' < "$mf"`
+ # Find all dependency output files, they are included files with
+ # $(DEPDIR) in their names. We invoke sed twice because it is the
+ # simplest approach to changing $(DEPDIR) to its actual value in the
+ # expansion.
+ for file in `sed -n "
+ s/^$am__include $am__quote\(.*(DEPDIR).*\)$am__quote"'$/\1/p' <"$mf" | \
+ sed -e 's/\$(DEPDIR)/'"$DEPDIR"'/g' -e 's/\$U/'"$U"'/g'`; do
+ # Make sure the directory exists.
+ test -f "$dirpart/$file" && continue
+ fdir=`AS_DIRNAME(["$file"])`
+ AS_MKDIR_P([$dirpart/$fdir])
+ # echo "creating $dirpart/$file"
+ echo '# dummy' > "$dirpart/$file"
+ done
+ done
+}
+])# _AM_OUTPUT_DEPENDENCY_COMMANDS
+
+
+# AM_OUTPUT_DEPENDENCY_COMMANDS
+# -----------------------------
+# This macro should only be invoked once -- use via AC_REQUIRE.
+#
+# This code is only required when automatic dependency tracking
+# is enabled. FIXME. This creates each `.P' file that we will
+# need in order to bootstrap the dependency handling code.
+AC_DEFUN([AM_OUTPUT_DEPENDENCY_COMMANDS],
+[AC_CONFIG_COMMANDS([depfiles],
+ [test x"$AMDEP_TRUE" != x"" || _AM_OUTPUT_DEPENDENCY_COMMANDS],
+ [AMDEP_TRUE="$AMDEP_TRUE" ac_aux_dir="$ac_aux_dir"])
+])
+
+# Do all the work for Automake. -*- Autoconf -*-
+
+# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
+# 2005, 2006, 2008, 2009 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# serial 16
+
+# This macro actually does too much. Some checks are only needed if
+# your package does certain things. But this isn't really a big deal.
+
+# AM_INIT_AUTOMAKE(PACKAGE, VERSION, [NO-DEFINE])
+# AM_INIT_AUTOMAKE([OPTIONS])
+# -----------------------------------------------
+# The call with PACKAGE and VERSION arguments is the old style
+# call (pre autoconf-2.50), which is being phased out. PACKAGE
+# and VERSION should now be passed to AC_INIT and removed from
+# the call to AM_INIT_AUTOMAKE.
+# We support both call styles for the transition. After
+# the next Automake release, Autoconf can make the AC_INIT
+# arguments mandatory, and then we can depend on a new Autoconf
+# release and drop the old call support.
+AC_DEFUN([AM_INIT_AUTOMAKE],
+[AC_PREREQ([2.62])dnl
+dnl Autoconf wants to disallow AM_ names. We explicitly allow
+dnl the ones we care about.
+m4_pattern_allow([^AM_[A-Z]+FLAGS$])dnl
+AC_REQUIRE([AM_SET_CURRENT_AUTOMAKE_VERSION])dnl
+AC_REQUIRE([AC_PROG_INSTALL])dnl
+if test "`cd $srcdir && pwd`" != "`pwd`"; then
+ # Use -I$(srcdir) only when $(srcdir) != ., so that make's output
+ # is not polluted with repeated "-I."
+ AC_SUBST([am__isrc], [' -I$(srcdir)'])_AM_SUBST_NOTMAKE([am__isrc])dnl
+ # test to see if srcdir already configured
+ if test -f $srcdir/config.status; then
+ AC_MSG_ERROR([source directory already configured; run "make distclean" there first])
+ fi
+fi
+
+# test whether we have cygpath
+if test -z "$CYGPATH_W"; then
+ if (cygpath --version) >/dev/null 2>/dev/null; then
+ CYGPATH_W='cygpath -w'
+ else
+ CYGPATH_W=echo
+ fi
+fi
+AC_SUBST([CYGPATH_W])
+
+# Define the identity of the package.
+dnl Distinguish between old-style and new-style calls.
+m4_ifval([$2],
+[m4_ifval([$3], [_AM_SET_OPTION([no-define])])dnl
+ AC_SUBST([PACKAGE], [$1])dnl
+ AC_SUBST([VERSION], [$2])],
+[_AM_SET_OPTIONS([$1])dnl
+dnl Diagnose old-style AC_INIT with new-style AM_AUTOMAKE_INIT.
+m4_if(m4_ifdef([AC_PACKAGE_NAME], 1)m4_ifdef([AC_PACKAGE_VERSION], 1), 11,,
+ [m4_fatal([AC_INIT should be called with package and version arguments])])dnl
+ AC_SUBST([PACKAGE], ['AC_PACKAGE_TARNAME'])dnl
+ AC_SUBST([VERSION], ['AC_PACKAGE_VERSION'])])dnl
+
+_AM_IF_OPTION([no-define],,
+[AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE", [Name of package])
+ AC_DEFINE_UNQUOTED(VERSION, "$VERSION", [Version number of package])])dnl
+
+# Some tools Automake needs.
+AC_REQUIRE([AM_SANITY_CHECK])dnl
+AC_REQUIRE([AC_ARG_PROGRAM])dnl
+AM_MISSING_PROG(ACLOCAL, aclocal-${am__api_version})
+AM_MISSING_PROG(AUTOCONF, autoconf)
+AM_MISSING_PROG(AUTOMAKE, automake-${am__api_version})
+AM_MISSING_PROG(AUTOHEADER, autoheader)
+AM_MISSING_PROG(MAKEINFO, makeinfo)
+AC_REQUIRE([AM_PROG_INSTALL_SH])dnl
+AC_REQUIRE([AM_PROG_INSTALL_STRIP])dnl
+AC_REQUIRE([AM_PROG_MKDIR_P])dnl
+# We need awk for the "check" target. The system "awk" is bad on
+# some platforms.
+AC_REQUIRE([AC_PROG_AWK])dnl
+AC_REQUIRE([AC_PROG_MAKE_SET])dnl
+AC_REQUIRE([AM_SET_LEADING_DOT])dnl
+_AM_IF_OPTION([tar-ustar], [_AM_PROG_TAR([ustar])],
+ [_AM_IF_OPTION([tar-pax], [_AM_PROG_TAR([pax])],
+ [_AM_PROG_TAR([v7])])])
+_AM_IF_OPTION([no-dependencies],,
+[AC_PROVIDE_IFELSE([AC_PROG_CC],
+ [_AM_DEPENDENCIES(CC)],
+ [define([AC_PROG_CC],
+ defn([AC_PROG_CC])[_AM_DEPENDENCIES(CC)])])dnl
+AC_PROVIDE_IFELSE([AC_PROG_CXX],
+ [_AM_DEPENDENCIES(CXX)],
+ [define([AC_PROG_CXX],
+ defn([AC_PROG_CXX])[_AM_DEPENDENCIES(CXX)])])dnl
+AC_PROVIDE_IFELSE([AC_PROG_OBJC],
+ [_AM_DEPENDENCIES(OBJC)],
+ [define([AC_PROG_OBJC],
+ defn([AC_PROG_OBJC])[_AM_DEPENDENCIES(OBJC)])])dnl
+])
+_AM_IF_OPTION([silent-rules], [AC_REQUIRE([AM_SILENT_RULES])])dnl
+dnl The `parallel-tests' driver may need to know about EXEEXT, so add the
+dnl `am__EXEEXT' conditional if _AM_COMPILER_EXEEXT was seen. This macro
+dnl is hooked onto _AC_COMPILER_EXEEXT early, see below.
+AC_CONFIG_COMMANDS_PRE(dnl
+[m4_provide_if([_AM_COMPILER_EXEEXT],
+ [AM_CONDITIONAL([am__EXEEXT], [test -n "$EXEEXT"])])])dnl
+])
+
+dnl Hook into `_AC_COMPILER_EXEEXT' early to learn its expansion. Do not
+dnl add the conditional right here, as _AC_COMPILER_EXEEXT may be further
+dnl mangled by Autoconf and run in a shell conditional statement.
+m4_define([_AC_COMPILER_EXEEXT],
+m4_defn([_AC_COMPILER_EXEEXT])[m4_provide([_AM_COMPILER_EXEEXT])])
+
+
+# When config.status generates a header, we must update the stamp-h file.
+# This file resides in the same directory as the config header
+# that is generated. The stamp files are numbered to have different names.
+
+# Autoconf calls _AC_AM_CONFIG_HEADER_HOOK (when defined) in the
+# loop where config.status creates the headers, so we can generate
+# our stamp files there.
+AC_DEFUN([_AC_AM_CONFIG_HEADER_HOOK],
+[# Compute $1's index in $config_headers.
+_am_arg=$1
+_am_stamp_count=1
+for _am_header in $config_headers :; do
+ case $_am_header in
+ $_am_arg | $_am_arg:* )
+ break ;;
+ * )
+ _am_stamp_count=`expr $_am_stamp_count + 1` ;;
+ esac
+done
+echo "timestamp for $_am_arg" >`AS_DIRNAME(["$_am_arg"])`/stamp-h[]$_am_stamp_count])
+
+# Copyright (C) 2001, 2003, 2005, 2008 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# AM_PROG_INSTALL_SH
+# ------------------
+# Define $install_sh.
+AC_DEFUN([AM_PROG_INSTALL_SH],
+[AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl
+if test x"${install_sh}" != xset; then
+ case $am_aux_dir in
+ *\ * | *\ *)
+ install_sh="\${SHELL} '$am_aux_dir/install-sh'" ;;
+ *)
+ install_sh="\${SHELL} $am_aux_dir/install-sh"
+ esac
+fi
+AC_SUBST(install_sh)])
+
+# Copyright (C) 2003, 2005 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# serial 2
+
+# Check whether the underlying file-system supports filenames
+# with a leading dot. For instance MS-DOS doesn't.
+AC_DEFUN([AM_SET_LEADING_DOT],
+[rm -rf .tst 2>/dev/null
+mkdir .tst 2>/dev/null
+if test -d .tst; then
+ am__leading_dot=.
+else
+ am__leading_dot=_
+fi
+rmdir .tst 2>/dev/null
+AC_SUBST([am__leading_dot])])
+
+# Check to see how 'make' treats includes. -*- Autoconf -*-
+
+# Copyright (C) 2001, 2002, 2003, 2005, 2009 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# serial 4
+
+# AM_MAKE_INCLUDE()
+# -----------------
+# Check to see how make treats includes.
+AC_DEFUN([AM_MAKE_INCLUDE],
+[am_make=${MAKE-make}
+cat > confinc << 'END'
+am__doit:
+ @echo this is the am__doit target
+.PHONY: am__doit
+END
+# If we don't find an include directive, just comment out the code.
+AC_MSG_CHECKING([for style of include used by $am_make])
+am__include="#"
+am__quote=
+_am_result=none
+# First try GNU make style include.
+echo "include confinc" > confmf
+# Ignore all kinds of additional output from `make'.
+case `$am_make -s -f confmf 2> /dev/null` in #(
+*the\ am__doit\ target*)
+ am__include=include
+ am__quote=
+ _am_result=GNU
+ ;;
+esac
+# Now try BSD make style include.
+if test "$am__include" = "#"; then
+ echo '.include "confinc"' > confmf
+ case `$am_make -s -f confmf 2> /dev/null` in #(
+ *the\ am__doit\ target*)
+ am__include=.include
+ am__quote="\""
+ _am_result=BSD
+ ;;
+ esac
+fi
+AC_SUBST([am__include])
+AC_SUBST([am__quote])
+AC_MSG_RESULT([$_am_result])
+rm -f confinc confmf
+])
+
+# Copyright (C) 1999, 2000, 2001, 2003, 2004, 2005, 2008
+# Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# serial 6
+
+# AM_PROG_CC_C_O
+# --------------
+# Like AC_PROG_CC_C_O, but changed for automake.
+AC_DEFUN([AM_PROG_CC_C_O],
+[AC_REQUIRE([AC_PROG_CC_C_O])dnl
+AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl
+AC_REQUIRE_AUX_FILE([compile])dnl
+# FIXME: we rely on the cache variable name because
+# there is no other way.
+set dummy $CC
+am_cc=`echo $[2] | sed ['s/[^a-zA-Z0-9_]/_/g;s/^[0-9]/_/']`
+eval am_t=\$ac_cv_prog_cc_${am_cc}_c_o
+if test "$am_t" != yes; then
+ # Losing compiler, so override with the script.
+ # FIXME: It is wrong to rewrite CC.
+ # But if we don't then we get into trouble of one sort or another.
+ # A longer-term fix would be to have automake use am__CC in this case,
+ # and then we could set am__CC="\$(top_srcdir)/compile \$(CC)"
+ CC="$am_aux_dir/compile $CC"
+fi
+dnl Make sure AC_PROG_CC is never called again, or it will override our
+dnl setting of CC.
+m4_define([AC_PROG_CC],
+ [m4_fatal([AC_PROG_CC cannot be called after AM_PROG_CC_C_O])])
+])
+
+# Fake the existence of programs that GNU maintainers use. -*- Autoconf -*-
+
+# Copyright (C) 1997, 1999, 2000, 2001, 2003, 2004, 2005, 2008
+# Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# serial 6
+
+# AM_MISSING_PROG(NAME, PROGRAM)
+# ------------------------------
+AC_DEFUN([AM_MISSING_PROG],
+[AC_REQUIRE([AM_MISSING_HAS_RUN])
+$1=${$1-"${am_missing_run}$2"}
+AC_SUBST($1)])
+
+
+# AM_MISSING_HAS_RUN
+# ------------------
+# Define MISSING if not defined so far and test if it supports --run.
+# If it does, set am_missing_run to use it, otherwise, to nothing.
+AC_DEFUN([AM_MISSING_HAS_RUN],
+[AC_REQUIRE([AM_AUX_DIR_EXPAND])dnl
+AC_REQUIRE_AUX_FILE([missing])dnl
+if test x"${MISSING+set}" != xset; then
+ case $am_aux_dir in
+ *\ * | *\ *)
+ MISSING="\${SHELL} \"$am_aux_dir/missing\"" ;;
+ *)
+ MISSING="\${SHELL} $am_aux_dir/missing" ;;
+ esac
+fi
+# Use eval to expand $SHELL
+if eval "$MISSING --run true"; then
+ am_missing_run="$MISSING --run "
+else
+ am_missing_run=
+ AC_MSG_WARN([`missing' script is too old or missing])
+fi
+])
+
+# Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# AM_PROG_MKDIR_P
+# ---------------
+# Check for `mkdir -p'.
+AC_DEFUN([AM_PROG_MKDIR_P],
+[AC_PREREQ([2.60])dnl
+AC_REQUIRE([AC_PROG_MKDIR_P])dnl
+dnl Automake 1.8 to 1.9.6 used to define mkdir_p. We now use MKDIR_P,
+dnl while keeping a definition of mkdir_p for backward compatibility.
+dnl @MKDIR_P@ is magic: AC_OUTPUT adjusts its value for each Makefile.
+dnl However we cannot define mkdir_p as $(MKDIR_P) for the sake of
+dnl Makefile.ins that do not define MKDIR_P, so we do our own
+dnl adjustment using top_builddir (which is defined more often than
+dnl MKDIR_P).
+AC_SUBST([mkdir_p], ["$MKDIR_P"])dnl
+case $mkdir_p in
+ [[\\/$]]* | ?:[[\\/]]*) ;;
+ */*) mkdir_p="\$(top_builddir)/$mkdir_p" ;;
+esac
+])
+
+# Helper functions for option handling. -*- Autoconf -*-
+
+# Copyright (C) 2001, 2002, 2003, 2005, 2008 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# serial 4
+
+# _AM_MANGLE_OPTION(NAME)
+# -----------------------
+AC_DEFUN([_AM_MANGLE_OPTION],
+[[_AM_OPTION_]m4_bpatsubst($1, [[^a-zA-Z0-9_]], [_])])
+
+# _AM_SET_OPTION(NAME)
+# ------------------------------
+# Set option NAME. Presently that only means defining a flag for this option.
+AC_DEFUN([_AM_SET_OPTION],
+[m4_define(_AM_MANGLE_OPTION([$1]), 1)])
+
+# _AM_SET_OPTIONS(OPTIONS)
+# ----------------------------------
+# OPTIONS is a space-separated list of Automake options.
+AC_DEFUN([_AM_SET_OPTIONS],
+[m4_foreach_w([_AM_Option], [$1], [_AM_SET_OPTION(_AM_Option)])])
+
+# _AM_IF_OPTION(OPTION, IF-SET, [IF-NOT-SET])
+# -------------------------------------------
+# Execute IF-SET if OPTION is set, IF-NOT-SET otherwise.
+AC_DEFUN([_AM_IF_OPTION],
+[m4_ifset(_AM_MANGLE_OPTION([$1]), [$2], [$3])])
+
+# Copyright (C) 2001, 2003, 2005 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# AM_RUN_LOG(COMMAND)
+# -------------------
+# Run COMMAND, save the exit status in ac_status, and log it.
+# (This has been adapted from Autoconf's _AC_RUN_LOG macro.)
+AC_DEFUN([AM_RUN_LOG],
+[{ echo "$as_me:$LINENO: $1" >&AS_MESSAGE_LOG_FD
+ ($1) >&AS_MESSAGE_LOG_FD 2>&AS_MESSAGE_LOG_FD
+ ac_status=$?
+ echo "$as_me:$LINENO: \$? = $ac_status" >&AS_MESSAGE_LOG_FD
+ (exit $ac_status); }])
+
+# Check to make sure that the build environment is sane. -*- Autoconf -*-
+
+# Copyright (C) 1996, 1997, 2000, 2001, 2003, 2005, 2008
+# Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# serial 5
+
+# AM_SANITY_CHECK
+# ---------------
+AC_DEFUN([AM_SANITY_CHECK],
+[AC_MSG_CHECKING([whether build environment is sane])
+# Just in case
+sleep 1
+echo timestamp > conftest.file
+# Reject unsafe characters in $srcdir or the absolute working directory
+# name. Accept space and tab only in the latter.
+am_lf='
+'
+case `pwd` in
+ *[[\\\"\#\$\&\'\`$am_lf]]*)
+ AC_MSG_ERROR([unsafe absolute working directory name]);;
+esac
+case $srcdir in
+ *[[\\\"\#\$\&\'\`$am_lf\ \ ]]*)
+ AC_MSG_ERROR([unsafe srcdir value: `$srcdir']);;
+esac
+
+# Do `set' in a subshell so we don't clobber the current shell's
+# arguments. Must try -L first in case configure is actually a
+# symlink; some systems play weird games with the mod time of symlinks
+# (eg FreeBSD returns the mod time of the symlink's containing
+# directory).
+if (
+ set X `ls -Lt "$srcdir/configure" conftest.file 2> /dev/null`
+ if test "$[*]" = "X"; then
+ # -L didn't work.
+ set X `ls -t "$srcdir/configure" conftest.file`
+ fi
+ rm -f conftest.file
+ if test "$[*]" != "X $srcdir/configure conftest.file" \
+ && test "$[*]" != "X conftest.file $srcdir/configure"; then
+
+ # If neither matched, then we have a broken ls. This can happen
+ # if, for instance, CONFIG_SHELL is bash and it inherits a
+ # broken ls alias from the environment. This has actually
+ # happened. Such a system could not be considered "sane".
+ AC_MSG_ERROR([ls -t appears to fail. Make sure there is not a broken
+alias in your environment])
+ fi
+
+ test "$[2]" = conftest.file
+ )
+then
+ # Ok.
+ :
+else
+ AC_MSG_ERROR([newly created file is older than distributed files!
+Check your system clock])
+fi
+AC_MSG_RESULT(yes)])
+
+# Copyright (C) 2009 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# serial 1
+
+# AM_SILENT_RULES([DEFAULT])
+# --------------------------
+# Enable less verbose build rules; with the default set to DEFAULT
+# (`yes' being less verbose, `no' or empty being verbose).
+AC_DEFUN([AM_SILENT_RULES],
+[AC_ARG_ENABLE([silent-rules],
+[ --enable-silent-rules less verbose build output (undo: `make V=1')
+ --disable-silent-rules verbose build output (undo: `make V=0')])
+case $enable_silent_rules in
+yes) AM_DEFAULT_VERBOSITY=0;;
+no) AM_DEFAULT_VERBOSITY=1;;
+*) AM_DEFAULT_VERBOSITY=m4_if([$1], [yes], [0], [1]);;
+esac
+AC_SUBST([AM_DEFAULT_VERBOSITY])dnl
+AM_BACKSLASH='\'
+AC_SUBST([AM_BACKSLASH])dnl
+_AM_SUBST_NOTMAKE([AM_BACKSLASH])dnl
+])
+
+# Copyright (C) 2001, 2003, 2005 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# AM_PROG_INSTALL_STRIP
+# ---------------------
+# One issue with vendor `install' (even GNU) is that you can't
+# specify the program used to strip binaries. This is especially
+# annoying in cross-compiling environments, where the build's strip
+# is unlikely to handle the host's binaries.
+# Fortunately install-sh will honor a STRIPPROG variable, so we
+# always use install-sh in `make install-strip', and initialize
+# STRIPPROG with the value of the STRIP variable (set by the user).
+AC_DEFUN([AM_PROG_INSTALL_STRIP],
+[AC_REQUIRE([AM_PROG_INSTALL_SH])dnl
+# Installed binaries are usually stripped using `strip' when the user
+# run `make install-strip'. However `strip' might not be the right
+# tool to use in cross-compilation environments, therefore Automake
+# will honor the `STRIP' environment variable to overrule this program.
+dnl Don't test for $cross_compiling = yes, because it might be `maybe'.
+if test "$cross_compiling" != no; then
+ AC_CHECK_TOOL([STRIP], [strip], :)
+fi
+INSTALL_STRIP_PROGRAM="\$(install_sh) -c -s"
+AC_SUBST([INSTALL_STRIP_PROGRAM])])
+
+# Copyright (C) 2006, 2008 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# serial 2
+
+# _AM_SUBST_NOTMAKE(VARIABLE)
+# ---------------------------
+# Prevent Automake from outputting VARIABLE = @VARIABLE@ in Makefile.in.
+# This macro is traced by Automake.
+AC_DEFUN([_AM_SUBST_NOTMAKE])
+
+# AM_SUBST_NOTMAKE(VARIABLE)
+# ---------------------------
+# Public sister of _AM_SUBST_NOTMAKE.
+AC_DEFUN([AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE($@)])
+
+# Check how to create a tarball. -*- Autoconf -*-
+
+# Copyright (C) 2004, 2005 Free Software Foundation, Inc.
+#
+# This file is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# serial 2
+
+# _AM_PROG_TAR(FORMAT)
+# --------------------
+# Check how to create a tarball in format FORMAT.
+# FORMAT should be one of `v7', `ustar', or `pax'.
+#
+# Substitute a variable $(am__tar) that is a command
+# writing to stdout a FORMAT-tarball containing the directory
+# $tardir.
+# tardir=directory && $(am__tar) > result.tar
+#
+# Substitute a variable $(am__untar) that extract such
+# a tarball read from stdin.
+# $(am__untar) < result.tar
+AC_DEFUN([_AM_PROG_TAR],
+[# Always define AMTAR for backward compatibility.
+AM_MISSING_PROG([AMTAR], [tar])
+m4_if([$1], [v7],
+ [am__tar='${AMTAR} chof - "$$tardir"'; am__untar='${AMTAR} xf -'],
+ [m4_case([$1], [ustar],, [pax],,
+ [m4_fatal([Unknown tar format])])
+AC_MSG_CHECKING([how to create a $1 tar archive])
+# Loop over all known methods to create a tar archive until one works.
+_am_tools='gnutar m4_if([$1], [ustar], [plaintar]) pax cpio none'
+_am_tools=${am_cv_prog_tar_$1-$_am_tools}
+# Do not fold the above two line into one, because Tru64 sh and
+# Solaris sh will not grok spaces in the rhs of `-'.
+for _am_tool in $_am_tools
+do
+ case $_am_tool in
+ gnutar)
+ for _am_tar in tar gnutar gtar;
+ do
+ AM_RUN_LOG([$_am_tar --version]) && break
+ done
+ am__tar="$_am_tar --format=m4_if([$1], [pax], [posix], [$1]) -chf - "'"$$tardir"'
+ am__tar_="$_am_tar --format=m4_if([$1], [pax], [posix], [$1]) -chf - "'"$tardir"'
+ am__untar="$_am_tar -xf -"
+ ;;
+ plaintar)
+ # Must skip GNU tar: if it does not support --format= it doesn't create
+ # ustar tarball either.
+ (tar --version) >/dev/null 2>&1 && continue
+ am__tar='tar chf - "$$tardir"'
+ am__tar_='tar chf - "$tardir"'
+ am__untar='tar xf -'
+ ;;
+ pax)
+ am__tar='pax -L -x $1 -w "$$tardir"'
+ am__tar_='pax -L -x $1 -w "$tardir"'
+ am__untar='pax -r'
+ ;;
+ cpio)
+ am__tar='find "$$tardir" -print | cpio -o -H $1 -L'
+ am__tar_='find "$tardir" -print | cpio -o -H $1 -L'
+ am__untar='cpio -i -H $1 -d'
+ ;;
+ none)
+ am__tar=false
+ am__tar_=false
+ am__untar=false
+ ;;
+ esac
+
+ # If the value was cached, stop now. We just wanted to have am__tar
+ # and am__untar set.
+ test -n "${am_cv_prog_tar_$1}" && break
+
+ # tar/untar a dummy directory, and stop if the command works
+ rm -rf conftest.dir
+ mkdir conftest.dir
+ echo GrepMe > conftest.dir/file
+ AM_RUN_LOG([tardir=conftest.dir && eval $am__tar_ >conftest.tar])
+ rm -rf conftest.dir
+ if test -s conftest.tar; then
+ AM_RUN_LOG([$am__untar <conftest.tar])
+ grep GrepMe conftest.dir/file >/dev/null 2>&1 && break
+ fi
+done
+rm -rf conftest.dir
+
+AC_CACHE_VAL([am_cv_prog_tar_$1], [am_cv_prog_tar_$1=$_am_tool])
+AC_MSG_RESULT([$am_cv_prog_tar_$1])])
+AC_SUBST([am__tar])
+AC_SUBST([am__untar])
+]) # _AM_PROG_TAR
+
+m4_include([config/fi_provider.m4])
+m4_include([config/libtool.m4])
+m4_include([config/ltoptions.m4])
+m4_include([config/ltsugar.m4])
+m4_include([config/ltversion.m4])
+m4_include([config/lt~obsolete.m4])
diff --git a/autogen.sh b/autogen.sh
deleted file mode 100755
index df14219..0000000
--- a/autogen.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#! /bin/sh
-
-if test ! -d .git && test ! -f src/fabric.c; then
- echo You really need to run this script in the top-level libfabric directory
- exit 1
-fi
-
-set -x
-
-autoreconf -ivf
diff --git a/config.h.in b/config.h.in
new file mode 100644
index 0000000..e398266
--- /dev/null
+++ b/config.h.in
@@ -0,0 +1,207 @@
+/* config.h.in. Generated from configure.ac by autoheader. */
+
+/* defined to 1 if libfabric was configured with --enable-debug, 0 otherwise
+ */
+#undef ENABLE_DEBUG
+
+/* define when building with FABRIC_DIRECT support */
+#undef FABRIC_DIRECT_ENABLED
+
+/* Define to 1 if the linker supports alias attribute. */
+#undef HAVE_ALIAS_ATTRIBUTE
+
+/* Set to 1 to use c11 atomic functions */
+#undef HAVE_ATOMICS
+
+/* Define to 1 if clock_gettime is available. */
+#undef HAVE_CLOCK_GETTIME
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#undef HAVE_DLFCN_H
+
+/* Define if you have epoll support. */
+#undef HAVE_EPOLL
+
+/* Define to 1 if you have the `epoll_create' function. */
+#undef HAVE_EPOLL_CREATE
+
+/* Define to 1 if you have the `getifaddrs' function. */
+#undef HAVE_GETIFADDRS
+
+/* gni provider is built */
+#undef HAVE_GNI
+
+/* Define to 1 if the system has the type `gni_ct_cqw_post_descriptor_t'. */
+#undef HAVE_GNI_CT_CQW_POST_DESCRIPTOR_T
+
+/* gni provider is built as DSO */
+#undef HAVE_GNI_DL
+
+/* Define to 1 if host_clock_get_service is available. */
+#undef HAVE_HOST_GET_CLOCK_SERVICE
+
+/* Define to 1 if you have the <infiniband/verbs.h> header file. */
+#undef HAVE_INFINIBAND_VERBS_H
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#undef HAVE_INTTYPES_H
+
+/* Define to 1 if you have the `dl' library (-ldl). */
+#undef HAVE_LIBDL
+
+/* Whether we have libl or libnl3 */
+#undef HAVE_LIBNL3
+
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#undef HAVE_LIBPTHREAD
+
+/* Define to 1 if you have the <memory.h> header file. */
+#undef HAVE_MEMORY_H
+
+/* mxm provider is built */
+#undef HAVE_MXM
+
+/* Define to 1 if you have the <mxm/api/mxm_api.h> header file. */
+#undef HAVE_MXM_API_MXM_API_H
+
+/* mxm provider is built as DSO */
+#undef HAVE_MXM_DL
+
+/* Define to 1 if you have the <netlink/netlink.h> header file. */
+#undef HAVE_NETLINK_NETLINK_H
+
+/* Define to 1 if you have the <netlink/version.h> header file. */
+#undef HAVE_NETLINK_VERSION_H
+
+/* psm provider is built */
+#undef HAVE_PSM
+
+/* psm2 provider is built */
+#undef HAVE_PSM2
+
+/* psm2 provider is built as DSO */
+#undef HAVE_PSM2_DL
+
+/* Define to 1 if you have the <psm2.h> header file. */
+#undef HAVE_PSM2_H
+
+/* psm provider is built as DSO */
+#undef HAVE_PSM_DL
+
+/* Define to 1 if you have the <psm.h> header file. */
+#undef HAVE_PSM_H
+
+/* Define to 1 if you have the <rdma/rsocket.h> header file. */
+#undef HAVE_RDMA_RSOCKET_H
+
+/* rxd provider is built */
+#undef HAVE_RXD
+
+/* rxd provider is built as DSO */
+#undef HAVE_RXD_DL
+
+/* rxm provider is built */
+#undef HAVE_RXM
+
+/* rxm provider is built as DSO */
+#undef HAVE_RXM_DL
+
+/* sockets provider is built */
+#undef HAVE_SOCKETS
+
+/* sockets provider is built as DSO */
+#undef HAVE_SOCKETS_DL
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#undef HAVE_STDINT_H
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#undef HAVE_STDLIB_H
+
+/* Define to 1 if you have the <strings.h> header file. */
+#undef HAVE_STRINGS_H
+
+/* Define to 1 if you have the <string.h> header file. */
+#undef HAVE_STRING_H
+
+/* Define to 1 if compiler/linker support symbol versioning. */
+#undef HAVE_SYMVER_SUPPORT
+
+/* Define to 1 if you have the <sys/mman.h> header file. */
+#undef HAVE_SYS_MMAN_H
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#undef HAVE_SYS_STAT_H
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#undef HAVE_SYS_TYPES_H
+
+/* Define to 1 if typeof works with your compiler. */
+#undef HAVE_TYPEOF
+
+/* udp provider is built */
+#undef HAVE_UDP
+
+/* udp provider is built as DSO */
+#undef HAVE_UDP_DL
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#undef HAVE_UNISTD_H
+
+/* usnic provider is built */
+#undef HAVE_USNIC
+
+/* usnic provider is built as DSO */
+#undef HAVE_USNIC_DL
+
+/* verbs provider is built */
+#undef HAVE_VERBS
+
+/* verbs provider is built as DSO */
+#undef HAVE_VERBS_DL
+
+/* Define to 1 if xpmem available */
+#undef HAVE_XPMEM
+
+/* Define to 1 to enable valgrind annotations */
+#undef INCLUDE_VALGRIND
+
+/* Define to the sub-directory in which libtool stores uninstalled libraries.
+ */
+#undef LT_OBJDIR
+
+/* Define to 1 if your C compiler doesn't accept -c and -o together. */
+#undef NO_MINUS_C_MINUS_O
+
+/* Name of package */
+#undef PACKAGE
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#undef PACKAGE_NAME
+
+/* Define to the full name and version of this package. */
+#undef PACKAGE_STRING
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the home page for this package. */
+#undef PACKAGE_URL
+
+/* Define to the version of this package. */
+#undef PACKAGE_VERSION
+
+/* Define to 1 if pthread_spin_init is available. */
+#undef PT_LOCK_SPIN
+
+/* Define to 1 if you have the ANSI C header files. */
+#undef STDC_HEADERS
+
+/* Version number of package */
+#undef VERSION
+
+/* Define to __typeof__ if your compiler spells it that way. */
+#undef typeof
diff --git a/config/compile b/config/compile
new file mode 100755
index 0000000..c0096a7
--- /dev/null
+++ b/config/compile
@@ -0,0 +1,143 @@
+#! /bin/sh
+# Wrapper for compilers which do not understand `-c -o'.
+
+scriptversion=2009-10-06.20; # UTC
+
+# Copyright (C) 1999, 2000, 2003, 2004, 2005, 2009 Free Software
+# Foundation, Inc.
+# Written by Tom Tromey <tromey at cygnus.com>.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# This file is maintained in Automake, please report
+# bugs to <bug-automake at gnu.org> or send patches to
+# <automake-patches at gnu.org>.
+
+case $1 in
+ '')
+ echo "$0: No command. Try \`$0 --help' for more information." 1>&2
+ exit 1;
+ ;;
+ -h | --h*)
+ cat <<\EOF
+Usage: compile [--help] [--version] PROGRAM [ARGS]
+
+Wrapper for compilers which do not understand `-c -o'.
+Remove `-o dest.o' from ARGS, run PROGRAM with the remaining
+arguments, and rename the output as expected.
+
+If you are trying to build a whole package this is not the
+right script to run: please start by reading the file `INSTALL'.
+
+Report bugs to <bug-automake at gnu.org>.
+EOF
+ exit $?
+ ;;
+ -v | --v*)
+ echo "compile $scriptversion"
+ exit $?
+ ;;
+esac
+
+ofile=
+cfile=
+eat=
+
+for arg
+do
+ if test -n "$eat"; then
+ eat=
+ else
+ case $1 in
+ -o)
+ # configure might choose to run compile as `compile cc -o foo foo.c'.
+ # So we strip `-o arg' only if arg is an object.
+ eat=1
+ case $2 in
+ *.o | *.obj)
+ ofile=$2
+ ;;
+ *)
+ set x "$@" -o "$2"
+ shift
+ ;;
+ esac
+ ;;
+ *.c)
+ cfile=$1
+ set x "$@" "$1"
+ shift
+ ;;
+ *)
+ set x "$@" "$1"
+ shift
+ ;;
+ esac
+ fi
+ shift
+done
+
+if test -z "$ofile" || test -z "$cfile"; then
+ # If no `-o' option was seen then we might have been invoked from a
+ # pattern rule where we don't need one. That is ok -- this is a
+ # normal compilation that the losing compiler can handle. If no
+ # `.c' file was seen then we are probably linking. That is also
+ # ok.
+ exec "$@"
+fi
+
+# Name of file we expect compiler to create.
+cofile=`echo "$cfile" | sed 's|^.*[\\/]||; s|^[a-zA-Z]:||; s/\.c$/.o/'`
+
+# Create the lock directory.
+# Note: use `[/\\:.-]' here to ensure that we don't use the same name
+# that we are using for the .o file. Also, base the name on the expected
+# object file name, since that is what matters with a parallel build.
+lockdir=`echo "$cofile" | sed -e 's|[/\\:.-]|_|g'`.d
+while true; do
+ if mkdir "$lockdir" >/dev/null 2>&1; then
+ break
+ fi
+ sleep 1
+done
+# FIXME: race condition here if user kills between mkdir and trap.
+trap "rmdir '$lockdir'; exit 1" 1 2 15
+
+# Run the compile.
+"$@"
+ret=$?
+
+if test -f "$cofile"; then
+ test "$cofile" = "$ofile" || mv "$cofile" "$ofile"
+elif test -f "${cofile}bj"; then
+ test "${cofile}bj" = "$ofile" || mv "${cofile}bj" "$ofile"
+fi
+
+rmdir "$lockdir"
+exit $ret
+
+# Local Variables:
+# mode: shell-script
+# sh-indentation: 2
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "scriptversion="
+# time-stamp-format: "%:y-%02m-%02d.%02H"
+# time-stamp-time-zone: "UTC"
+# time-stamp-end: "; # UTC"
+# End:
diff --git a/config/config.guess b/config/config.guess
new file mode 100755
index 0000000..dc84c68
--- /dev/null
+++ b/config/config.guess
@@ -0,0 +1,1501 @@
+#! /bin/sh
+# Attempt to guess a canonical system name.
+# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
+# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009
+# Free Software Foundation, Inc.
+
+timestamp='2009-11-20'
+
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+
+# Originally written by Per Bothner. Please send patches (context
+# diff format) to <config-patches at gnu.org> and include a ChangeLog
+# entry.
+#
+# This script attempts to guess a canonical system name similar to
+# config.sub. If it succeeds, it prints the system name on stdout, and
+# exits with 0. Otherwise, it exits with 1.
+#
+# You can get the latest version of this script from:
+# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
+
+me=`echo "$0" | sed -e 's,.*/,,'`
+
+usage="\
+Usage: $0 [OPTION]
+
+Output the configuration name of the system \`$me' is run on.
+
+Operation modes:
+ -h, --help print this help, then exit
+ -t, --time-stamp print date of last modification, then exit
+ -v, --version print version number, then exit
+
+Report bugs and patches to <config-patches at gnu.org>."
+
+version="\
+GNU config.guess ($timestamp)
+
+Originally written by Per Bothner.
+Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001,
+2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
+
+This is free software; see the source for copying conditions. There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
+
+help="
+Try \`$me --help' for more information."
+
+# Parse command line
+while test $# -gt 0 ; do
+ case $1 in
+ --time-stamp | --time* | -t )
+ echo "$timestamp" ; exit ;;
+ --version | -v )
+ echo "$version" ; exit ;;
+ --help | --h* | -h )
+ echo "$usage"; exit ;;
+ -- ) # Stop option processing
+ shift; break ;;
+ - ) # Use stdin as input.
+ break ;;
+ -* )
+ echo "$me: invalid option $1$help" >&2
+ exit 1 ;;
+ * )
+ break ;;
+ esac
+done
+
+if test $# != 0; then
+ echo "$me: too many arguments$help" >&2
+ exit 1
+fi
+
+trap 'exit 1' 1 2 15
+
+# CC_FOR_BUILD -- compiler used by this script. Note that the use of a
+# compiler to aid in system detection is discouraged as it requires
+# temporary files to be created and, as you can see below, it is a
+# headache to deal with in a portable fashion.
+
+# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still
+# use `HOST_CC' if defined, but it is deprecated.
+
+# Portable tmp directory creation inspired by the Autoconf team.
+
+set_cc_for_build='
+trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ;
+trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ;
+: ${TMPDIR=/tmp} ;
+ { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } ||
+ { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } ||
+ { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } ||
+ { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ;
+dummy=$tmp/dummy ;
+tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ;
+case $CC_FOR_BUILD,$HOST_CC,$CC in
+ ,,) echo "int x;" > $dummy.c ;
+ for c in cc gcc c89 c99 ; do
+ if ($c -c -o $dummy.o $dummy.c) >/dev/null 2>&1 ; then
+ CC_FOR_BUILD="$c"; break ;
+ fi ;
+ done ;
+ if test x"$CC_FOR_BUILD" = x ; then
+ CC_FOR_BUILD=no_compiler_found ;
+ fi
+ ;;
+ ,,*) CC_FOR_BUILD=$CC ;;
+ ,*,*) CC_FOR_BUILD=$HOST_CC ;;
+esac ; set_cc_for_build= ;'
+
+# This is needed to find uname on a Pyramid OSx when run in the BSD universe.
+# (ghazi at noc.rutgers.edu 1994-08-24)
+if (test -f /.attbin/uname) >/dev/null 2>&1 ; then
+ PATH=$PATH:/.attbin ; export PATH
+fi
+
+UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown
+UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
+UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown
+UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
+
+# Note: order is significant - the case branches are not exclusive.
+
+case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+ *:NetBSD:*:*)
+ # NetBSD (nbsd) targets should (where applicable) match one or
+ # more of the tupples: *-*-netbsdelf*, *-*-netbsdaout*,
+ # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently
+ # switched to ELF, *-*-netbsd* would select the old
+ # object file format. This provides both forward
+ # compatibility and a consistent mechanism for selecting the
+ # object file format.
+ #
+ # Note: NetBSD doesn't particularly care about the vendor
+ # portion of the name. We always set it to "unknown".
+ sysctl="sysctl -n hw.machine_arch"
+ UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \
+ /usr/sbin/$sysctl 2>/dev/null || echo unknown)`
+ case "${UNAME_MACHINE_ARCH}" in
+ armeb) machine=armeb-unknown ;;
+ arm*) machine=arm-unknown ;;
+ sh3el) machine=shl-unknown ;;
+ sh3eb) machine=sh-unknown ;;
+ sh5el) machine=sh5le-unknown ;;
+ *) machine=${UNAME_MACHINE_ARCH}-unknown ;;
+ esac
+ # The Operating System including object format, if it has switched
+ # to ELF recently, or will in the future.
+ case "${UNAME_MACHINE_ARCH}" in
+ arm*|i386|m68k|ns32k|sh3*|sparc|vax)
+ eval $set_cc_for_build
+ if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \
+ | grep -q __ELF__
+ then
+ # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout).
+ # Return netbsd for either. FIX?
+ os=netbsd
+ else
+ os=netbsdelf
+ fi
+ ;;
+ *)
+ os=netbsd
+ ;;
+ esac
+ # The OS release
+ # Debian GNU/NetBSD machines have a different userland, and
+ # thus, need a distinct triplet. However, they do not need
+ # kernel version information, so it can be replaced with a
+ # suitable tag, in the style of linux-gnu.
+ case "${UNAME_VERSION}" in
+ Debian*)
+ release='-gnu'
+ ;;
+ *)
+ release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+ ;;
+ esac
+ # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM:
+ # contains redundant information, the shorter form:
+ # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used.
+ echo "${machine}-${os}${release}"
+ exit ;;
+ *:OpenBSD:*:*)
+ UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'`
+ echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE}
+ exit ;;
+ *:ekkoBSD:*:*)
+ echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE}
+ exit ;;
+ *:SolidBSD:*:*)
+ echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE}
+ exit ;;
+ macppc:MirBSD:*:*)
+ echo powerpc-unknown-mirbsd${UNAME_RELEASE}
+ exit ;;
+ *:MirBSD:*:*)
+ echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE}
+ exit ;;
+ alpha:OSF1:*:*)
+ case $UNAME_RELEASE in
+ *4.0)
+ UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'`
+ ;;
+ *5.*)
+ UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'`
+ ;;
+ esac
+ # According to Compaq, /usr/sbin/psrinfo has been available on
+ # OSF/1 and Tru64 systems produced since 1995. I hope that
+ # covers most systems running today. This code pipes the CPU
+ # types through head -n 1, so we only detect the type of CPU 0.
+ ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1`
+ case "$ALPHA_CPU_TYPE" in
+ "EV4 (21064)")
+ UNAME_MACHINE="alpha" ;;
+ "EV4.5 (21064)")
+ UNAME_MACHINE="alpha" ;;
+ "LCA4 (21066/21068)")
+ UNAME_MACHINE="alpha" ;;
+ "EV5 (21164)")
+ UNAME_MACHINE="alphaev5" ;;
+ "EV5.6 (21164A)")
+ UNAME_MACHINE="alphaev56" ;;
+ "EV5.6 (21164PC)")
+ UNAME_MACHINE="alphapca56" ;;
+ "EV5.7 (21164PC)")
+ UNAME_MACHINE="alphapca57" ;;
+ "EV6 (21264)")
+ UNAME_MACHINE="alphaev6" ;;
+ "EV6.7 (21264A)")
+ UNAME_MACHINE="alphaev67" ;;
+ "EV6.8CB (21264C)")
+ UNAME_MACHINE="alphaev68" ;;
+ "EV6.8AL (21264B)")
+ UNAME_MACHINE="alphaev68" ;;
+ "EV6.8CX (21264D)")
+ UNAME_MACHINE="alphaev68" ;;
+ "EV6.9A (21264/EV69A)")
+ UNAME_MACHINE="alphaev69" ;;
+ "EV7 (21364)")
+ UNAME_MACHINE="alphaev7" ;;
+ "EV7.9 (21364A)")
+ UNAME_MACHINE="alphaev79" ;;
+ esac
+ # A Pn.n version is a patched version.
+ # A Vn.n version is a released version.
+ # A Tn.n version is a released field test version.
+ # A Xn.n version is an unreleased experimental baselevel.
+ # 1.2 uses "1.2" for uname -r.
+ echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
+ exit ;;
+ Alpha\ *:Windows_NT*:*)
+ # How do we know it's Interix rather than the generic POSIX subsystem?
+ # Should we change UNAME_MACHINE based on the output of uname instead
+ # of the specific Alpha model?
+ echo alpha-pc-interix
+ exit ;;
+ 21064:Windows_NT:50:3)
+ echo alpha-dec-winnt3.5
+ exit ;;
+ Amiga*:UNIX_System_V:4.0:*)
+ echo m68k-unknown-sysv4
+ exit ;;
+ *:[Aa]miga[Oo][Ss]:*:*)
+ echo ${UNAME_MACHINE}-unknown-amigaos
+ exit ;;
+ *:[Mm]orph[Oo][Ss]:*:*)
+ echo ${UNAME_MACHINE}-unknown-morphos
+ exit ;;
+ *:OS/390:*:*)
+ echo i370-ibm-openedition
+ exit ;;
+ *:z/VM:*:*)
+ echo s390-ibm-zvmoe
+ exit ;;
+ *:OS400:*:*)
+ echo powerpc-ibm-os400
+ exit ;;
+ arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
+ echo arm-acorn-riscix${UNAME_RELEASE}
+ exit ;;
+ arm:riscos:*:*|arm:RISCOS:*:*)
+ echo arm-unknown-riscos
+ exit ;;
+ SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*)
+ echo hppa1.1-hitachi-hiuxmpp
+ exit ;;
+ Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*)
+ # akee at wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE.
+ if test "`(/bin/universe) 2>/dev/null`" = att ; then
+ echo pyramid-pyramid-sysv3
+ else
+ echo pyramid-pyramid-bsd
+ fi
+ exit ;;
+ NILE*:*:*:dcosx)
+ echo pyramid-pyramid-svr4
+ exit ;;
+ DRS?6000:unix:4.0:6*)
+ echo sparc-icl-nx6
+ exit ;;
+ DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*)
+ case `/usr/bin/uname -p` in
+ sparc) echo sparc-icl-nx7; exit ;;
+ esac ;;
+ s390x:SunOS:*:*)
+ echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+ exit ;;
+ sun4H:SunOS:5.*:*)
+ echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+ exit ;;
+ sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*)
+ echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+ exit ;;
+ i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*)
+ echo i386-pc-auroraux${UNAME_RELEASE}
+ exit ;;
+ i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*)
+ eval $set_cc_for_build
+ SUN_ARCH="i386"
+ # If there is a compiler, see if it is configured for 64-bit objects.
+ # Note that the Sun cc does not turn __LP64__ into 1 like gcc does.
+ # This test works for both compilers.
+ if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
+ if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \
+ (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
+ grep IS_64BIT_ARCH >/dev/null
+ then
+ SUN_ARCH="x86_64"
+ fi
+ fi
+ echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+ exit ;;
+ sun4*:SunOS:6*:*)
+ # According to config.sub, this is the proper way to canonicalize
+ # SunOS6. Hard to guess exactly what SunOS6 will be like, but
+ # it's likely to be more like Solaris than SunOS4.
+ echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+ exit ;;
+ sun4*:SunOS:*:*)
+ case "`/usr/bin/arch -k`" in
+ Series*|S4*)
+ UNAME_RELEASE=`uname -v`
+ ;;
+ esac
+ # Japanese Language versions have a version number like `4.1.3-JL'.
+ echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'`
+ exit ;;
+ sun3*:SunOS:*:*)
+ echo m68k-sun-sunos${UNAME_RELEASE}
+ exit ;;
+ sun*:*:4.2BSD:*)
+ UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null`
+ test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3
+ case "`/bin/arch`" in
+ sun3)
+ echo m68k-sun-sunos${UNAME_RELEASE}
+ ;;
+ sun4)
+ echo sparc-sun-sunos${UNAME_RELEASE}
+ ;;
+ esac
+ exit ;;
+ aushp:SunOS:*:*)
+ echo sparc-auspex-sunos${UNAME_RELEASE}
+ exit ;;
+ # The situation for MiNT is a little confusing. The machine name
+ # can be virtually everything (everything which is not
+ # "atarist" or "atariste" at least should have a processor
+ # > m68000). The system name ranges from "MiNT" over "FreeMiNT"
+ # to the lowercase version "mint" (or "freemint"). Finally
+ # the system name "TOS" denotes a system which is actually not
+ # MiNT. But MiNT is downward compatible to TOS, so this should
+ # be no problem.
+ atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*)
+ echo m68k-atari-mint${UNAME_RELEASE}
+ exit ;;
+ atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*)
+ echo m68k-atari-mint${UNAME_RELEASE}
+ exit ;;
+ *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*)
+ echo m68k-atari-mint${UNAME_RELEASE}
+ exit ;;
+ milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*)
+ echo m68k-milan-mint${UNAME_RELEASE}
+ exit ;;
+ hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*)
+ echo m68k-hades-mint${UNAME_RELEASE}
+ exit ;;
+ *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*)
+ echo m68k-unknown-mint${UNAME_RELEASE}
+ exit ;;
+ m68k:machten:*:*)
+ echo m68k-apple-machten${UNAME_RELEASE}
+ exit ;;
+ powerpc:machten:*:*)
+ echo powerpc-apple-machten${UNAME_RELEASE}
+ exit ;;
+ RISC*:Mach:*:*)
+ echo mips-dec-mach_bsd4.3
+ exit ;;
+ RISC*:ULTRIX:*:*)
+ echo mips-dec-ultrix${UNAME_RELEASE}
+ exit ;;
+ VAX*:ULTRIX*:*:*)
+ echo vax-dec-ultrix${UNAME_RELEASE}
+ exit ;;
+ 2020:CLIX:*:* | 2430:CLIX:*:*)
+ echo clipper-intergraph-clix${UNAME_RELEASE}
+ exit ;;
+ mips:*:*:UMIPS | mips:*:*:RISCos)
+ eval $set_cc_for_build
+ sed 's/^ //' << EOF >$dummy.c
+#ifdef __cplusplus
+#include <stdio.h> /* for printf() prototype */
+ int main (int argc, char *argv[]) {
+#else
+ int main (argc, argv) int argc; char *argv[]; {
+#endif
+ #if defined (host_mips) && defined (MIPSEB)
+ #if defined (SYSTYPE_SYSV)
+ printf ("mips-mips-riscos%ssysv\n", argv[1]); exit (0);
+ #endif
+ #if defined (SYSTYPE_SVR4)
+ printf ("mips-mips-riscos%ssvr4\n", argv[1]); exit (0);
+ #endif
+ #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD)
+ printf ("mips-mips-riscos%sbsd\n", argv[1]); exit (0);
+ #endif
+ #endif
+ exit (-1);
+ }
+EOF
+ $CC_FOR_BUILD -o $dummy $dummy.c &&
+ dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` &&
+ SYSTEM_NAME=`$dummy $dummyarg` &&
+ { echo "$SYSTEM_NAME"; exit; }
+ echo mips-mips-riscos${UNAME_RELEASE}
+ exit ;;
+ Motorola:PowerMAX_OS:*:*)
+ echo powerpc-motorola-powermax
+ exit ;;
+ Motorola:*:4.3:PL8-*)
+ echo powerpc-harris-powermax
+ exit ;;
+ Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*)
+ echo powerpc-harris-powermax
+ exit ;;
+ Night_Hawk:Power_UNIX:*:*)
+ echo powerpc-harris-powerunix
+ exit ;;
+ m88k:CX/UX:7*:*)
+ echo m88k-harris-cxux7
+ exit ;;
+ m88k:*:4*:R4*)
+ echo m88k-motorola-sysv4
+ exit ;;
+ m88k:*:3*:R3*)
+ echo m88k-motorola-sysv3
+ exit ;;
+ AViiON:dgux:*:*)
+ # DG/UX returns AViiON for all architectures
+ UNAME_PROCESSOR=`/usr/bin/uname -p`
+ if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ]
+ then
+ if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \
+ [ ${TARGET_BINARY_INTERFACE}x = x ]
+ then
+ echo m88k-dg-dgux${UNAME_RELEASE}
+ else
+ echo m88k-dg-dguxbcs${UNAME_RELEASE}
+ fi
+ else
+ echo i586-dg-dgux${UNAME_RELEASE}
+ fi
+ exit ;;
+ M88*:DolphinOS:*:*) # DolphinOS (SVR3)
+ echo m88k-dolphin-sysv3
+ exit ;;
+ M88*:*:R3*:*)
+ # Delta 88k system running SVR3
+ echo m88k-motorola-sysv3
+ exit ;;
+ XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3)
+ echo m88k-tektronix-sysv3
+ exit ;;
+ Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD)
+ echo m68k-tektronix-bsd
+ exit ;;
+ *:IRIX*:*:*)
+ echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'`
+ exit ;;
+ ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX.
+ echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id
+ exit ;; # Note that: echo "'`uname -s`'" gives 'AIX '
+ i*86:AIX:*:*)
+ echo i386-ibm-aix
+ exit ;;
+ ia64:AIX:*:*)
+ if [ -x /usr/bin/oslevel ] ; then
+ IBM_REV=`/usr/bin/oslevel`
+ else
+ IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE}
+ fi
+ echo ${UNAME_MACHINE}-ibm-aix${IBM_REV}
+ exit ;;
+ *:AIX:2:3)
+ if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then
+ eval $set_cc_for_build
+ sed 's/^ //' << EOF >$dummy.c
+ #include <sys/systemcfg.h>
+
+ main()
+ {
+ if (!__power_pc())
+ exit(1);
+ puts("powerpc-ibm-aix3.2.5");
+ exit(0);
+ }
+EOF
+ if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy`
+ then
+ echo "$SYSTEM_NAME"
+ else
+ echo rs6000-ibm-aix3.2.5
+ fi
+ elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then
+ echo rs6000-ibm-aix3.2.4
+ else
+ echo rs6000-ibm-aix3.2
+ fi
+ exit ;;
+ *:AIX:*:[456])
+ IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'`
+ if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then
+ IBM_ARCH=rs6000
+ else
+ IBM_ARCH=powerpc
+ fi
+ if [ -x /usr/bin/oslevel ] ; then
+ IBM_REV=`/usr/bin/oslevel`
+ else
+ IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE}
+ fi
+ echo ${IBM_ARCH}-ibm-aix${IBM_REV}
+ exit ;;
+ *:AIX:*:*)
+ echo rs6000-ibm-aix
+ exit ;;
+ ibmrt:4.4BSD:*|romp-ibm:BSD:*)
+ echo romp-ibm-bsd4.4
+ exit ;;
+ ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and
+ echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to
+ exit ;; # report: romp-ibm BSD 4.3
+ *:BOSX:*:*)
+ echo rs6000-bull-bosx
+ exit ;;
+ DPX/2?00:B.O.S.:*:*)
+ echo m68k-bull-sysv3
+ exit ;;
+ 9000/[34]??:4.3bsd:1.*:*)
+ echo m68k-hp-bsd
+ exit ;;
+ hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*)
+ echo m68k-hp-bsd4.4
+ exit ;;
+ 9000/[34678]??:HP-UX:*:*)
+ HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'`
+ case "${UNAME_MACHINE}" in
+ 9000/31? ) HP_ARCH=m68000 ;;
+ 9000/[34]?? ) HP_ARCH=m68k ;;
+ 9000/[678][0-9][0-9])
+ if [ -x /usr/bin/getconf ]; then
+ sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null`
+ sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null`
+ case "${sc_cpu_version}" in
+ 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0
+ 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1
+ 532) # CPU_PA_RISC2_0
+ case "${sc_kernel_bits}" in
+ 32) HP_ARCH="hppa2.0n" ;;
+ 64) HP_ARCH="hppa2.0w" ;;
+ '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20
+ esac ;;
+ esac
+ fi
+ if [ "${HP_ARCH}" = "" ]; then
+ eval $set_cc_for_build
+ sed 's/^ //' << EOF >$dummy.c
+
+ #define _HPUX_SOURCE
+ #include <stdlib.h>
+ #include <unistd.h>
+
+ int main ()
+ {
+ #if defined(_SC_KERNEL_BITS)
+ long bits = sysconf(_SC_KERNEL_BITS);
+ #endif
+ long cpu = sysconf (_SC_CPU_VERSION);
+
+ switch (cpu)
+ {
+ case CPU_PA_RISC1_0: puts ("hppa1.0"); break;
+ case CPU_PA_RISC1_1: puts ("hppa1.1"); break;
+ case CPU_PA_RISC2_0:
+ #if defined(_SC_KERNEL_BITS)
+ switch (bits)
+ {
+ case 64: puts ("hppa2.0w"); break;
+ case 32: puts ("hppa2.0n"); break;
+ default: puts ("hppa2.0"); break;
+ } break;
+ #else /* !defined(_SC_KERNEL_BITS) */
+ puts ("hppa2.0"); break;
+ #endif
+ default: puts ("hppa1.0"); break;
+ }
+ exit (0);
+ }
+EOF
+ (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy`
+ test -z "$HP_ARCH" && HP_ARCH=hppa
+ fi ;;
+ esac
+ if [ ${HP_ARCH} = "hppa2.0w" ]
+ then
+ eval $set_cc_for_build
+
+ # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating
+ # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler
+ # generating 64-bit code. GNU and HP use different nomenclature:
+ #
+ # $ CC_FOR_BUILD=cc ./config.guess
+ # => hppa2.0w-hp-hpux11.23
+ # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess
+ # => hppa64-hp-hpux11.23
+
+ if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) |
+ grep -q __LP64__
+ then
+ HP_ARCH="hppa2.0w"
+ else
+ HP_ARCH="hppa64"
+ fi
+ fi
+ echo ${HP_ARCH}-hp-hpux${HPUX_REV}
+ exit ;;
+ ia64:HP-UX:*:*)
+ HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'`
+ echo ia64-hp-hpux${HPUX_REV}
+ exit ;;
+ 3050*:HI-UX:*:*)
+ eval $set_cc_for_build
+ sed 's/^ //' << EOF >$dummy.c
+ #include <unistd.h>
+ int
+ main ()
+ {
+ long cpu = sysconf (_SC_CPU_VERSION);
+ /* The order matters, because CPU_IS_HP_MC68K erroneously returns
+ true for CPU_PA_RISC1_0. CPU_IS_PA_RISC returns correct
+ results, however. */
+ if (CPU_IS_PA_RISC (cpu))
+ {
+ switch (cpu)
+ {
+ case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break;
+ case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break;
+ case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break;
+ default: puts ("hppa-hitachi-hiuxwe2"); break;
+ }
+ }
+ else if (CPU_IS_HP_MC68K (cpu))
+ puts ("m68k-hitachi-hiuxwe2");
+ else puts ("unknown-hitachi-hiuxwe2");
+ exit (0);
+ }
+EOF
+ $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` &&
+ { echo "$SYSTEM_NAME"; exit; }
+ echo unknown-hitachi-hiuxwe2
+ exit ;;
+ 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* )
+ echo hppa1.1-hp-bsd
+ exit ;;
+ 9000/8??:4.3bsd:*:*)
+ echo hppa1.0-hp-bsd
+ exit ;;
+ *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*)
+ echo hppa1.0-hp-mpeix
+ exit ;;
+ hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* )
+ echo hppa1.1-hp-osf
+ exit ;;
+ hp8??:OSF1:*:*)
+ echo hppa1.0-hp-osf
+ exit ;;
+ i*86:OSF1:*:*)
+ if [ -x /usr/sbin/sysversion ] ; then
+ echo ${UNAME_MACHINE}-unknown-osf1mk
+ else
+ echo ${UNAME_MACHINE}-unknown-osf1
+ fi
+ exit ;;
+ parisc*:Lites*:*:*)
+ echo hppa1.1-hp-lites
+ exit ;;
+ C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*)
+ echo c1-convex-bsd
+ exit ;;
+ C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*)
+ if getsysinfo -f scalar_acc
+ then echo c32-convex-bsd
+ else echo c2-convex-bsd
+ fi
+ exit ;;
+ C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*)
+ echo c34-convex-bsd
+ exit ;;
+ C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*)
+ echo c38-convex-bsd
+ exit ;;
+ C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*)
+ echo c4-convex-bsd
+ exit ;;
+ CRAY*Y-MP:*:*:*)
+ echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+ exit ;;
+ CRAY*[A-Z]90:*:*:*)
+ echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \
+ | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \
+ -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \
+ -e 's/\.[^.]*$/.X/'
+ exit ;;
+ CRAY*TS:*:*:*)
+ echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+ exit ;;
+ CRAY*T3E:*:*:*)
+ echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+ exit ;;
+ CRAY*SV1:*:*:*)
+ echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+ exit ;;
+ *:UNICOS/mp:*:*)
+ echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+ exit ;;
+ F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*)
+ FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
+ FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
+ FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'`
+ echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
+ exit ;;
+ 5000:UNIX_System_V:4.*:*)
+ FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
+ FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'`
+ echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
+ exit ;;
+ i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*)
+ echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE}
+ exit ;;
+ sparc*:BSD/OS:*:*)
+ echo sparc-unknown-bsdi${UNAME_RELEASE}
+ exit ;;
+ *:BSD/OS:*:*)
+ echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE}
+ exit ;;
+ *:FreeBSD:*:*)
+ case ${UNAME_MACHINE} in
+ pc98)
+ echo i386-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;;
+ amd64)
+ echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;;
+ *)
+ echo ${UNAME_MACHINE}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;;
+ esac
+ exit ;;
+ i*:CYGWIN*:*)
+ echo ${UNAME_MACHINE}-pc-cygwin
+ exit ;;
+ *:MINGW*:*)
+ echo ${UNAME_MACHINE}-pc-mingw32
+ exit ;;
+ i*:windows32*:*)
+ # uname -m includes "-pc" on this system.
+ echo ${UNAME_MACHINE}-mingw32
+ exit ;;
+ i*:PW*:*)
+ echo ${UNAME_MACHINE}-pc-pw32
+ exit ;;
+ *:Interix*:*)
+ case ${UNAME_MACHINE} in
+ x86)
+ echo i586-pc-interix${UNAME_RELEASE}
+ exit ;;
+ authenticamd | genuineintel | EM64T)
+ echo x86_64-unknown-interix${UNAME_RELEASE}
+ exit ;;
+ IA64)
+ echo ia64-unknown-interix${UNAME_RELEASE}
+ exit ;;
+ esac ;;
+ [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*)
+ echo i${UNAME_MACHINE}-pc-mks
+ exit ;;
+ 8664:Windows_NT:*)
+ echo x86_64-pc-mks
+ exit ;;
+ i*:Windows_NT*:* | Pentium*:Windows_NT*:*)
+ # How do we know it's Interix rather than the generic POSIX subsystem?
+ # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we
+ # UNAME_MACHINE based on the output of uname instead of i386?
+ echo i586-pc-interix
+ exit ;;
+ i*:UWIN*:*)
+ echo ${UNAME_MACHINE}-pc-uwin
+ exit ;;
+ amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*)
+ echo x86_64-unknown-cygwin
+ exit ;;
+ p*:CYGWIN*:*)
+ echo powerpcle-unknown-cygwin
+ exit ;;
+ prep*:SunOS:5.*:*)
+ echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+ exit ;;
+ *:GNU:*:*)
+ # the GNU system
+ echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
+ exit ;;
+ *:GNU/*:*:*)
+ # other systems with GNU libc and userland
+ echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu
+ exit ;;
+ i*86:Minix:*:*)
+ echo ${UNAME_MACHINE}-pc-minix
+ exit ;;
+ alpha:Linux:*:*)
+ case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in
+ EV5) UNAME_MACHINE=alphaev5 ;;
+ EV56) UNAME_MACHINE=alphaev56 ;;
+ PCA56) UNAME_MACHINE=alphapca56 ;;
+ PCA57) UNAME_MACHINE=alphapca56 ;;
+ EV6) UNAME_MACHINE=alphaev6 ;;
+ EV67) UNAME_MACHINE=alphaev67 ;;
+ EV68*) UNAME_MACHINE=alphaev68 ;;
+ esac
+ objdump --private-headers /bin/sh | grep -q ld.so.1
+ if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi
+ echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC}
+ exit ;;
+ arm*:Linux:*:*)
+ eval $set_cc_for_build
+ if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \
+ | grep -q __ARM_EABI__
+ then
+ echo ${UNAME_MACHINE}-unknown-linux-gnu
+ else
+ echo ${UNAME_MACHINE}-unknown-linux-gnueabi
+ fi
+ exit ;;
+ avr32*:Linux:*:*)
+ echo ${UNAME_MACHINE}-unknown-linux-gnu
+ exit ;;
+ cris:Linux:*:*)
+ echo cris-axis-linux-gnu
+ exit ;;
+ crisv32:Linux:*:*)
+ echo crisv32-axis-linux-gnu
+ exit ;;
+ frv:Linux:*:*)
+ echo frv-unknown-linux-gnu
+ exit ;;
+ i*86:Linux:*:*)
+ LIBC=gnu
+ eval $set_cc_for_build
+ sed 's/^ //' << EOF >$dummy.c
+ #ifdef __dietlibc__
+ LIBC=dietlibc
+ #endif
+EOF
+ eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'`
+ echo "${UNAME_MACHINE}-pc-linux-${LIBC}"
+ exit ;;
+ ia64:Linux:*:*)
+ echo ${UNAME_MACHINE}-unknown-linux-gnu
+ exit ;;
+ m32r*:Linux:*:*)
+ echo ${UNAME_MACHINE}-unknown-linux-gnu
+ exit ;;
+ m68*:Linux:*:*)
+ echo ${UNAME_MACHINE}-unknown-linux-gnu
+ exit ;;
+ mips:Linux:*:* | mips64:Linux:*:*)
+ eval $set_cc_for_build
+ sed 's/^ //' << EOF >$dummy.c
+ #undef CPU
+ #undef ${UNAME_MACHINE}
+ #undef ${UNAME_MACHINE}el
+ #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL)
+ CPU=${UNAME_MACHINE}el
+ #else
+ #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB)
+ CPU=${UNAME_MACHINE}
+ #else
+ CPU=
+ #endif
+ #endif
+EOF
+ eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'`
+ test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; }
+ ;;
+ or32:Linux:*:*)
+ echo or32-unknown-linux-gnu
+ exit ;;
+ padre:Linux:*:*)
+ echo sparc-unknown-linux-gnu
+ exit ;;
+ parisc64:Linux:*:* | hppa64:Linux:*:*)
+ echo hppa64-unknown-linux-gnu
+ exit ;;
+ parisc:Linux:*:* | hppa:Linux:*:*)
+ # Look for CPU level
+ case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in
+ PA7*) echo hppa1.1-unknown-linux-gnu ;;
+ PA8*) echo hppa2.0-unknown-linux-gnu ;;
+ *) echo hppa-unknown-linux-gnu ;;
+ esac
+ exit ;;
+ ppc64:Linux:*:*)
+ echo powerpc64-unknown-linux-gnu
+ exit ;;
+ ppc:Linux:*:*)
+ echo powerpc-unknown-linux-gnu
+ exit ;;
+ s390:Linux:*:* | s390x:Linux:*:*)
+ echo ${UNAME_MACHINE}-ibm-linux
+ exit ;;
+ sh64*:Linux:*:*)
+ echo ${UNAME_MACHINE}-unknown-linux-gnu
+ exit ;;
+ sh*:Linux:*:*)
+ echo ${UNAME_MACHINE}-unknown-linux-gnu
+ exit ;;
+ sparc:Linux:*:* | sparc64:Linux:*:*)
+ echo ${UNAME_MACHINE}-unknown-linux-gnu
+ exit ;;
+ vax:Linux:*:*)
+ echo ${UNAME_MACHINE}-dec-linux-gnu
+ exit ;;
+ x86_64:Linux:*:*)
+ echo x86_64-unknown-linux-gnu
+ exit ;;
+ xtensa*:Linux:*:*)
+ echo ${UNAME_MACHINE}-unknown-linux-gnu
+ exit ;;
+ i*86:DYNIX/ptx:4*:*)
+ # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.
+ # earlier versions are messed up and put the nodename in both
+ # sysname and nodename.
+ echo i386-sequent-sysv4
+ exit ;;
+ i*86:UNIX_SV:4.2MP:2.*)
+ # Unixware is an offshoot of SVR4, but it has its own version
+ # number series starting with 2...
+ # I am not positive that other SVR4 systems won't match this,
+ # I just have to hope. -- rms.
+ # Use sysv4.2uw... so that sysv4* matches it.
+ echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION}
+ exit ;;
+ i*86:OS/2:*:*)
+ # If we were able to find `uname', then EMX Unix compatibility
+ # is probably installed.
+ echo ${UNAME_MACHINE}-pc-os2-emx
+ exit ;;
+ i*86:XTS-300:*:STOP)
+ echo ${UNAME_MACHINE}-unknown-stop
+ exit ;;
+ i*86:atheos:*:*)
+ echo ${UNAME_MACHINE}-unknown-atheos
+ exit ;;
+ i*86:syllable:*:*)
+ echo ${UNAME_MACHINE}-pc-syllable
+ exit ;;
+ i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*)
+ echo i386-unknown-lynxos${UNAME_RELEASE}
+ exit ;;
+ i*86:*DOS:*:*)
+ echo ${UNAME_MACHINE}-pc-msdosdjgpp
+ exit ;;
+ i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*)
+ UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'`
+ if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then
+ echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL}
+ else
+ echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL}
+ fi
+ exit ;;
+ i*86:*:5:[678]*)
+ # UnixWare 7.x, OpenUNIX and OpenServer 6.
+ case `/bin/uname -X | grep "^Machine"` in
+ *486*) UNAME_MACHINE=i486 ;;
+ *Pentium) UNAME_MACHINE=i586 ;;
+ *Pent*|*Celeron) UNAME_MACHINE=i686 ;;
+ esac
+ echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION}
+ exit ;;
+ i*86:*:3.2:*)
+ if test -f /usr/options/cb.name; then
+ UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name`
+ echo ${UNAME_MACHINE}-pc-isc$UNAME_REL
+ elif /bin/uname -X 2>/dev/null >/dev/null ; then
+ UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')`
+ (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486
+ (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \
+ && UNAME_MACHINE=i586
+ (/bin/uname -X|grep '^Machine.*Pent *II' >/dev/null) \
+ && UNAME_MACHINE=i686
+ (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \
+ && UNAME_MACHINE=i686
+ echo ${UNAME_MACHINE}-pc-sco$UNAME_REL
+ else
+ echo ${UNAME_MACHINE}-pc-sysv32
+ fi
+ exit ;;
+ pc:*:*:*)
+ # Left here for compatibility:
+ # uname -m prints for DJGPP always 'pc', but it prints nothing about
+ # the processor, so we play safe by assuming i586.
+ # Note: whatever this is, it MUST be the same as what config.sub
+ # prints for the "djgpp" host, or else GDB configury will decide that
+ # this is a cross-build.
+ echo i586-pc-msdosdjgpp
+ exit ;;
+ Intel:Mach:3*:*)
+ echo i386-pc-mach3
+ exit ;;
+ paragon:*:*:*)
+ echo i860-intel-osf1
+ exit ;;
+ i860:*:4.*:*) # i860-SVR4
+ if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then
+ echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4
+ else # Add other i860-SVR4 vendors below as they are discovered.
+ echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4
+ fi
+ exit ;;
+ mini*:CTIX:SYS*5:*)
+ # "miniframe"
+ echo m68010-convergent-sysv
+ exit ;;
+ mc68k:UNIX:SYSTEM5:3.51m)
+ echo m68k-convergent-sysv
+ exit ;;
+ M680?0:D-NIX:5.3:*)
+ echo m68k-diab-dnix
+ exit ;;
+ M68*:*:R3V[5678]*:*)
+ test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;;
+ 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0)
+ OS_REL=''
+ test -r /etc/.relid \
+ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid`
+ /bin/uname -p 2>/dev/null | grep 86 >/dev/null \
+ && { echo i486-ncr-sysv4.3${OS_REL}; exit; }
+ /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
+ && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;;
+ 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*)
+ /bin/uname -p 2>/dev/null | grep 86 >/dev/null \
+ && { echo i486-ncr-sysv4; exit; } ;;
+ NCR*:*:4.2:* | MPRAS*:*:4.2:*)
+ OS_REL='.3'
+ test -r /etc/.relid \
+ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid`
+ /bin/uname -p 2>/dev/null | grep 86 >/dev/null \
+ && { echo i486-ncr-sysv4.3${OS_REL}; exit; }
+ /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
+ && { echo i586-ncr-sysv4.3${OS_REL}; exit; }
+ /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \
+ && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;;
+ m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*)
+ echo m68k-unknown-lynxos${UNAME_RELEASE}
+ exit ;;
+ mc68030:UNIX_System_V:4.*:*)
+ echo m68k-atari-sysv4
+ exit ;;
+ TSUNAMI:LynxOS:2.*:*)
+ echo sparc-unknown-lynxos${UNAME_RELEASE}
+ exit ;;
+ rs6000:LynxOS:2.*:*)
+ echo rs6000-unknown-lynxos${UNAME_RELEASE}
+ exit ;;
+ PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*)
+ echo powerpc-unknown-lynxos${UNAME_RELEASE}
+ exit ;;
+ SM[BE]S:UNIX_SV:*:*)
+ echo mips-dde-sysv${UNAME_RELEASE}
+ exit ;;
+ RM*:ReliantUNIX-*:*:*)
+ echo mips-sni-sysv4
+ exit ;;
+ RM*:SINIX-*:*:*)
+ echo mips-sni-sysv4
+ exit ;;
+ *:SINIX-*:*:*)
+ if uname -p 2>/dev/null >/dev/null ; then
+ UNAME_MACHINE=`(uname -p) 2>/dev/null`
+ echo ${UNAME_MACHINE}-sni-sysv4
+ else
+ echo ns32k-sni-sysv
+ fi
+ exit ;;
+ PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort
+ # says <Richard.M.Bartel at ccMail.Census.GOV>
+ echo i586-unisys-sysv4
+ exit ;;
+ *:UNIX_System_V:4*:FTX*)
+ # From Gerald Hewes <hewes at openmarket.com>.
+ # How about differentiating between stratus architectures? -djm
+ echo hppa1.1-stratus-sysv4
+ exit ;;
+ *:*:*:FTX*)
+ # From seanf at swdc.stratus.com.
+ echo i860-stratus-sysv4
+ exit ;;
+ i*86:VOS:*:*)
+ # From Paul.Green at stratus.com.
+ echo ${UNAME_MACHINE}-stratus-vos
+ exit ;;
+ *:VOS:*:*)
+ # From Paul.Green at stratus.com.
+ echo hppa1.1-stratus-vos
+ exit ;;
+ mc68*:A/UX:*:*)
+ echo m68k-apple-aux${UNAME_RELEASE}
+ exit ;;
+ news*:NEWS-OS:6*:*)
+ echo mips-sony-newsos6
+ exit ;;
+ R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*)
+ if [ -d /usr/nec ]; then
+ echo mips-nec-sysv${UNAME_RELEASE}
+ else
+ echo mips-unknown-sysv${UNAME_RELEASE}
+ fi
+ exit ;;
+ BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only.
+ echo powerpc-be-beos
+ exit ;;
+ BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only.
+ echo powerpc-apple-beos
+ exit ;;
+ BePC:BeOS:*:*) # BeOS running on Intel PC compatible.
+ echo i586-pc-beos
+ exit ;;
+ BePC:Haiku:*:*) # Haiku running on Intel PC compatible.
+ echo i586-pc-haiku
+ exit ;;
+ SX-4:SUPER-UX:*:*)
+ echo sx4-nec-superux${UNAME_RELEASE}
+ exit ;;
+ SX-5:SUPER-UX:*:*)
+ echo sx5-nec-superux${UNAME_RELEASE}
+ exit ;;
+ SX-6:SUPER-UX:*:*)
+ echo sx6-nec-superux${UNAME_RELEASE}
+ exit ;;
+ SX-7:SUPER-UX:*:*)
+ echo sx7-nec-superux${UNAME_RELEASE}
+ exit ;;
+ SX-8:SUPER-UX:*:*)
+ echo sx8-nec-superux${UNAME_RELEASE}
+ exit ;;
+ SX-8R:SUPER-UX:*:*)
+ echo sx8r-nec-superux${UNAME_RELEASE}
+ exit ;;
+ Power*:Rhapsody:*:*)
+ echo powerpc-apple-rhapsody${UNAME_RELEASE}
+ exit ;;
+ *:Rhapsody:*:*)
+ echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE}
+ exit ;;
+ *:Darwin:*:*)
+ UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown
+ case $UNAME_PROCESSOR in
+ i386)
+ eval $set_cc_for_build
+ if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
+ if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
+ (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
+ grep IS_64BIT_ARCH >/dev/null
+ then
+ UNAME_PROCESSOR="x86_64"
+ fi
+ fi ;;
+ unknown) UNAME_PROCESSOR=powerpc ;;
+ esac
+ echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE}
+ exit ;;
+ *:procnto*:*:* | *:QNX:[0123456789]*:*)
+ UNAME_PROCESSOR=`uname -p`
+ if test "$UNAME_PROCESSOR" = "x86"; then
+ UNAME_PROCESSOR=i386
+ UNAME_MACHINE=pc
+ fi
+ echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE}
+ exit ;;
+ *:QNX:*:4*)
+ echo i386-pc-qnx
+ exit ;;
+ NSE-?:NONSTOP_KERNEL:*:*)
+ echo nse-tandem-nsk${UNAME_RELEASE}
+ exit ;;
+ NSR-?:NONSTOP_KERNEL:*:*)
+ echo nsr-tandem-nsk${UNAME_RELEASE}
+ exit ;;
+ *:NonStop-UX:*:*)
+ echo mips-compaq-nonstopux
+ exit ;;
+ BS2000:POSIX*:*:*)
+ echo bs2000-siemens-sysv
+ exit ;;
+ DS/*:UNIX_System_V:*:*)
+ echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE}
+ exit ;;
+ *:Plan9:*:*)
+ # "uname -m" is not consistent, so use $cputype instead. 386
+ # is converted to i386 for consistency with other x86
+ # operating systems.
+ if test "$cputype" = "386"; then
+ UNAME_MACHINE=i386
+ else
+ UNAME_MACHINE="$cputype"
+ fi
+ echo ${UNAME_MACHINE}-unknown-plan9
+ exit ;;
+ *:TOPS-10:*:*)
+ echo pdp10-unknown-tops10
+ exit ;;
+ *:TENEX:*:*)
+ echo pdp10-unknown-tenex
+ exit ;;
+ KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*)
+ echo pdp10-dec-tops20
+ exit ;;
+ XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*)
+ echo pdp10-xkl-tops20
+ exit ;;
+ *:TOPS-20:*:*)
+ echo pdp10-unknown-tops20
+ exit ;;
+ *:ITS:*:*)
+ echo pdp10-unknown-its
+ exit ;;
+ SEI:*:*:SEIUX)
+ echo mips-sei-seiux${UNAME_RELEASE}
+ exit ;;
+ *:DragonFly:*:*)
+ echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`
+ exit ;;
+ *:*VMS:*:*)
+ UNAME_MACHINE=`(uname -p) 2>/dev/null`
+ case "${UNAME_MACHINE}" in
+ A*) echo alpha-dec-vms ; exit ;;
+ I*) echo ia64-dec-vms ; exit ;;
+ V*) echo vax-dec-vms ; exit ;;
+ esac ;;
+ *:XENIX:*:SysV)
+ echo i386-pc-xenix
+ exit ;;
+ i*86:skyos:*:*)
+ echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//'
+ exit ;;
+ i*86:rdos:*:*)
+ echo ${UNAME_MACHINE}-pc-rdos
+ exit ;;
+ i*86:AROS:*:*)
+ echo ${UNAME_MACHINE}-pc-aros
+ exit ;;
+esac
+
+#echo '(No uname command or uname output not recognized.)' 1>&2
+#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2
+
+eval $set_cc_for_build
+cat >$dummy.c <<EOF
+#ifdef _SEQUENT_
+# include <sys/types.h>
+# include <sys/utsname.h>
+#endif
+main ()
+{
+#if defined (sony)
+#if defined (MIPSEB)
+ /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed,
+ I don't know.... */
+ printf ("mips-sony-bsd\n"); exit (0);
+#else
+#include <sys/param.h>
+ printf ("m68k-sony-newsos%s\n",
+#ifdef NEWSOS4
+ "4"
+#else
+ ""
+#endif
+ ); exit (0);
+#endif
+#endif
+
+#if defined (__arm) && defined (__acorn) && defined (__unix)
+ printf ("arm-acorn-riscix\n"); exit (0);
+#endif
+
+#if defined (hp300) && !defined (hpux)
+ printf ("m68k-hp-bsd\n"); exit (0);
+#endif
+
+#if defined (NeXT)
+#if !defined (__ARCHITECTURE__)
+#define __ARCHITECTURE__ "m68k"
+#endif
+ int version;
+ version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`;
+ if (version < 4)
+ printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version);
+ else
+ printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version);
+ exit (0);
+#endif
+
+#if defined (MULTIMAX) || defined (n16)
+#if defined (UMAXV)
+ printf ("ns32k-encore-sysv\n"); exit (0);
+#else
+#if defined (CMU)
+ printf ("ns32k-encore-mach\n"); exit (0);
+#else
+ printf ("ns32k-encore-bsd\n"); exit (0);
+#endif
+#endif
+#endif
+
+#if defined (__386BSD__)
+ printf ("i386-pc-bsd\n"); exit (0);
+#endif
+
+#if defined (sequent)
+#if defined (i386)
+ printf ("i386-sequent-dynix\n"); exit (0);
+#endif
+#if defined (ns32000)
+ printf ("ns32k-sequent-dynix\n"); exit (0);
+#endif
+#endif
+
+#if defined (_SEQUENT_)
+ struct utsname un;
+
+ uname(&un);
+
+ if (strncmp(un.version, "V2", 2) == 0) {
+ printf ("i386-sequent-ptx2\n"); exit (0);
+ }
+ if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */
+ printf ("i386-sequent-ptx1\n"); exit (0);
+ }
+ printf ("i386-sequent-ptx\n"); exit (0);
+
+#endif
+
+#if defined (vax)
+# if !defined (ultrix)
+# include <sys/param.h>
+# if defined (BSD)
+# if BSD == 43
+ printf ("vax-dec-bsd4.3\n"); exit (0);
+# else
+# if BSD == 199006
+ printf ("vax-dec-bsd4.3reno\n"); exit (0);
+# else
+ printf ("vax-dec-bsd\n"); exit (0);
+# endif
+# endif
+# else
+ printf ("vax-dec-bsd\n"); exit (0);
+# endif
+# else
+ printf ("vax-dec-ultrix\n"); exit (0);
+# endif
+#endif
+
+#if defined (alliant) && defined (i860)
+ printf ("i860-alliant-bsd\n"); exit (0);
+#endif
+
+ exit (1);
+}
+EOF
+
+$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` &&
+ { echo "$SYSTEM_NAME"; exit; }
+
+# Apollos put the system type in the environment.
+
+test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; }
+
+# Convex versions that predate uname can use getsysinfo(1)
+
+if [ -x /usr/convex/getsysinfo ]
+then
+ case `getsysinfo -f cpu_type` in
+ c1*)
+ echo c1-convex-bsd
+ exit ;;
+ c2*)
+ if getsysinfo -f scalar_acc
+ then echo c32-convex-bsd
+ else echo c2-convex-bsd
+ fi
+ exit ;;
+ c34*)
+ echo c34-convex-bsd
+ exit ;;
+ c38*)
+ echo c38-convex-bsd
+ exit ;;
+ c4*)
+ echo c4-convex-bsd
+ exit ;;
+ esac
+fi
+
+cat >&2 <<EOF
+$0: unable to guess system type
+
+This script, last modified $timestamp, has failed to recognize
+the operating system you are using. It is advised that you
+download the most up to date version of the config scripts from
+
+ http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
+and
+ http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
+
+If the version you run ($0) is already up to date, please
+send the following data and any information you think might be
+pertinent to <config-patches at gnu.org> in order to provide the needed
+information to handle your system.
+
+config.guess timestamp = $timestamp
+
+uname -m = `(uname -m) 2>/dev/null || echo unknown`
+uname -r = `(uname -r) 2>/dev/null || echo unknown`
+uname -s = `(uname -s) 2>/dev/null || echo unknown`
+uname -v = `(uname -v) 2>/dev/null || echo unknown`
+
+/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null`
+/bin/uname -X = `(/bin/uname -X) 2>/dev/null`
+
+hostinfo = `(hostinfo) 2>/dev/null`
+/bin/universe = `(/bin/universe) 2>/dev/null`
+/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null`
+/bin/arch = `(/bin/arch) 2>/dev/null`
+/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null`
+/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null`
+
+UNAME_MACHINE = ${UNAME_MACHINE}
+UNAME_RELEASE = ${UNAME_RELEASE}
+UNAME_SYSTEM = ${UNAME_SYSTEM}
+UNAME_VERSION = ${UNAME_VERSION}
+EOF
+
+exit 1
+
+# Local variables:
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "timestamp='"
+# time-stamp-format: "%:y-%02m-%02d"
+# time-stamp-end: "'"
+# End:
diff --git a/config/config.sub b/config/config.sub
new file mode 100755
index 0000000..2a55a50
--- /dev/null
+++ b/config/config.sub
@@ -0,0 +1,1705 @@
+#! /bin/sh
+# Configuration validation subroutine script.
+# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
+# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009
+# Free Software Foundation, Inc.
+
+timestamp='2009-11-20'
+
+# This file is (in principle) common to ALL GNU software.
+# The presence of a machine in this file suggests that SOME GNU software
+# can handle that machine. It does not imply ALL GNU software can.
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+
+# Please send patches to <config-patches at gnu.org>. Submit a context
+# diff and a properly formatted GNU ChangeLog entry.
+#
+# Configuration subroutine to validate and canonicalize a configuration type.
+# Supply the specified configuration type as an argument.
+# If it is invalid, we print an error message on stderr and exit with code 1.
+# Otherwise, we print the canonical config type on stdout and succeed.
+
+# You can get the latest version of this script from:
+# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
+
+# This file is supposed to be the same for all GNU packages
+# and recognize all the CPU types, system types and aliases
+# that are meaningful with *any* GNU software.
+# Each package is responsible for reporting which valid configurations
+# it does not support. The user should be able to distinguish
+# a failure to support a valid configuration from a meaningless
+# configuration.
+
+# The goal of this file is to map all the various variations of a given
+# machine specification into a single specification in the form:
+# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM
+# or in some cases, the newer four-part form:
+# CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM
+# It is wrong to echo any other type of specification.
+
+me=`echo "$0" | sed -e 's,.*/,,'`
+
+usage="\
+Usage: $0 [OPTION] CPU-MFR-OPSYS
+ $0 [OPTION] ALIAS
+
+Canonicalize a configuration name.
+
+Operation modes:
+ -h, --help print this help, then exit
+ -t, --time-stamp print date of last modification, then exit
+ -v, --version print version number, then exit
+
+Report bugs and patches to <config-patches at gnu.org>."
+
+version="\
+GNU config.sub ($timestamp)
+
+Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001,
+2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
+
+This is free software; see the source for copying conditions. There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
+
+help="
+Try \`$me --help' for more information."
+
+# Parse command line
+while test $# -gt 0 ; do
+ case $1 in
+ --time-stamp | --time* | -t )
+ echo "$timestamp" ; exit ;;
+ --version | -v )
+ echo "$version" ; exit ;;
+ --help | --h* | -h )
+ echo "$usage"; exit ;;
+ -- ) # Stop option processing
+ shift; break ;;
+ - ) # Use stdin as input.
+ break ;;
+ -* )
+ echo "$me: invalid option $1$help"
+ exit 1 ;;
+
+ *local*)
+ # First pass through any local machine types.
+ echo $1
+ exit ;;
+
+ * )
+ break ;;
+ esac
+done
+
+case $# in
+ 0) echo "$me: missing argument$help" >&2
+ exit 1;;
+ 1) ;;
+ *) echo "$me: too many arguments$help" >&2
+ exit 1;;
+esac
+
+# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any).
+# Here we must recognize all the valid KERNEL-OS combinations.
+maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
+case $maybe_os in
+ nto-qnx* | linux-gnu* | linux-dietlibc | linux-newlib* | linux-uclibc* | \
+ uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* | \
+ kopensolaris*-gnu* | \
+ storm-chaos* | os2-emx* | rtmk-nova*)
+ os=-$maybe_os
+ basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
+ ;;
+ *)
+ basic_machine=`echo $1 | sed 's/-[^-]*$//'`
+ if [ $basic_machine != $1 ]
+ then os=`echo $1 | sed 's/.*-/-/'`
+ else os=; fi
+ ;;
+esac
+
+### Let's recognize common machines as not being operating systems so
+### that things like config.sub decstation-3100 work. We also
+### recognize some manufacturers as not being operating systems, so we
+### can provide default operating systems below.
+case $os in
+ -sun*os*)
+ # Prevent following clause from handling this invalid input.
+ ;;
+ -dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \
+ -att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \
+ -unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \
+ -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
+ -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
+ -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \
+ -apple | -axis | -knuth | -cray | -microblaze)
+ os=
+ basic_machine=$1
+ ;;
+ -bluegene*)
+ os=-cnk
+ ;;
+ -sim | -cisco | -oki | -wec | -winbond)
+ os=
+ basic_machine=$1
+ ;;
+ -scout)
+ ;;
+ -wrs)
+ os=-vxworks
+ basic_machine=$1
+ ;;
+ -chorusos*)
+ os=-chorusos
+ basic_machine=$1
+ ;;
+ -chorusrdb)
+ os=-chorusrdb
+ basic_machine=$1
+ ;;
+ -hiux*)
+ os=-hiuxwe2
+ ;;
+ -sco6)
+ os=-sco5v6
+ basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+ ;;
+ -sco5)
+ os=-sco3.2v5
+ basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+ ;;
+ -sco4)
+ os=-sco3.2v4
+ basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+ ;;
+ -sco3.2.[4-9]*)
+ os=`echo $os | sed -e 's/sco3.2./sco3.2v/'`
+ basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+ ;;
+ -sco3.2v[4-9]*)
+ # Don't forget version if it is 3.2v4 or newer.
+ basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+ ;;
+ -sco5v6*)
+ # Don't forget version if it is 3.2v4 or newer.
+ basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+ ;;
+ -sco*)
+ os=-sco3.2v2
+ basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+ ;;
+ -udk*)
+ basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+ ;;
+ -isc)
+ os=-isc2.2
+ basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+ ;;
+ -clix*)
+ basic_machine=clipper-intergraph
+ ;;
+ -isc*)
+ basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+ ;;
+ -lynx*)
+ os=-lynxos
+ ;;
+ -ptx*)
+ basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'`
+ ;;
+ -windowsnt*)
+ os=`echo $os | sed -e 's/windowsnt/winnt/'`
+ ;;
+ -psos*)
+ os=-psos
+ ;;
+ -mint | -mint[0-9]*)
+ basic_machine=m68k-atari
+ os=-mint
+ ;;
+esac
+
+# Decode aliases for certain CPU-COMPANY combinations.
+case $basic_machine in
+ # Recognize the basic CPU types without company name.
+ # Some are omitted here because they have special meanings below.
+ 1750a | 580 \
+ | a29k \
+ | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \
+ | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \
+ | am33_2.0 \
+ | arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr | avr32 \
+ | bfin \
+ | c4x | clipper \
+ | d10v | d30v | dlx | dsp16xx \
+ | fido | fr30 | frv \
+ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
+ | i370 | i860 | i960 | ia64 \
+ | ip2k | iq2000 \
+ | lm32 \
+ | m32c | m32r | m32rle | m68000 | m68k | m88k \
+ | maxq | mb | microblaze | mcore | mep | metag \
+ | mips | mipsbe | mipseb | mipsel | mipsle \
+ | mips16 \
+ | mips64 | mips64el \
+ | mips64octeon | mips64octeonel \
+ | mips64orion | mips64orionel \
+ | mips64r5900 | mips64r5900el \
+ | mips64vr | mips64vrel \
+ | mips64vr4100 | mips64vr4100el \
+ | mips64vr4300 | mips64vr4300el \
+ | mips64vr5000 | mips64vr5000el \
+ | mips64vr5900 | mips64vr5900el \
+ | mipsisa32 | mipsisa32el \
+ | mipsisa32r2 | mipsisa32r2el \
+ | mipsisa64 | mipsisa64el \
+ | mipsisa64r2 | mipsisa64r2el \
+ | mipsisa64sb1 | mipsisa64sb1el \
+ | mipsisa64sr71k | mipsisa64sr71kel \
+ | mipstx39 | mipstx39el \
+ | mn10200 | mn10300 \
+ | moxie \
+ | mt \
+ | msp430 \
+ | nios | nios2 \
+ | ns16k | ns32k \
+ | or32 \
+ | pdp10 | pdp11 | pj | pjl \
+ | powerpc | powerpc64 | powerpc64le | powerpcle | ppcbe \
+ | pyramid \
+ | rx \
+ | score \
+ | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \
+ | sh64 | sh64le \
+ | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \
+ | sparcv8 | sparcv9 | sparcv9b | sparcv9v \
+ | spu | strongarm \
+ | tahoe | thumb | tic4x | tic80 | tron \
+ | ubicom32 \
+ | v850 | v850e \
+ | we32k \
+ | x86 | xc16x | xscale | xscalee[bl] | xstormy16 | xtensa \
+ | z8k | z80)
+ basic_machine=$basic_machine-unknown
+ ;;
+ m6811 | m68hc11 | m6812 | m68hc12 | picochip)
+ # Motorola 68HC11/12.
+ basic_machine=$basic_machine-unknown
+ os=-none
+ ;;
+ m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k)
+ ;;
+ ms1)
+ basic_machine=mt-unknown
+ ;;
+
+ # We use `pc' rather than `unknown'
+ # because (1) that's what they normally are, and
+ # (2) the word "unknown" tends to confuse beginning users.
+ i*86 | x86_64)
+ basic_machine=$basic_machine-pc
+ ;;
+ # Object if more than one company name word.
+ *-*-*)
+ echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+ exit 1
+ ;;
+ # Recognize the basic CPU types with company name.
+ 580-* \
+ | a29k-* \
+ | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \
+ | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
+ | alphapca5[67]-* | alpha64pca5[67]-* | arc-* \
+ | arm-* | armbe-* | armle-* | armeb-* | armv*-* \
+ | avr-* | avr32-* \
+ | bfin-* | bs2000-* \
+ | c[123]* | c30-* | [cjt]90-* | c4x-* | c54x-* | c55x-* | c6x-* \
+ | clipper-* | craynv-* | cydra-* \
+ | d10v-* | d30v-* | dlx-* \
+ | elxsi-* \
+ | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \
+ | h8300-* | h8500-* \
+ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \
+ | i*86-* | i860-* | i960-* | ia64-* \
+ | ip2k-* | iq2000-* \
+ | lm32-* \
+ | m32c-* | m32r-* | m32rle-* \
+ | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \
+ | m88110-* | m88k-* | maxq-* | mcore-* | metag-* | microblaze-* \
+ | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \
+ | mips16-* \
+ | mips64-* | mips64el-* \
+ | mips64octeon-* | mips64octeonel-* \
+ | mips64orion-* | mips64orionel-* \
+ | mips64r5900-* | mips64r5900el-* \
+ | mips64vr-* | mips64vrel-* \
+ | mips64vr4100-* | mips64vr4100el-* \
+ | mips64vr4300-* | mips64vr4300el-* \
+ | mips64vr5000-* | mips64vr5000el-* \
+ | mips64vr5900-* | mips64vr5900el-* \
+ | mipsisa32-* | mipsisa32el-* \
+ | mipsisa32r2-* | mipsisa32r2el-* \
+ | mipsisa64-* | mipsisa64el-* \
+ | mipsisa64r2-* | mipsisa64r2el-* \
+ | mipsisa64sb1-* | mipsisa64sb1el-* \
+ | mipsisa64sr71k-* | mipsisa64sr71kel-* \
+ | mipstx39-* | mipstx39el-* \
+ | mmix-* \
+ | mt-* \
+ | msp430-* \
+ | nios-* | nios2-* \
+ | none-* | np1-* | ns16k-* | ns32k-* \
+ | orion-* \
+ | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
+ | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* | ppcbe-* \
+ | pyramid-* \
+ | romp-* | rs6000-* | rx-* \
+ | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \
+ | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \
+ | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \
+ | sparclite-* \
+ | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | strongarm-* | sv1-* | sx?-* \
+ | tahoe-* | thumb-* \
+ | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* | tile-* \
+ | tron-* \
+ | ubicom32-* \
+ | v850-* | v850e-* | vax-* \
+ | we32k-* \
+ | x86-* | x86_64-* | xc16x-* | xps100-* | xscale-* | xscalee[bl]-* \
+ | xstormy16-* | xtensa*-* \
+ | ymp-* \
+ | z8k-* | z80-*)
+ ;;
+ # Recognize the basic CPU types without company name, with glob match.
+ xtensa*)
+ basic_machine=$basic_machine-unknown
+ ;;
+ # Recognize the various machine names and aliases which stand
+ # for a CPU type and a company and sometimes even an OS.
+ 386bsd)
+ basic_machine=i386-unknown
+ os=-bsd
+ ;;
+ 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc)
+ basic_machine=m68000-att
+ ;;
+ 3b*)
+ basic_machine=we32k-att
+ ;;
+ a29khif)
+ basic_machine=a29k-amd
+ os=-udi
+ ;;
+ abacus)
+ basic_machine=abacus-unknown
+ ;;
+ adobe68k)
+ basic_machine=m68010-adobe
+ os=-scout
+ ;;
+ alliant | fx80)
+ basic_machine=fx80-alliant
+ ;;
+ altos | altos3068)
+ basic_machine=m68k-altos
+ ;;
+ am29k)
+ basic_machine=a29k-none
+ os=-bsd
+ ;;
+ amd64)
+ basic_machine=x86_64-pc
+ ;;
+ amd64-*)
+ basic_machine=x86_64-`echo $basic_machine | sed 's/^[^-]*-//'`
+ ;;
+ amdahl)
+ basic_machine=580-amdahl
+ os=-sysv
+ ;;
+ amiga | amiga-*)
+ basic_machine=m68k-unknown
+ ;;
+ amigaos | amigados)
+ basic_machine=m68k-unknown
+ os=-amigaos
+ ;;
+ amigaunix | amix)
+ basic_machine=m68k-unknown
+ os=-sysv4
+ ;;
+ apollo68)
+ basic_machine=m68k-apollo
+ os=-sysv
+ ;;
+ apollo68bsd)
+ basic_machine=m68k-apollo
+ os=-bsd
+ ;;
+ aros)
+ basic_machine=i386-pc
+ os=-aros
+ ;;
+ aux)
+ basic_machine=m68k-apple
+ os=-aux
+ ;;
+ balance)
+ basic_machine=ns32k-sequent
+ os=-dynix
+ ;;
+ blackfin)
+ basic_machine=bfin-unknown
+ os=-linux
+ ;;
+ blackfin-*)
+ basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'`
+ os=-linux
+ ;;
+ bluegene*)
+ basic_machine=powerpc-ibm
+ os=-cnk
+ ;;
+ c90)
+ basic_machine=c90-cray
+ os=-unicos
+ ;;
+ cegcc)
+ basic_machine=arm-unknown
+ os=-cegcc
+ ;;
+ convex-c1)
+ basic_machine=c1-convex
+ os=-bsd
+ ;;
+ convex-c2)
+ basic_machine=c2-convex
+ os=-bsd
+ ;;
+ convex-c32)
+ basic_machine=c32-convex
+ os=-bsd
+ ;;
+ convex-c34)
+ basic_machine=c34-convex
+ os=-bsd
+ ;;
+ convex-c38)
+ basic_machine=c38-convex
+ os=-bsd
+ ;;
+ cray | j90)
+ basic_machine=j90-cray
+ os=-unicos
+ ;;
+ craynv)
+ basic_machine=craynv-cray
+ os=-unicosmp
+ ;;
+ cr16)
+ basic_machine=cr16-unknown
+ os=-elf
+ ;;
+ crds | unos)
+ basic_machine=m68k-crds
+ ;;
+ crisv32 | crisv32-* | etraxfs*)
+ basic_machine=crisv32-axis
+ ;;
+ cris | cris-* | etrax*)
+ basic_machine=cris-axis
+ ;;
+ crx)
+ basic_machine=crx-unknown
+ os=-elf
+ ;;
+ da30 | da30-*)
+ basic_machine=m68k-da30
+ ;;
+ decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn)
+ basic_machine=mips-dec
+ ;;
+ decsystem10* | dec10*)
+ basic_machine=pdp10-dec
+ os=-tops10
+ ;;
+ decsystem20* | dec20*)
+ basic_machine=pdp10-dec
+ os=-tops20
+ ;;
+ delta | 3300 | motorola-3300 | motorola-delta \
+ | 3300-motorola | delta-motorola)
+ basic_machine=m68k-motorola
+ ;;
+ delta88)
+ basic_machine=m88k-motorola
+ os=-sysv3
+ ;;
+ dicos)
+ basic_machine=i686-pc
+ os=-dicos
+ ;;
+ djgpp)
+ basic_machine=i586-pc
+ os=-msdosdjgpp
+ ;;
+ dpx20 | dpx20-*)
+ basic_machine=rs6000-bull
+ os=-bosx
+ ;;
+ dpx2* | dpx2*-bull)
+ basic_machine=m68k-bull
+ os=-sysv3
+ ;;
+ ebmon29k)
+ basic_machine=a29k-amd
+ os=-ebmon
+ ;;
+ elxsi)
+ basic_machine=elxsi-elxsi
+ os=-bsd
+ ;;
+ encore | umax | mmax)
+ basic_machine=ns32k-encore
+ ;;
+ es1800 | OSE68k | ose68k | ose | OSE)
+ basic_machine=m68k-ericsson
+ os=-ose
+ ;;
+ fx2800)
+ basic_machine=i860-alliant
+ ;;
+ genix)
+ basic_machine=ns32k-ns
+ ;;
+ gmicro)
+ basic_machine=tron-gmicro
+ os=-sysv
+ ;;
+ go32)
+ basic_machine=i386-pc
+ os=-go32
+ ;;
+ h3050r* | hiux*)
+ basic_machine=hppa1.1-hitachi
+ os=-hiuxwe2
+ ;;
+ h8300hms)
+ basic_machine=h8300-hitachi
+ os=-hms
+ ;;
+ h8300xray)
+ basic_machine=h8300-hitachi
+ os=-xray
+ ;;
+ h8500hms)
+ basic_machine=h8500-hitachi
+ os=-hms
+ ;;
+ harris)
+ basic_machine=m88k-harris
+ os=-sysv3
+ ;;
+ hp300-*)
+ basic_machine=m68k-hp
+ ;;
+ hp300bsd)
+ basic_machine=m68k-hp
+ os=-bsd
+ ;;
+ hp300hpux)
+ basic_machine=m68k-hp
+ os=-hpux
+ ;;
+ hp3k9[0-9][0-9] | hp9[0-9][0-9])
+ basic_machine=hppa1.0-hp
+ ;;
+ hp9k2[0-9][0-9] | hp9k31[0-9])
+ basic_machine=m68000-hp
+ ;;
+ hp9k3[2-9][0-9])
+ basic_machine=m68k-hp
+ ;;
+ hp9k6[0-9][0-9] | hp6[0-9][0-9])
+ basic_machine=hppa1.0-hp
+ ;;
+ hp9k7[0-79][0-9] | hp7[0-79][0-9])
+ basic_machine=hppa1.1-hp
+ ;;
+ hp9k78[0-9] | hp78[0-9])
+ # FIXME: really hppa2.0-hp
+ basic_machine=hppa1.1-hp
+ ;;
+ hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893)
+ # FIXME: really hppa2.0-hp
+ basic_machine=hppa1.1-hp
+ ;;
+ hp9k8[0-9][13679] | hp8[0-9][13679])
+ basic_machine=hppa1.1-hp
+ ;;
+ hp9k8[0-9][0-9] | hp8[0-9][0-9])
+ basic_machine=hppa1.0-hp
+ ;;
+ hppa-next)
+ os=-nextstep3
+ ;;
+ hppaosf)
+ basic_machine=hppa1.1-hp
+ os=-osf
+ ;;
+ hppro)
+ basic_machine=hppa1.1-hp
+ os=-proelf
+ ;;
+ i370-ibm* | ibm*)
+ basic_machine=i370-ibm
+ ;;
+# I'm not sure what "Sysv32" means. Should this be sysv3.2?
+ i*86v32)
+ basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
+ os=-sysv32
+ ;;
+ i*86v4*)
+ basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
+ os=-sysv4
+ ;;
+ i*86v)
+ basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
+ os=-sysv
+ ;;
+ i*86sol2)
+ basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
+ os=-solaris2
+ ;;
+ i386mach)
+ basic_machine=i386-mach
+ os=-mach
+ ;;
+ i386-vsta | vsta)
+ basic_machine=i386-unknown
+ os=-vsta
+ ;;
+ iris | iris4d)
+ basic_machine=mips-sgi
+ case $os in
+ -irix*)
+ ;;
+ *)
+ os=-irix4
+ ;;
+ esac
+ ;;
+ isi68 | isi)
+ basic_machine=m68k-isi
+ os=-sysv
+ ;;
+ m68knommu)
+ basic_machine=m68k-unknown
+ os=-linux
+ ;;
+ m68knommu-*)
+ basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'`
+ os=-linux
+ ;;
+ m88k-omron*)
+ basic_machine=m88k-omron
+ ;;
+ magnum | m3230)
+ basic_machine=mips-mips
+ os=-sysv
+ ;;
+ merlin)
+ basic_machine=ns32k-utek
+ os=-sysv
+ ;;
+ microblaze)
+ basic_machine=microblaze-xilinx
+ ;;
+ mingw32)
+ basic_machine=i386-pc
+ os=-mingw32
+ ;;
+ mingw32ce)
+ basic_machine=arm-unknown
+ os=-mingw32ce
+ ;;
+ miniframe)
+ basic_machine=m68000-convergent
+ ;;
+ *mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*)
+ basic_machine=m68k-atari
+ os=-mint
+ ;;
+ mips3*-*)
+ basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`
+ ;;
+ mips3*)
+ basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown
+ ;;
+ monitor)
+ basic_machine=m68k-rom68k
+ os=-coff
+ ;;
+ morphos)
+ basic_machine=powerpc-unknown
+ os=-morphos
+ ;;
+ msdos)
+ basic_machine=i386-pc
+ os=-msdos
+ ;;
+ ms1-*)
+ basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'`
+ ;;
+ mvs)
+ basic_machine=i370-ibm
+ os=-mvs
+ ;;
+ ncr3000)
+ basic_machine=i486-ncr
+ os=-sysv4
+ ;;
+ netbsd386)
+ basic_machine=i386-unknown
+ os=-netbsd
+ ;;
+ netwinder)
+ basic_machine=armv4l-rebel
+ os=-linux
+ ;;
+ news | news700 | news800 | news900)
+ basic_machine=m68k-sony
+ os=-newsos
+ ;;
+ news1000)
+ basic_machine=m68030-sony
+ os=-newsos
+ ;;
+ news-3600 | risc-news)
+ basic_machine=mips-sony
+ os=-newsos
+ ;;
+ necv70)
+ basic_machine=v70-nec
+ os=-sysv
+ ;;
+ next | m*-next )
+ basic_machine=m68k-next
+ case $os in
+ -nextstep* )
+ ;;
+ -ns2*)
+ os=-nextstep2
+ ;;
+ *)
+ os=-nextstep3
+ ;;
+ esac
+ ;;
+ nh3000)
+ basic_machine=m68k-harris
+ os=-cxux
+ ;;
+ nh[45]000)
+ basic_machine=m88k-harris
+ os=-cxux
+ ;;
+ nindy960)
+ basic_machine=i960-intel
+ os=-nindy
+ ;;
+ mon960)
+ basic_machine=i960-intel
+ os=-mon960
+ ;;
+ nonstopux)
+ basic_machine=mips-compaq
+ os=-nonstopux
+ ;;
+ np1)
+ basic_machine=np1-gould
+ ;;
+ nsr-tandem)
+ basic_machine=nsr-tandem
+ ;;
+ op50n-* | op60c-*)
+ basic_machine=hppa1.1-oki
+ os=-proelf
+ ;;
+ openrisc | openrisc-*)
+ basic_machine=or32-unknown
+ ;;
+ os400)
+ basic_machine=powerpc-ibm
+ os=-os400
+ ;;
+ OSE68000 | ose68000)
+ basic_machine=m68000-ericsson
+ os=-ose
+ ;;
+ os68k)
+ basic_machine=m68k-none
+ os=-os68k
+ ;;
+ pa-hitachi)
+ basic_machine=hppa1.1-hitachi
+ os=-hiuxwe2
+ ;;
+ paragon)
+ basic_machine=i860-intel
+ os=-osf
+ ;;
+ parisc)
+ basic_machine=hppa-unknown
+ os=-linux
+ ;;
+ parisc-*)
+ basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'`
+ os=-linux
+ ;;
+ pbd)
+ basic_machine=sparc-tti
+ ;;
+ pbb)
+ basic_machine=m68k-tti
+ ;;
+ pc532 | pc532-*)
+ basic_machine=ns32k-pc532
+ ;;
+ pc98)
+ basic_machine=i386-pc
+ ;;
+ pc98-*)
+ basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'`
+ ;;
+ pentium | p5 | k5 | k6 | nexgen | viac3)
+ basic_machine=i586-pc
+ ;;
+ pentiumpro | p6 | 6x86 | athlon | athlon_*)
+ basic_machine=i686-pc
+ ;;
+ pentiumii | pentium2 | pentiumiii | pentium3)
+ basic_machine=i686-pc
+ ;;
+ pentium4)
+ basic_machine=i786-pc
+ ;;
+ pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*)
+ basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'`
+ ;;
+ pentiumpro-* | p6-* | 6x86-* | athlon-*)
+ basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'`
+ ;;
+ pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*)
+ basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'`
+ ;;
+ pentium4-*)
+ basic_machine=i786-`echo $basic_machine | sed 's/^[^-]*-//'`
+ ;;
+ pn)
+ basic_machine=pn-gould
+ ;;
+ power) basic_machine=power-ibm
+ ;;
+ ppc) basic_machine=powerpc-unknown
+ ;;
+ ppc-*) basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
+ ;;
+ ppcle | powerpclittle | ppc-le | powerpc-little)
+ basic_machine=powerpcle-unknown
+ ;;
+ ppcle-* | powerpclittle-*)
+ basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'`
+ ;;
+ ppc64) basic_machine=powerpc64-unknown
+ ;;
+ ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
+ ;;
+ ppc64le | powerpc64little | ppc64-le | powerpc64-little)
+ basic_machine=powerpc64le-unknown
+ ;;
+ ppc64le-* | powerpc64little-*)
+ basic_machine=powerpc64le-`echo $basic_machine | sed 's/^[^-]*-//'`
+ ;;
+ ps2)
+ basic_machine=i386-ibm
+ ;;
+ pw32)
+ basic_machine=i586-unknown
+ os=-pw32
+ ;;
+ rdos)
+ basic_machine=i386-pc
+ os=-rdos
+ ;;
+ rom68k)
+ basic_machine=m68k-rom68k
+ os=-coff
+ ;;
+ rm[46]00)
+ basic_machine=mips-siemens
+ ;;
+ rtpc | rtpc-*)
+ basic_machine=romp-ibm
+ ;;
+ s390 | s390-*)
+ basic_machine=s390-ibm
+ ;;
+ s390x | s390x-*)
+ basic_machine=s390x-ibm
+ ;;
+ sa29200)
+ basic_machine=a29k-amd
+ os=-udi
+ ;;
+ sb1)
+ basic_machine=mipsisa64sb1-unknown
+ ;;
+ sb1el)
+ basic_machine=mipsisa64sb1el-unknown
+ ;;
+ sde)
+ basic_machine=mipsisa32-sde
+ os=-elf
+ ;;
+ sei)
+ basic_machine=mips-sei
+ os=-seiux
+ ;;
+ sequent)
+ basic_machine=i386-sequent
+ ;;
+ sh)
+ basic_machine=sh-hitachi
+ os=-hms
+ ;;
+ sh5el)
+ basic_machine=sh5le-unknown
+ ;;
+ sh64)
+ basic_machine=sh64-unknown
+ ;;
+ sparclite-wrs | simso-wrs)
+ basic_machine=sparclite-wrs
+ os=-vxworks
+ ;;
+ sps7)
+ basic_machine=m68k-bull
+ os=-sysv2
+ ;;
+ spur)
+ basic_machine=spur-unknown
+ ;;
+ st2000)
+ basic_machine=m68k-tandem
+ ;;
+ stratus)
+ basic_machine=i860-stratus
+ os=-sysv4
+ ;;
+ sun2)
+ basic_machine=m68000-sun
+ ;;
+ sun2os3)
+ basic_machine=m68000-sun
+ os=-sunos3
+ ;;
+ sun2os4)
+ basic_machine=m68000-sun
+ os=-sunos4
+ ;;
+ sun3os3)
+ basic_machine=m68k-sun
+ os=-sunos3
+ ;;
+ sun3os4)
+ basic_machine=m68k-sun
+ os=-sunos4
+ ;;
+ sun4os3)
+ basic_machine=sparc-sun
+ os=-sunos3
+ ;;
+ sun4os4)
+ basic_machine=sparc-sun
+ os=-sunos4
+ ;;
+ sun4sol2)
+ basic_machine=sparc-sun
+ os=-solaris2
+ ;;
+ sun3 | sun3-*)
+ basic_machine=m68k-sun
+ ;;
+ sun4)
+ basic_machine=sparc-sun
+ ;;
+ sun386 | sun386i | roadrunner)
+ basic_machine=i386-sun
+ ;;
+ sv1)
+ basic_machine=sv1-cray
+ os=-unicos
+ ;;
+ symmetry)
+ basic_machine=i386-sequent
+ os=-dynix
+ ;;
+ t3e)
+ basic_machine=alphaev5-cray
+ os=-unicos
+ ;;
+ t90)
+ basic_machine=t90-cray
+ os=-unicos
+ ;;
+ tic54x | c54x*)
+ basic_machine=tic54x-unknown
+ os=-coff
+ ;;
+ tic55x | c55x*)
+ basic_machine=tic55x-unknown
+ os=-coff
+ ;;
+ tic6x | c6x*)
+ basic_machine=tic6x-unknown
+ os=-coff
+ ;;
+ tile*)
+ basic_machine=tile-unknown
+ os=-linux-gnu
+ ;;
+ tx39)
+ basic_machine=mipstx39-unknown
+ ;;
+ tx39el)
+ basic_machine=mipstx39el-unknown
+ ;;
+ toad1)
+ basic_machine=pdp10-xkl
+ os=-tops20
+ ;;
+ tower | tower-32)
+ basic_machine=m68k-ncr
+ ;;
+ tpf)
+ basic_machine=s390x-ibm
+ os=-tpf
+ ;;
+ udi29k)
+ basic_machine=a29k-amd
+ os=-udi
+ ;;
+ ultra3)
+ basic_machine=a29k-nyu
+ os=-sym1
+ ;;
+ v810 | necv810)
+ basic_machine=v810-nec
+ os=-none
+ ;;
+ vaxv)
+ basic_machine=vax-dec
+ os=-sysv
+ ;;
+ vms)
+ basic_machine=vax-dec
+ os=-vms
+ ;;
+ vpp*|vx|vx-*)
+ basic_machine=f301-fujitsu
+ ;;
+ vxworks960)
+ basic_machine=i960-wrs
+ os=-vxworks
+ ;;
+ vxworks68)
+ basic_machine=m68k-wrs
+ os=-vxworks
+ ;;
+ vxworks29k)
+ basic_machine=a29k-wrs
+ os=-vxworks
+ ;;
+ w65*)
+ basic_machine=w65-wdc
+ os=-none
+ ;;
+ w89k-*)
+ basic_machine=hppa1.1-winbond
+ os=-proelf
+ ;;
+ xbox)
+ basic_machine=i686-pc
+ os=-mingw32
+ ;;
+ xps | xps100)
+ basic_machine=xps100-honeywell
+ ;;
+ ymp)
+ basic_machine=ymp-cray
+ os=-unicos
+ ;;
+ z8k-*-coff)
+ basic_machine=z8k-unknown
+ os=-sim
+ ;;
+ z80-*-coff)
+ basic_machine=z80-unknown
+ os=-sim
+ ;;
+ none)
+ basic_machine=none-none
+ os=-none
+ ;;
+
+# Here we handle the default manufacturer of certain CPU types. It is in
+# some cases the only manufacturer, in others, it is the most popular.
+ w89k)
+ basic_machine=hppa1.1-winbond
+ ;;
+ op50n)
+ basic_machine=hppa1.1-oki
+ ;;
+ op60c)
+ basic_machine=hppa1.1-oki
+ ;;
+ romp)
+ basic_machine=romp-ibm
+ ;;
+ mmix)
+ basic_machine=mmix-knuth
+ ;;
+ rs6000)
+ basic_machine=rs6000-ibm
+ ;;
+ vax)
+ basic_machine=vax-dec
+ ;;
+ pdp10)
+ # there are many clones, so DEC is not a safe bet
+ basic_machine=pdp10-unknown
+ ;;
+ pdp11)
+ basic_machine=pdp11-dec
+ ;;
+ we32k)
+ basic_machine=we32k-att
+ ;;
+ sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele)
+ basic_machine=sh-unknown
+ ;;
+ sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v)
+ basic_machine=sparc-sun
+ ;;
+ cydra)
+ basic_machine=cydra-cydrome
+ ;;
+ orion)
+ basic_machine=orion-highlevel
+ ;;
+ orion105)
+ basic_machine=clipper-highlevel
+ ;;
+ mac | mpw | mac-mpw)
+ basic_machine=m68k-apple
+ ;;
+ pmac | pmac-mpw)
+ basic_machine=powerpc-apple
+ ;;
+ *-unknown)
+ # Make sure to match an already-canonicalized machine name.
+ ;;
+ *)
+ echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+ exit 1
+ ;;
+esac
+
+# Here we canonicalize certain aliases for manufacturers.
+case $basic_machine in
+ *-digital*)
+ basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'`
+ ;;
+ *-commodore*)
+ basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'`
+ ;;
+ *)
+ ;;
+esac
+
+# Decode manufacturer-specific aliases for certain operating systems.
+
+if [ x"$os" != x"" ]
+then
+case $os in
+ # First match some system type aliases
+ # that might get confused with valid system types.
+ # -solaris* is a basic system type, with this one exception.
+ -auroraux)
+ os=-auroraux
+ ;;
+ -solaris1 | -solaris1.*)
+ os=`echo $os | sed -e 's|solaris1|sunos4|'`
+ ;;
+ -solaris)
+ os=-solaris2
+ ;;
+ -svr4*)
+ os=-sysv4
+ ;;
+ -unixware*)
+ os=-sysv4.2uw
+ ;;
+ -gnu/linux*)
+ os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'`
+ ;;
+ # First accept the basic system types.
+ # The portable systems comes first.
+ # Each alternative MUST END IN A *, to match a version number.
+ # -sysv* is not here because it comes later, after sysvr4.
+ -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
+ | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\
+ | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
+ | -sym* | -kopensolaris* \
+ | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
+ | -aos* | -aros* \
+ | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
+ | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
+ | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \
+ | -openbsd* | -solidbsd* \
+ | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \
+ | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
+ | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \
+ | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \
+ | -chorusos* | -chorusrdb* | -cegcc* \
+ | -cygwin* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
+ | -mingw32* | -linux-gnu* | -linux-newlib* | -linux-uclibc* \
+ | -uxpv* | -beos* | -mpeix* | -udk* \
+ | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \
+ | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \
+ | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \
+ | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \
+ | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \
+ | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
+ | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*)
+ # Remember, each alternative MUST END IN *, to match a version number.
+ ;;
+ -qnx*)
+ case $basic_machine in
+ x86-* | i*86-*)
+ ;;
+ *)
+ os=-nto$os
+ ;;
+ esac
+ ;;
+ -nto-qnx*)
+ ;;
+ -nto*)
+ os=`echo $os | sed -e 's|nto|nto-qnx|'`
+ ;;
+ -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \
+ | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \
+ | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*)
+ ;;
+ -mac*)
+ os=`echo $os | sed -e 's|mac|macos|'`
+ ;;
+ -linux-dietlibc)
+ os=-linux-dietlibc
+ ;;
+ -linux*)
+ os=`echo $os | sed -e 's|linux|linux-gnu|'`
+ ;;
+ -sunos5*)
+ os=`echo $os | sed -e 's|sunos5|solaris2|'`
+ ;;
+ -sunos6*)
+ os=`echo $os | sed -e 's|sunos6|solaris3|'`
+ ;;
+ -opened*)
+ os=-openedition
+ ;;
+ -os400*)
+ os=-os400
+ ;;
+ -wince*)
+ os=-wince
+ ;;
+ -osfrose*)
+ os=-osfrose
+ ;;
+ -osf*)
+ os=-osf
+ ;;
+ -utek*)
+ os=-bsd
+ ;;
+ -dynix*)
+ os=-bsd
+ ;;
+ -acis*)
+ os=-aos
+ ;;
+ -atheos*)
+ os=-atheos
+ ;;
+ -syllable*)
+ os=-syllable
+ ;;
+ -386bsd)
+ os=-bsd
+ ;;
+ -ctix* | -uts*)
+ os=-sysv
+ ;;
+ -nova*)
+ os=-rtmk-nova
+ ;;
+ -ns2 )
+ os=-nextstep2
+ ;;
+ -nsk*)
+ os=-nsk
+ ;;
+ # Preserve the version number of sinix5.
+ -sinix5.*)
+ os=`echo $os | sed -e 's|sinix|sysv|'`
+ ;;
+ -sinix*)
+ os=-sysv4
+ ;;
+ -tpf*)
+ os=-tpf
+ ;;
+ -triton*)
+ os=-sysv3
+ ;;
+ -oss*)
+ os=-sysv3
+ ;;
+ -svr4)
+ os=-sysv4
+ ;;
+ -svr3)
+ os=-sysv3
+ ;;
+ -sysvr4)
+ os=-sysv4
+ ;;
+ # This must come after -sysvr4.
+ -sysv*)
+ ;;
+ -ose*)
+ os=-ose
+ ;;
+ -es1800*)
+ os=-ose
+ ;;
+ -xenix)
+ os=-xenix
+ ;;
+ -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*)
+ os=-mint
+ ;;
+ -aros*)
+ os=-aros
+ ;;
+ -kaos*)
+ os=-kaos
+ ;;
+ -zvmoe)
+ os=-zvmoe
+ ;;
+ -dicos*)
+ os=-dicos
+ ;;
+ -none)
+ ;;
+ *)
+ # Get rid of the `-' at the beginning of $os.
+ os=`echo $os | sed 's/[^-]*-//'`
+ echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2
+ exit 1
+ ;;
+esac
+else
+
+# Here we handle the default operating systems that come with various machines.
+# The value should be what the vendor currently ships out the door with their
+# machine or put another way, the most popular os provided with the machine.
+
+# Note that if you're going to try to match "-MANUFACTURER" here (say,
+# "-sun"), then you have to tell the case statement up towards the top
+# that MANUFACTURER isn't an operating system. Otherwise, code above
+# will signal an error saying that MANUFACTURER isn't an operating
+# system, and we'll never get to this point.
+
+case $basic_machine in
+ score-*)
+ os=-elf
+ ;;
+ spu-*)
+ os=-elf
+ ;;
+ *-acorn)
+ os=-riscix1.2
+ ;;
+ arm*-rebel)
+ os=-linux
+ ;;
+ arm*-semi)
+ os=-aout
+ ;;
+ c4x-* | tic4x-*)
+ os=-coff
+ ;;
+ # This must come before the *-dec entry.
+ pdp10-*)
+ os=-tops20
+ ;;
+ pdp11-*)
+ os=-none
+ ;;
+ *-dec | vax-*)
+ os=-ultrix4.2
+ ;;
+ m68*-apollo)
+ os=-domain
+ ;;
+ i386-sun)
+ os=-sunos4.0.2
+ ;;
+ m68000-sun)
+ os=-sunos3
+ # This also exists in the configure program, but was not the
+ # default.
+ # os=-sunos4
+ ;;
+ m68*-cisco)
+ os=-aout
+ ;;
+ mep-*)
+ os=-elf
+ ;;
+ mips*-cisco)
+ os=-elf
+ ;;
+ mips*-*)
+ os=-elf
+ ;;
+ or32-*)
+ os=-coff
+ ;;
+ *-tti) # must be before sparc entry or we get the wrong os.
+ os=-sysv3
+ ;;
+ sparc-* | *-sun)
+ os=-sunos4.1.1
+ ;;
+ *-be)
+ os=-beos
+ ;;
+ *-haiku)
+ os=-haiku
+ ;;
+ *-ibm)
+ os=-aix
+ ;;
+ *-knuth)
+ os=-mmixware
+ ;;
+ *-wec)
+ os=-proelf
+ ;;
+ *-winbond)
+ os=-proelf
+ ;;
+ *-oki)
+ os=-proelf
+ ;;
+ *-hp)
+ os=-hpux
+ ;;
+ *-hitachi)
+ os=-hiux
+ ;;
+ i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent)
+ os=-sysv
+ ;;
+ *-cbm)
+ os=-amigaos
+ ;;
+ *-dg)
+ os=-dgux
+ ;;
+ *-dolphin)
+ os=-sysv3
+ ;;
+ m68k-ccur)
+ os=-rtu
+ ;;
+ m88k-omron*)
+ os=-luna
+ ;;
+ *-next )
+ os=-nextstep
+ ;;
+ *-sequent)
+ os=-ptx
+ ;;
+ *-crds)
+ os=-unos
+ ;;
+ *-ns)
+ os=-genix
+ ;;
+ i370-*)
+ os=-mvs
+ ;;
+ *-next)
+ os=-nextstep3
+ ;;
+ *-gould)
+ os=-sysv
+ ;;
+ *-highlevel)
+ os=-bsd
+ ;;
+ *-encore)
+ os=-bsd
+ ;;
+ *-sgi)
+ os=-irix
+ ;;
+ *-siemens)
+ os=-sysv4
+ ;;
+ *-masscomp)
+ os=-rtu
+ ;;
+ f30[01]-fujitsu | f700-fujitsu)
+ os=-uxpv
+ ;;
+ *-rom68k)
+ os=-coff
+ ;;
+ *-*bug)
+ os=-coff
+ ;;
+ *-apple)
+ os=-macos
+ ;;
+ *-atari*)
+ os=-mint
+ ;;
+ *)
+ os=-none
+ ;;
+esac
+fi
+
+# Here we handle the case where we know the os, and the CPU type, but not the
+# manufacturer. We pick the logical manufacturer.
+vendor=unknown
+case $basic_machine in
+ *-unknown)
+ case $os in
+ -riscix*)
+ vendor=acorn
+ ;;
+ -sunos*)
+ vendor=sun
+ ;;
+ -cnk*|-aix*)
+ vendor=ibm
+ ;;
+ -beos*)
+ vendor=be
+ ;;
+ -hpux*)
+ vendor=hp
+ ;;
+ -mpeix*)
+ vendor=hp
+ ;;
+ -hiux*)
+ vendor=hitachi
+ ;;
+ -unos*)
+ vendor=crds
+ ;;
+ -dgux*)
+ vendor=dg
+ ;;
+ -luna*)
+ vendor=omron
+ ;;
+ -genix*)
+ vendor=ns
+ ;;
+ -mvs* | -opened*)
+ vendor=ibm
+ ;;
+ -os400*)
+ vendor=ibm
+ ;;
+ -ptx*)
+ vendor=sequent
+ ;;
+ -tpf*)
+ vendor=ibm
+ ;;
+ -vxsim* | -vxworks* | -windiss*)
+ vendor=wrs
+ ;;
+ -aux*)
+ vendor=apple
+ ;;
+ -hms*)
+ vendor=hitachi
+ ;;
+ -mpw* | -macos*)
+ vendor=apple
+ ;;
+ -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*)
+ vendor=atari
+ ;;
+ -vos*)
+ vendor=stratus
+ ;;
+ esac
+ basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"`
+ ;;
+esac
+
+echo $basic_machine$os
+exit
+
+# Local variables:
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "timestamp='"
+# time-stamp-format: "%:y-%02m-%02d"
+# time-stamp-end: "'"
+# End:
diff --git a/config/cron-make-nightly-tarball.pl b/config/cron-make-nightly-tarball.pl
deleted file mode 100755
index fd3c442..0000000
--- a/config/cron-make-nightly-tarball.pl
+++ /dev/null
@@ -1,292 +0,0 @@
-#!/usr/bin/env perl
-
-#
-# Script to automate the steps to make pristine nightly tarballs and
-# place them in a specified output directory (presumeably where the
-# web server can serve them up to the world). Designed to be invoked
-# via cron (i.e., no output unless --verbose is specified).
-#
-# The specific steps are:
-#
-# 1. Ensure we have a pristine, clean git tree
-# 2. git pull the latest down from upstream
-# 3. Use "git describe" to get a unique string to represent this tarball
-# 4. If we already have a tarball for this git HEAD in the destination
-# directory, no need to do anything further / quit
-# 5. Re-write configure.ac to include the git describe string in the
-# version argument to AC_INIT
-# 6. autogen.sh/configure/make distcheck
-# 7. Move the resulting tarballs to the destination directory
-# 8. Re-create sym links for libfabric-latest.tar.(gz|bz2)
-# 9. Re-generate md5 and sh1 hash files
-#
-# Note that this script intentionally does *not* prune old nightly
-# tarballs as result of an OFIWG phone discussion, the conlcusion of
-# which was "What the heck; we have lots of disk space. Keep
-# everything."
-#
-
-use strict;
-use warnings;
-
-use File::Temp qw/ tempdir /;
-use File::Basename;
-use Getopt::Long;
-
-my $libfabric_dir_arg;
-my $fabtests_dir_arg;
-my $download_dir_arg;
-my $libfabric_coverity_token_arg;
-my $fabtests_coverity_token_arg;
-my $logfile_dir_arg;
-my $help_arg;
-my $verbose_arg;
-my $debug_arg;
-
-my $ok = Getopt::Long::GetOptions("libfabric-source-dir=s" => \$libfabric_dir_arg,
- "fabtests-source-dir=s" => \$fabtests_dir_arg,
- "download-dir=s" => \$download_dir_arg,
- "libfabric-coverity-token=s" => \$libfabric_coverity_token_arg,
- "fabtests-coverity-token=s" => \$fabtests_coverity_token_arg,
- "logfile-dir=s" => \$logfile_dir_arg,
- "verbose" => \$verbose_arg,
- "debug" => \$debug_arg,
- "help|h" => \$help_arg,
- );
-
-if ($help_arg || !$ok) {
- print "$0 --libfabric-source-dir libfabric_git_tree --fabtests-source-dir fabtests_git_tree --download-dir download_tree\n";
- exit($ok);
-}
-
-# Sanity checks
-die "Must specify --libfabric-source-dir, --fabtests-source-dir, --download-dir, and --logfile-dir"
- if (!defined($libfabric_dir_arg) || $libfabric_dir_arg eq "" ||
- !defined($fabtests_dir_arg) || $fabtests_dir_arg eq "" ||
- !defined($download_dir_arg) || $download_dir_arg eq "" ||
- !defined($logfile_dir_arg) || $logfile_dir_arg eq "");
-die "$libfabric_dir_arg is not a valid directory"
- if (! -d $libfabric_dir_arg);
-die "$libfabric_dir_arg is not libfabric git clone"
- if (! -d "$libfabric_dir_arg/.git" || ! -f "$libfabric_dir_arg/src/fi_tostr.c");
-die "$download_dir_arg is not a valid directory"
- if (! -d $download_dir_arg);
-die "$logfile_dir_arg is not a valid directory"
- if (! -d $logfile_dir_arg);
-
-$verbose_arg = 1
- if ($debug_arg);
-
-#####################################################################
-
-sub doit {
- my $allowed_to_fail = shift;
- my $cmd = shift;
- my $stdout_file = shift;
-
- # Redirect stdout if requested
- if (defined $stdout_file) {
- $stdout_file = "$logfile_dir_arg/$stdout_file.log";
- unlink($stdout_file);
- $cmd .= " >$stdout_file";
- } elsif (!$verbose_arg && $cmd !~ />/) {
- $cmd .= " >/dev/null";
- }
- $cmd .= " 2>&1";
-
- my $rc = system($cmd);
- if (0 != $rc && !$allowed_to_fail) {
- # If we die/fail, ensure to a) restore the git tree to a clean
- # state, and b) change out of the temp tree so that it can be
- # removed upon exit.
- chdir($libfabric_dir_arg);
- system("git clean -dfx 2>&1 > /dev/null");
- system("git reset --hard HEAD 2>&1 > /dev/null");
- chdir("/");
-
- print "Command $cmd failed: exit status $rc. Stdout:\n";
- system("cat $stdout_file")
- if (defined($stdout_file) && -f $stdout_file);
- die "Cannot continue";
- }
- system("cat $stdout_file")
- if ($debug_arg && defined($stdout_file) && -f $stdout_file);
-}
-
-sub verbose {
- print @_
- if ($verbose_arg);
-}
-
-sub git_cleanup {
- verbose("*** Ensuring we have a clean git tree...\n");
-
- doit(0, "git clean -dfx", "git-clean");
- doit(0, "git reset --hard HEAD", "git-reset");
- doit(0, "git pull", "git-pull");
-}
-
-sub get_git_version {
- # Get a git describe id (minus the initial 'v' in the tag name, if any)
- my $version = `git describe --tags --always`;
- chomp($version);
-
- verbose("*** Git describe: $version\n");
- $version =~ s/^v//;
- $version =~ y/-/./;
- return $version;
-}
-
-sub make_tarball {
- my $base_name = shift;
- my $source_dir = shift;
- my $version = shift;
- my $installdir = shift;
-
- my $configure_args = "CPPFLAGS=-I$installdir/include LDFLAGS=-L$installdir/lib";
-
- # Read in configure.ac
- verbose("*** Reading version number from configure.ac...\n");
- open(IN, "configure.ac") || die "Can't open configure.ac for reading";
- my $config;
- $config .= $_
- while(<IN>);
- close(IN);
-
- # Get the original version number
- $config =~ m/AC_INIT\(\[\Q$base_name\E\], \[(.+?)\]/;
- my $orig_version = $1;
- verbose("*** Replacing configure.ac version: $orig_version\n");
- verbose("*** Nightly tarball version: $version\n");
-
- # Update the version number with the output from "git describe"
- verbose("*** Re-writing configure.ac with git describe results...\n");
- $config =~ s/(AC_INIT\(\[\Q$base_name\E\], \[).+?\]/$1$version]/;
- open(OUT, ">configure.ac");
- print OUT $config;
- close(OUT);
-
- # Now make the tarball
- verbose("*** Running autogen.sh...\n");
- doit(0, "./autogen.sh", "autogen");
- verbose("*** Running configure...\n");
- doit(0, "./configure $configure_args --prefix=$installdir", "configure");
- verbose("*** Running make install...\n");
- doit(0, "make install", "make-install");
-
- # Is there already a tarball of this version in the download
- # directory? If so, just exit now without doing anything.
- if (-f "$download_dir_arg/$base_name-$version.tar.gz") {
- verbose("*** Target tarball already exists: $base_name-$version.tar.gz\n");
- return 0;
- }
-
- # Note that distscript.pl, invoked by "make dist", checks for a dirty
- # git tree. We have to tell it that a modified configure.ac is ok.
- # Put the name "configure.ac" in a magic environment variable.
- $ENV{'LIBFABRIC_DISTSCRIPT_DIRTY_FILES'} = "configure.ac";
-
- verbose("*** Running make distcheck...\n");
- doit(0, "AM_MAKEFLAGS=V=1 DISTCHECK_CONFIGURE_FLAGS=\"$configure_args\" make distcheck", "distcheck");
-
- delete $ENV{'LIBFABRIC_DISTSCRIPT_DIRTY_FILES'};
-
- # Restore configure.ac
- verbose("*** Restoring configure.ac...\n");
- doit(0, "git checkout configure.ac");
-
- # Move the resulting tarballs to the downloads directory
- verbose("*** Placing tarballs in download directory...\n");
- doit(0, "mv $base_name-$version.tar.gz $base_name-$version.tar.bz2 $download_dir_arg");
-
- # Make sym links to these newest tarballs
- chdir($download_dir_arg);
- unlink("$base_name-latest.tar.gz");
- unlink("$base_name-latest.tar.bz2");
- doit(0, "ln -s $base_name-$version.tar.gz $base_name-latest.tar.gz");
- doit(0, "ln -s $base_name-$version.tar.bz2 $base_name-latest.tar.bz2");
-
- return 1;
-}
-
-sub submit_to_coverity {
- my $project_name = shift;
- my $version = shift;
- my $configure_args = shift;
- my $coverity_token = shift;
-
- verbose("*** Preparing/submitting to Coverity...\n");
-
- # The coverity script will be in the same directory as this script
- my $dir = dirname($0);
- my $base_name = $project_name;
- $base_name =~ s/^ofiwg(%2F|-)([a-z]+)$/$2/;
- my $cmd = "$dir/cron-submit-coverity.pl " .
- "--filename $download_dir_arg/$base_name-$version.tar.bz2 " .
- "--coverity-token $coverity_token " .
- "--make-args=-j8 " .
- "--configure-args=\"$configure_args\" " .
- "--project=$project_name ";
-
- $cmd .= "--verbose "
- if ($verbose_arg);
- $cmd .= "--debug "
- if ($debug_arg);
- $cmd .= "--logfile-dir=$logfile_dir_arg"
- if (defined($logfile_dir_arg));
-
- # Coverity script will do its own logging
- doit(0, $cmd);
-}
-
-#####################################################################
-
-# Create a temporary directory to install into
-my $installdir = tempdir(CLEANUP => 1);
-
-verbose("*** Building libfabric...\n");
-chdir($libfabric_dir_arg);
-git_cleanup();
-my $libfabric_version = get_git_version();
-my $rebuilt_libfabric = make_tarball("libfabric", $libfabric_dir_arg,
- $libfabric_version, $installdir);
-
-verbose("\n\n*** Building fabtests...\n");
-chdir($fabtests_dir_arg);
-git_cleanup();
-my $fabtests_version = get_git_version();
-my $rebuilt_fabtests = make_tarball("fabtests", $fabtests_dir_arg,
- $fabtests_version, $installdir);
-
-if ($rebuilt_libfabric || $rebuilt_fabtests) {
- # Re-generate hashes
- verbose("*** Re-generating md5/sha1sums...\n");
- chdir($download_dir_arg);
- doit(0, "md5sum libfabric*tar* fabtests*tar* > md5sums.txt");
- doit(0, "sha1sum libfabric*tar* fabtests*tar* > sha1sums.txt");
-}
-
-if ($rebuilt_libfabric) {
- # Re-write latest.txt
- verbose("*** Re-creating latest.txt...\n");
- unlink("latest.txt");
- open(OUT, ">latest.txt") || die "Can't write to latest.txt";
- print OUT "libfabric-$libfabric_version\n";
- close(OUT);
-}
-
-# Run the coverity script if requested
-if (defined($libfabric_coverity_token_arg) && $rebuilt_libfabric) {
- submit_to_coverity("ofiwg%2Flibfabric", $libfabric_version,
- "--enable-sockets --enable-udp --enable-verbs --enable-usnic",
- $libfabric_coverity_token_arg);
-}
-if (defined($fabtests_coverity_token_arg) && $rebuilt_fabtests) {
- submit_to_coverity("ofiwg%2Ffabtests", $fabtests_version,
- "CPPFLAGS=-I$installdir/include LDFLAGS=-L$installdir/lib",
- $fabtests_coverity_token_arg);
-}
-
-# All done
-verbose("*** All done / nightly tarball\n");
-exit(0);
diff --git a/config/cron-run-all-md2nroff.pl b/config/cron-run-all-md2nroff.pl
deleted file mode 100755
index 8dd0424..0000000
--- a/config/cron-run-all-md2nroff.pl
+++ /dev/null
@@ -1,211 +0,0 @@
-#!/usr/bin/env perl
-
-# Script to pull down the latest markdown man pages from the libfabric
-# git repo. Iterate over them, converting each to an nroff man page
-# and also copying+committing them to the gh-pages branch. Finally,
-# git push them back upstream (so that Github will render + serve them
-# up as web pages).
-
-use strict;
-use warnings;
-
-use POSIX;
-use File::Basename;
-use Getopt::Long;
-use File::Temp;
-
-my $repo_arg;
-my $source_branch_arg;
-my $pages_branch_arg;
-my $logfile_dir_arg = "/tmp";
-my $verbose_arg;
-my $help_arg;
-
-my $ok = Getopt::Long::GetOptions("repo=s" => \$repo_arg,
- "source-branch=s" => \$source_branch_arg,
- "pages-branch=s" => \$pages_branch_arg,
- "logfile-dir=s" => \$logfile_dir_arg,
- "help|h" => \$help_arg,
- "verbose" => \$verbose_arg,
- );
-
-# Sanity checks
-die "Must specify a git repo"
- if (!defined($repo_arg));
-die "Must specify a git source branch"
- if (!defined($source_branch_arg));
-
-#####################################################################
-
-my $logfile_dir = $logfile_dir_arg;
-my $logfile_counter = 1;
-
-sub doit {
- my $allowed_to_fail = shift;
- my $cmd = shift;
- my $stdout_file = shift;
-
- # Put a prefix on the logfiles so that we know that they belong to
- # this script, and put a counter so that we know the sequence of
- # logfiles
- $stdout_file = "runall-md2nroff-$logfile_counter-$stdout_file";
- ++$logfile_counter;
-
- # Redirect stdout if requested
- if (defined $stdout_file) {
- $stdout_file = "$logfile_dir/$stdout_file.log";
- unlink($stdout_file);
- $cmd .= " >$stdout_file";
- } elsif (!$verbose_arg && $cmd !~ />/) {
- $cmd .= " >/dev/null";
- }
- $cmd .= " 2>&1";
-
- my $rc = system($cmd);
- if (0 != $rc && !$allowed_to_fail) {
- # If we die/fail, ensure to change out of the temp tree so
- # that it can be removed upon exit.
- chdir("/");
- die "Command $cmd failed: exit status $rc";
- }
-
- system("cat $stdout_file")
- if ($verbose_arg && defined($stdout_file) && -f $stdout_file);
-}
-
-sub verbose {
- print @_
- if ($verbose_arg);
-}
-
-#####################################################################
-
-# Setup a logfile dir just for this run
-my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
- localtime(time);
-$logfile_dir =
- sprintf("%s/cron-run-all-md2nroff-logs-%04d-%02d-%02d-%02d%02d",
- $logfile_dir_arg, $year + 1900, $mon + 1, $mday,
- $hour, $min);
-my $rc = system("mkdir $logfile_dir");
-if ($rc != 0 || ! -d $logfile_dir || ! -w $logfile_dir) {
- chdir("/");
- die "mkdir of $logfile_dir failed, or can't write to it";
-}
-
-my $tmpdir = File::Temp->newdir();
-verbose("*** Working in: $tmpdir\n");
-chdir($tmpdir);
-
-# First, git clone the source branch of the repo
-verbose("*** Cloning repo: $repo_arg / $source_branch_arg...\n");
-doit(0, "git clone --single-branch --branch $source_branch_arg $repo_arg source", "git-clone");
-
-# Next, git clone the pages branch of repo
-if (defined($pages_branch_arg)) {
- verbose("*** Cloning repo: $repo_arg / $pages_branch_arg...\n");
- doit(0, "git clone --single-branch --branch $pages_branch_arg $repo_arg pages", "git-clone2");
-}
-
-#####################################################################
-
-# Find all the *.\d.md files in the source repo
-verbose("*** Finding markdown man pages...\n");
-opendir(DIR, "source/man");
-my @markdown_files = grep { /\.\d\.md$/ && -f "source/man/$_" } readdir(DIR);
-closedir(DIR);
-verbose("Found: @markdown_files\n");
-
-#####################################################################
-
-# Copy each of the markdown files to the pages branch checkout
-if (defined($pages_branch_arg)) {
- chdir("pages/master");
- foreach my $file (@markdown_files) {
- doit(0, "cp ../../source/man/$file man/$file", "loop-cp");
-
- # Is there a new man page? If so, we need to "git add" it.
- my $out = `git status --porcelain man/$file`;
- doit(0, "git add man/$file", "loop-git-add")
- if ($out =~ /^\?\?/);
- }
-
- # Generate a new index.md with all the files that we just
- # published. First, read in the header stub.
- if (!open(IN, "man/index-head.txt")) {
- chdir("/");
- die("failed to open index-head.txt");
- }
- my $str;
- $str .= $_
- while (<IN>);
- close(IN);
-
- # Write out the header stub into index.md itself
- if (!open(OUT, ">man/index.md")) {
- chdir("/");
- die("failed to write to new index.md file");
- }
- print OUT $str;
-
- # Now write out all the pages
- my @headings;
- push(@headings, { section=>7, title=>"General information" });
- push(@headings, { section=>3, title=>"API documentation" });
- foreach my $h (@headings) {
- print OUT "\n* $h->{title}\n";
- foreach my $file (sort(@markdown_files)) {
- if ($file =~ /\.$h->{section}\.md$/) {
- $file =~ m/^(.+)\.$h->{section}\.md$/;
- my $base = $1;
- print OUT " * [$base($h->{section})]($base.$h->{section}.html)\n";
- }
- }
- }
- close(OUT);
-
- # Git commit those files in the pages repo and push them to the
- # upstream repo so that they go live. If nothing changed, the commit
- # and push will be no-ops.
- chdir("..");
- doit(1, "git commit --no-verify -a -m \"Updated Markdown man pages from $source_branch_arg\"",
- "git-commit-first");
- doit(1, "git push", "git-push-first");
-}
-
-#####################################################################
-
-# Now process each of the Markdown files in the source repo and
-# generate new nroff man pages.
-chdir("$tmpdir/source/man");
-foreach my $file (@markdown_files) {
- doit(0, "../config/md2nroff.pl --source $file", "loop2-md2nroff");
-
- # Did we generate a new man page? If so, we need to "git add" it.
- my $man_file = basename($file);
-
- $man_file =~ m/\.(\d)\.md$/;
- my $section = $1;
-
- $man_file =~ s/\.md$//;
-
- my $full_filename = "man$section/$man_file";
-
- my $out = `git status --porcelain $full_filename`;
- doit(0, "git add $full_filename", "loop2-git-add")
- if ($out =~ /^\?\?/);
-}
-
-# Similar to above: commit the newly-generated nroff pages and push
-# them back upstream. If nothing changed, these will be no-ops.
-doit(1, "git commit --no-verify -a -m \"Updated nroff-generated man pages\"", "git-commit-final");
-doit(1, "git push", "git-push-final");
-
-# chdir out of the tmpdir so that it can be removed
-chdir("/");
-
-# If we get here, we finished successfully, so there's no need to keep
-# the logfile dir around
-system("rm -rf $logfile_dir");
-
-exit(0);
diff --git a/config/cron-submit-coverity.pl b/config/cron-submit-coverity.pl
deleted file mode 100755
index bce6348..0000000
--- a/config/cron-submit-coverity.pl
+++ /dev/null
@@ -1,147 +0,0 @@
-#!/usr/bin/env perl
-
-use warnings;
-use strict;
-
-use Getopt::Long;
-use File::Temp qw/ tempfile tempdir /;
-use File::Basename;
-
-my $filename_arg;
-my $coverity_token_arg;
-my $project_arg = "ofiwg%2Flibfabric";
-my $dry_run_arg = 0;
-my $verbose_arg = 0;
-my $debug_arg = 0;
-my $logfile_dir_arg;
-my $configure_args = "";
-my $make_args = "-j 32";
-my $help_arg = 0;
-
-&Getopt::Long::Configure("bundling");
-my $ok = Getopt::Long::GetOptions("filename=s" => \$filename_arg,
- "coverity-token=s" => \$coverity_token_arg,
- "project=s" => \$project_arg,
- "logfile-dir=s" => \$logfile_dir_arg,
- "configure-args=s" => \$configure_args,
- "make-args=s" => \$make_args,
- "dry-run!" => \$dry_run_arg,
- "verbose!" => \$verbose_arg,
- "debug!" => \$debug_arg,
- "help|h" => \$help_arg);
-
-$ok = 0
- if (!defined($filename_arg));
-$ok = 0
- if (!defined($coverity_token_arg));
-if (!$ok || $help_arg) {
- print "Usage: $0 --filename=FILENAME --coverity-token=TOKEN [--dry-run] [--verbose] [--help]\n";
- exit($ok);
-}
-
-die "Cannot read $filename_arg"
- if (! -r $filename_arg);
-
-$verbose_arg = 1
- if ($debug_arg);
-
-######################################################################
-
-sub verbose {
- print @_
- if ($verbose_arg);
-}
-
-# run a command and save the stdout / stderr
-sub doit {
- my $allowed_to_fail = shift;
- my $cmd = shift;
- my $stdout_file = shift;
-
- # Redirect stdout if requested or not verbose
- if (defined($stdout_file)) {
- $stdout_file = "$logfile_dir_arg/$stdout_file.log";
- unlink($stdout_file);
- $cmd .= " >$stdout_file";
- } elsif (!$debug_arg) {
- $cmd .= " >/dev/null";
- }
- $cmd .= " 2>&1";
-
- my $rc = system($cmd);
- if (0 != $rc && !$allowed_to_fail) {
- # If we die/fail, ensure to change out of the temp tree so
- # that it can be removed upon exit.
- chdir("/");
- die "Command $cmd failed: exit status $rc";
- }
- system("cat $stdout_file")
- if ($debug_arg && defined($stdout_file) && -f $stdout_file);
-}
-
-######################################################################
-
-# Make an area to work
-
-my $dir = tempdir(CLEANUP => 1);
-chdir($dir);
-verbose "*** Working in $dir\n";
-
-######################################################################
-
-# Get the coverity tool, put it in our path
-
-verbose "*** Downloading coverity tool\n";
-doit(0, "wget https://scan.coverity.com/download/linux-64 --post-data \"token=$coverity_token_arg\&project=$project_arg\" -O coverity_tool.tgz");
-doit(0, "tar xf coverity_tool.tgz");
-opendir(my $dh, ".") ||
- die "Can't opendir .";
-my @files = grep { /^cov/ && -d "./$_" } readdir($dh);
-closedir($dh);
-
-my $cov_dir = "$dir/$files[0]/bin";
-$ENV{PATH} = "$cov_dir:$ENV{PATH}";
-
-######################################################################
-
-# Expand the tarball, build it
-
-verbose "*** Extracting tarball\n";
-doit(0, "tar xf $filename_arg");
-my $base_name = $project_arg;
-$base_name =~ s/^ofiwg(%2F|-)([a-z]+)$/$2/;
-my $tarball_filename = basename($filename_arg);
-$tarball_filename =~ m/^\Q$base_name\E-(.+)\.tar.+$/;
-my $version = $1;
-chdir("$base_name-$version");
-
-verbose "*** Configuring tarball\n";
-doit(0, "./configure $configure_args", "coverity-configure");
-
-verbose "*** Building libtarball\n";
-doit(0, "cov-build --dir cov-int make $make_args", "coverity-build");
-
-# Tar up the Coverity results
-verbose "*** Tarring up results\n";
-doit(0, "tar jcf $version-analyzed.tar.bz2 cov-int");
-
-# If not dry-run, submit to Coverity
-if ($dry_run_arg) {
- verbose "*** Would have submitted, but this is a dry run\n";
-} else {
- verbose "*** Submitting results\n";
- doit(0, "curl --form token=$coverity_token_arg " .
- "--form email=ofiwg-bot\@openfabrics.org " .
- "--form file=\@$version-analyzed.tar.bz2 " .
- "--form version=$version " .
- "--form description=nightly-master " .
- "https://scan.coverity.com/builds?project=$project_arg",
- "coverity-submit");
-}
-
-verbose("*** All done / Coverity\n");
-
-# Chdir out of the tempdir so that it can be removed
-chdir("/");
-
-exit(0);
diff --git a/config/depcomp b/config/depcomp
new file mode 100755
index 0000000..df8eea7
--- /dev/null
+++ b/config/depcomp
@@ -0,0 +1,630 @@
+#! /bin/sh
+# depcomp - compile a program generating dependencies as side-effects
+
+scriptversion=2009-04-28.21; # UTC
+
+# Copyright (C) 1999, 2000, 2003, 2004, 2005, 2006, 2007, 2009 Free
+# Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Originally written by Alexandre Oliva <oliva at dcc.unicamp.br>.
+
+case $1 in
+ '')
+ echo "$0: No command. Try \`$0 --help' for more information." 1>&2
+ exit 1;
+ ;;
+ -h | --h*)
+ cat <<\EOF
+Usage: depcomp [--help] [--version] PROGRAM [ARGS]
+
+Run PROGRAMS ARGS to compile a file, generating dependencies
+as side-effects.
+
+Environment variables:
+ depmode Dependency tracking mode.
+ source Source file read by `PROGRAMS ARGS'.
+ object Object file output by `PROGRAMS ARGS'.
+ DEPDIR directory where to store dependencies.
+ depfile Dependency file to output.
+ tmpdepfile Temporary file to use when outputing dependencies.
+ libtool Whether libtool is used (yes/no).
+
+Report bugs to <bug-automake at gnu.org>.
+EOF
+ exit $?
+ ;;
+ -v | --v*)
+ echo "depcomp $scriptversion"
+ exit $?
+ ;;
+esac
+
+if test -z "$depmode" || test -z "$source" || test -z "$object"; then
+ echo "depcomp: Variables source, object and depmode must be set" 1>&2
+ exit 1
+fi
+
+# Dependencies for sub/bar.o or sub/bar.obj go into sub/.deps/bar.Po.
+depfile=${depfile-`echo "$object" |
+ sed 's|[^\\/]*$|'${DEPDIR-.deps}'/&|;s|\.\([^.]*\)$|.P\1|;s|Pobj$|Po|'`}
+tmpdepfile=${tmpdepfile-`echo "$depfile" | sed 's/\.\([^.]*\)$/.T\1/'`}
+
+rm -f "$tmpdepfile"
+
+# Some modes work just like other modes, but use different flags. We
+# parameterize here, but still list the modes in the big case below,
+# to make depend.m4 easier to write. Note that we *cannot* use a case
+# here, because this file can only contain one case statement.
+if test "$depmode" = hp; then
+ # HP compiler uses -M and no extra arg.
+ gccflag=-M
+ depmode=gcc
+fi
+
+if test "$depmode" = dashXmstdout; then
+ # This is just like dashmstdout with a different argument.
+ dashmflag=-xM
+ depmode=dashmstdout
+fi
+
+cygpath_u="cygpath -u -f -"
+if test "$depmode" = msvcmsys; then
+ # This is just like msvisualcpp but w/o cygpath translation.
+ # Just convert the backslash-escaped backslashes to single forward
+ # slashes to satisfy depend.m4
+ cygpath_u="sed s,\\\\\\\\,/,g"
+ depmode=msvisualcpp
+fi
+
+case "$depmode" in
+gcc3)
+## gcc 3 implements dependency tracking that does exactly what
+## we want. Yay! Note: for some reason libtool 1.4 doesn't like
+## it if -MD -MP comes after the -MF stuff. Hmm.
+## Unfortunately, FreeBSD c89 acceptance of flags depends upon
+## the command line argument order; so add the flags where they
+## appear in depend2.am. Note that the slowdown incurred here
+## affects only configure: in makefiles, %FASTDEP% shortcuts this.
+ for arg
+ do
+ case $arg in
+ -c) set fnord "$@" -MT "$object" -MD -MP -MF "$tmpdepfile" "$arg" ;;
+ *) set fnord "$@" "$arg" ;;
+ esac
+ shift # fnord
+ shift # $arg
+ done
+ "$@"
+ stat=$?
+ if test $stat -eq 0; then :
+ else
+ rm -f "$tmpdepfile"
+ exit $stat
+ fi
+ mv "$tmpdepfile" "$depfile"
+ ;;
+
+gcc)
+## There are various ways to get dependency output from gcc. Here's
+## why we pick this rather obscure method:
+## - Don't want to use -MD because we'd like the dependencies to end
+## up in a subdir. Having to rename by hand is ugly.
+## (We might end up doing this anyway to support other compilers.)
+## - The DEPENDENCIES_OUTPUT environment variable makes gcc act like
+## -MM, not -M (despite what the docs say).
+## - Using -M directly means running the compiler twice (even worse
+## than renaming).
+ if test -z "$gccflag"; then
+ gccflag=-MD,
+ fi
+ "$@" -Wp,"$gccflag$tmpdepfile"
+ stat=$?
+ if test $stat -eq 0; then :
+ else
+ rm -f "$tmpdepfile"
+ exit $stat
+ fi
+ rm -f "$depfile"
+ echo "$object : \\" > "$depfile"
+ alpha=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
+## The second -e expression handles DOS-style file names with drive letters.
+ sed -e 's/^[^:]*: / /' \
+ -e 's/^['$alpha']:\/[^:]*: / /' < "$tmpdepfile" >> "$depfile"
+## This next piece of magic avoids the `deleted header file' problem.
+## The problem is that when a header file which appears in a .P file
+## is deleted, the dependency causes make to die (because there is
+## typically no way to rebuild the header). We avoid this by adding
+## dummy dependencies for each header file. Too bad gcc doesn't do
+## this for us directly.
+ tr ' ' '
+' < "$tmpdepfile" |
+## Some versions of gcc put a space before the `:'. On the theory
+## that the space means something, we add a space to the output as
+## well.
+## Some versions of the HPUX 10.20 sed can't process this invocation
+## correctly. Breaking it into two sed invocations is a workaround.
+ sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile"
+ rm -f "$tmpdepfile"
+ ;;
+
+hp)
+ # This case exists only to let depend.m4 do its work. It works by
+ # looking at the text of this script. This case will never be run,
+ # since it is checked for above.
+ exit 1
+ ;;
+
+sgi)
+ if test "$libtool" = yes; then
+ "$@" "-Wp,-MDupdate,$tmpdepfile"
+ else
+ "$@" -MDupdate "$tmpdepfile"
+ fi
+ stat=$?
+ if test $stat -eq 0; then :
+ else
+ rm -f "$tmpdepfile"
+ exit $stat
+ fi
+ rm -f "$depfile"
+
+ if test -f "$tmpdepfile"; then # yes, the sourcefile depend on other files
+ echo "$object : \\" > "$depfile"
+
+ # Clip off the initial element (the dependent). Don't try to be
+ # clever and replace this with sed code, as IRIX sed won't handle
+ # lines with more than a fixed number of characters (4096 in
+ # IRIX 6.2 sed, 8192 in IRIX 6.5). We also remove comment lines;
+ # the IRIX cc adds comments like `#:fec' to the end of the
+ # dependency line.
+ tr ' ' '
+' < "$tmpdepfile" \
+ | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' | \
+ tr '
+' ' ' >> "$depfile"
+ echo >> "$depfile"
+
+ # The second pass generates a dummy entry for each header file.
+ tr ' ' '
+' < "$tmpdepfile" \
+ | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' -e 's/$/:/' \
+ >> "$depfile"
+ else
+ # The sourcefile does not contain any dependencies, so just
+ # store a dummy comment line, to avoid errors with the Makefile
+ # "include basename.Plo" scheme.
+ echo "#dummy" > "$depfile"
+ fi
+ rm -f "$tmpdepfile"
+ ;;
+
+aix)
+ # The C for AIX Compiler uses -M and outputs the dependencies
+ # in a .u file. In older versions, this file always lives in the
+ # current directory. Also, the AIX compiler puts `$object:' at the
+ # start of each line; $object doesn't have directory information.
+ # Version 6 uses the directory in both cases.
+ dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
+ test "x$dir" = "x$object" && dir=
+ base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
+ if test "$libtool" = yes; then
+ tmpdepfile1=$dir$base.u
+ tmpdepfile2=$base.u
+ tmpdepfile3=$dir.libs/$base.u
+ "$@" -Wc,-M
+ else
+ tmpdepfile1=$dir$base.u
+ tmpdepfile2=$dir$base.u
+ tmpdepfile3=$dir$base.u
+ "$@" -M
+ fi
+ stat=$?
+
+ if test $stat -eq 0; then :
+ else
+ rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3"
+ exit $stat
+ fi
+
+ for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3"
+ do
+ test -f "$tmpdepfile" && break
+ done
+ if test -f "$tmpdepfile"; then
+ # Each line is of the form `foo.o: dependent.h'.
+ # Do two passes, one to just change these to
+ # `$object: dependent.h' and one to simply `dependent.h:'.
+ sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile"
+ # That's a tab and a space in the [].
+ sed -e 's,^.*\.[a-z]*:[ ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile"
+ else
+ # The sourcefile does not contain any dependencies, so just
+ # store a dummy comment line, to avoid errors with the Makefile
+ # "include basename.Plo" scheme.
+ echo "#dummy" > "$depfile"
+ fi
+ rm -f "$tmpdepfile"
+ ;;
+
+icc)
+ # Intel's C compiler understands `-MD -MF file'. However on
+ # icc -MD -MF foo.d -c -o sub/foo.o sub/foo.c
+ # ICC 7.0 will fill foo.d with something like
+ # foo.o: sub/foo.c
+ # foo.o: sub/foo.h
+ # which is wrong. We want:
+ # sub/foo.o: sub/foo.c
+ # sub/foo.o: sub/foo.h
+ # sub/foo.c:
+ # sub/foo.h:
+ # ICC 7.1 will output
+ # foo.o: sub/foo.c sub/foo.h
+ # and will wrap long lines using \ :
+ # foo.o: sub/foo.c ... \
+ # sub/foo.h ... \
+ # ...
+
+ "$@" -MD -MF "$tmpdepfile"
+ stat=$?
+ if test $stat -eq 0; then :
+ else
+ rm -f "$tmpdepfile"
+ exit $stat
+ fi
+ rm -f "$depfile"
+ # Each line is of the form `foo.o: dependent.h',
+ # or `foo.o: dep1.h dep2.h \', or ` dep3.h dep4.h \'.
+ # Do two passes, one to just change these to
+ # `$object: dependent.h' and one to simply `dependent.h:'.
+ sed "s,^[^:]*:,$object :," < "$tmpdepfile" > "$depfile"
+ # Some versions of the HPUX 10.20 sed can't process this invocation
+ # correctly. Breaking it into two sed invocations is a workaround.
+ sed 's,^[^:]*: \(.*\)$,\1,;s/^\\$//;/^$/d;/:$/d' < "$tmpdepfile" |
+ sed -e 's/$/ :/' >> "$depfile"
+ rm -f "$tmpdepfile"
+ ;;
+
+hp2)
+ # The "hp" stanza above does not work with aCC (C++) and HP's ia64
+ # compilers, which have integrated preprocessors. The correct option
+ # to use with these is +Maked; it writes dependencies to a file named
+ # 'foo.d', which lands next to the object file, wherever that
+ # happens to be.
+ # Much of this is similar to the tru64 case; see comments there.
+ dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
+ test "x$dir" = "x$object" && dir=
+ base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
+ if test "$libtool" = yes; then
+ tmpdepfile1=$dir$base.d
+ tmpdepfile2=$dir.libs/$base.d
+ "$@" -Wc,+Maked
+ else
+ tmpdepfile1=$dir$base.d
+ tmpdepfile2=$dir$base.d
+ "$@" +Maked
+ fi
+ stat=$?
+ if test $stat -eq 0; then :
+ else
+ rm -f "$tmpdepfile1" "$tmpdepfile2"
+ exit $stat
+ fi
+
+ for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2"
+ do
+ test -f "$tmpdepfile" && break
+ done
+ if test -f "$tmpdepfile"; then
+ sed -e "s,^.*\.[a-z]*:,$object:," "$tmpdepfile" > "$depfile"
+ # Add `dependent.h:' lines.
+ sed -ne '2,${
+ s/^ *//
+ s/ \\*$//
+ s/$/:/
+ p
+ }' "$tmpdepfile" >> "$depfile"
+ else
+ echo "#dummy" > "$depfile"
+ fi
+ rm -f "$tmpdepfile" "$tmpdepfile2"
+ ;;
+
+tru64)
+ # The Tru64 compiler uses -MD to generate dependencies as a side
+ # effect. `cc -MD -o foo.o ...' puts the dependencies into `foo.o.d'.
+ # At least on Alpha/Redhat 6.1, Compaq CCC V6.2-504 seems to put
+ # dependencies in `foo.d' instead, so we check for that too.
+ # Subdirectories are respected.
+ dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
+ test "x$dir" = "x$object" && dir=
+ base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
+
+ if test "$libtool" = yes; then
+ # With Tru64 cc, shared objects can also be used to make a
+ # static library. This mechanism is used in libtool 1.4 series to
+ # handle both shared and static libraries in a single compilation.
+ # With libtool 1.4, dependencies were output in $dir.libs/$base.lo.d.
+ #
+ # With libtool 1.5 this exception was removed, and libtool now
+ # generates 2 separate objects for the 2 libraries. These two
+ # compilations output dependencies in $dir.libs/$base.o.d and
+ # in $dir$base.o.d. We have to check for both files, because
+ # one of the two compilations can be disabled. We should prefer
+ # $dir$base.o.d over $dir.libs/$base.o.d because the latter is
+ # automatically cleaned when .libs/ is deleted, while ignoring
+ # the former would cause a distcleancheck panic.
+ tmpdepfile1=$dir.libs/$base.lo.d # libtool 1.4
+ tmpdepfile2=$dir$base.o.d # libtool 1.5
+ tmpdepfile3=$dir.libs/$base.o.d # libtool 1.5
+ tmpdepfile4=$dir.libs/$base.d # Compaq CCC V6.2-504
+ "$@" -Wc,-MD
+ else
+ tmpdepfile1=$dir$base.o.d
+ tmpdepfile2=$dir$base.d
+ tmpdepfile3=$dir$base.d
+ tmpdepfile4=$dir$base.d
+ "$@" -MD
+ fi
+
+ stat=$?
+ if test $stat -eq 0; then :
+ else
+ rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4"
+ exit $stat
+ fi
+
+ for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4"
+ do
+ test -f "$tmpdepfile" && break
+ done
+ if test -f "$tmpdepfile"; then
+ sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile"
+ # That's a tab and a space in the [].
+ sed -e 's,^.*\.[a-z]*:[ ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile"
+ else
+ echo "#dummy" > "$depfile"
+ fi
+ rm -f "$tmpdepfile"
+ ;;
+
+#nosideeffect)
+ # This comment above is used by automake to tell side-effect
+ # dependency tracking mechanisms from slower ones.
+
+dashmstdout)
+ # Important note: in order to support this mode, a compiler *must*
+ # always write the preprocessed file to stdout, regardless of -o.
+ "$@" || exit $?
+
+ # Remove the call to Libtool.
+ if test "$libtool" = yes; then
+ while test "X$1" != 'X--mode=compile'; do
+ shift
+ done
+ shift
+ fi
+
+ # Remove `-o $object'.
+ IFS=" "
+ for arg
+ do
+ case $arg in
+ -o)
+ shift
+ ;;
+ $object)
+ shift
+ ;;
+ *)
+ set fnord "$@" "$arg"
+ shift # fnord
+ shift # $arg
+ ;;
+ esac
+ done
+
+ test -z "$dashmflag" && dashmflag=-M
+ # Require at least two characters before searching for `:'
+ # in the target name. This is to cope with DOS-style filenames:
+ # a dependency such as `c:/foo/bar' could be seen as target `c' otherwise.
+ "$@" $dashmflag |
+ sed 's:^[ ]*[^: ][^:][^:]*\:[ ]*:'"$object"'\: :' > "$tmpdepfile"
+ rm -f "$depfile"
+ cat < "$tmpdepfile" > "$depfile"
+ tr ' ' '
+' < "$tmpdepfile" | \
+## Some versions of the HPUX 10.20 sed can't process this invocation
+## correctly. Breaking it into two sed invocations is a workaround.
+ sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile"
+ rm -f "$tmpdepfile"
+ ;;
+
+dashXmstdout)
+ # This case only exists to satisfy depend.m4. It is never actually
+ # run, as this mode is specially recognized in the preamble.
+ exit 1
+ ;;
+
+makedepend)
+ "$@" || exit $?
+ # Remove any Libtool call
+ if test "$libtool" = yes; then
+ while test "X$1" != 'X--mode=compile'; do
+ shift
+ done
+ shift
+ fi
+ # X makedepend
+ shift
+ cleared=no eat=no
+ for arg
+ do
+ case $cleared in
+ no)
+ set ""; shift
+ cleared=yes ;;
+ esac
+ if test $eat = yes; then
+ eat=no
+ continue
+ fi
+ case "$arg" in
+ -D*|-I*)
+ set fnord "$@" "$arg"; shift ;;
+ # Strip any option that makedepend may not understand. Remove
+ # the object too, otherwise makedepend will parse it as a source file.
+ -arch)
+ eat=yes ;;
+ -*|$object)
+ ;;
+ *)
+ set fnord "$@" "$arg"; shift ;;
+ esac
+ done
+ obj_suffix=`echo "$object" | sed 's/^.*\././'`
+ touch "$tmpdepfile"
+ ${MAKEDEPEND-makedepend} -o"$obj_suffix" -f"$tmpdepfile" "$@"
+ rm -f "$depfile"
+ cat < "$tmpdepfile" > "$depfile"
+ sed '1,2d' "$tmpdepfile" | tr ' ' '
+' | \
+## Some versions of the HPUX 10.20 sed can't process this invocation
+## correctly. Breaking it into two sed invocations is a workaround.
+ sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile"
+ rm -f "$tmpdepfile" "$tmpdepfile".bak
+ ;;
+
+cpp)
+ # Important note: in order to support this mode, a compiler *must*
+ # always write the preprocessed file to stdout.
+ "$@" || exit $?
+
+ # Remove the call to Libtool.
+ if test "$libtool" = yes; then
+ while test "X$1" != 'X--mode=compile'; do
+ shift
+ done
+ shift
+ fi
+
+ # Remove `-o $object'.
+ IFS=" "
+ for arg
+ do
+ case $arg in
+ -o)
+ shift
+ ;;
+ $object)
+ shift
+ ;;
+ *)
+ set fnord "$@" "$arg"
+ shift # fnord
+ shift # $arg
+ ;;
+ esac
+ done
+
+ "$@" -E |
+ sed -n -e '/^# [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \
+ -e '/^#line [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' |
+ sed '$ s: \\$::' > "$tmpdepfile"
+ rm -f "$depfile"
+ echo "$object : \\" > "$depfile"
+ cat < "$tmpdepfile" >> "$depfile"
+ sed < "$tmpdepfile" '/^$/d;s/^ //;s/ \\$//;s/$/ :/' >> "$depfile"
+ rm -f "$tmpdepfile"
+ ;;
+
+msvisualcpp)
+ # Important note: in order to support this mode, a compiler *must*
+ # always write the preprocessed file to stdout.
+ "$@" || exit $?
+
+ # Remove the call to Libtool.
+ if test "$libtool" = yes; then
+ while test "X$1" != 'X--mode=compile'; do
+ shift
+ done
+ shift
+ fi
+
+ IFS=" "
+ for arg
+ do
+ case "$arg" in
+ -o)
+ shift
+ ;;
+ $object)
+ shift
+ ;;
+ "-Gm"|"/Gm"|"-Gi"|"/Gi"|"-ZI"|"/ZI")
+ set fnord "$@"
+ shift
+ shift
+ ;;
+ *)
+ set fnord "$@" "$arg"
+ shift
+ shift
+ ;;
+ esac
+ done
+ "$@" -E 2>/dev/null |
+ sed -n '/^#line [0-9][0-9]* "\([^"]*\)"/ s::\1:p' | $cygpath_u | sort -u > "$tmpdepfile"
+ rm -f "$depfile"
+ echo "$object : \\" > "$depfile"
+ sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s:: \1 \\:p' >> "$depfile"
+ echo " " >> "$depfile"
+ sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::\1\::p' >> "$depfile"
+ rm -f "$tmpdepfile"
+ ;;
+
+msvcmsys)
+ # This case exists only to let depend.m4 do its work. It works by
+ # looking at the text of this script. This case will never be run,
+ # since it is checked for above.
+ exit 1
+ ;;
+
+none)
+ exec "$@"
+ ;;
+
+*)
+ echo "Unknown depmode $depmode" 1>&2
+ exit 1
+ ;;
+esac
+
+exit 0
+
+# Local Variables:
+# mode: shell-script
+# sh-indentation: 2
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "scriptversion="
+# time-stamp-format: "%:y-%02m-%02d.%02H"
+# time-stamp-time-zone: "UTC"
+# time-stamp-end: "; # UTC"
+# End:
diff --git a/config/github-webhook/README b/config/github-webhook/README
deleted file mode 100644
index 8b854b2..0000000
--- a/config/github-webhook/README
+++ /dev/null
@@ -1,129 +0,0 @@
-This directory contains the Github webhook for verifying that all
-commits on an ofiwg PR contain a "Signed-off-by" line, and its
-relevant configuration.
-
-The bot runs as a Ruby Sinatra server on flatbed.openfabrics.org. It
-was initially heavily inspired by
-http://git-scm.com/book/en/v2/GitHub-Scripting-GitHub.
-
-The original flatbed server was running RHEL (CentOS, actually, but
-close enough). As of Feb 2016, it is now Ubuntu. So this README
-shows info about both Ubunutu and RHEL (mainly for hysterical
-raisins).
-
-===================
-
-RHEL background:
-
-Sadly, I did not take good notes about how to get Sinatra installed on
-Flatbed. Looking on flatbed on 17 Feb 2016, I see the following RPMs
-installed:
-
-- rubygems
-- rubygem-sinatra
-
-These seem to be the key RPMs to install. Hopefully, they're in the
-CentOS yum repo -- I do not have any notes on where I got them from.
-The following RPMs seem to be dependencies of Sinatra; hopefully yum
-just pull them in:
-
-- rubygem-crack
-- rubygem-daemons
-- rubygem-rack-test
-- rubygem-httparty
-- rubygem-eventmachine
-- rubygem-thin
-- rubygem-rack
-- rubygem-json
-
-Ubuntu backgruond:
-
-On Ubuntu, it was straightforward to install the Sinatra server:
-
-# apt-get install ruby-sinatra ruby-htttparty
-
-===================
-
-There are several important files in this directory:
-
-1. github-webhook-ofiwg-init.d-rhel and
- github-webhook-ofiwg-init.d-ubuntu
-
-The /etc/init.d/github-webhook-ofiwg (one for RHEL and one for Ubuntu)
-script that starts and stops the Ruby Sinatra server script that
-actually listens for the incoming Github webhook pings.
-
-2. verify-signed-off.rb
-
-The Ruby Sinatra server (i.e., a Ruby script that uses the Sinatra
-framework for listening for incoming HTTP requests) that listens for
-Github webhook pings. When it gets an incoming ping, if it's for a
-pull request, the script gets a listing of all the commits on that
-pull request and verifies that they all have a "Signed-off-by" line in
-them. If they all do, all commits have a "success" status set (at
-Github). If any are missing this line, then those commits have a
-"failed" status set, and the last commit on the PR also has a "failed"
-status set (because Github shows the overall status of the PR as the
-last commit's status).
-
-Every time commits are pushed to the PR, all the commits will be
-checked again. This ensures that even if you replace all the commits,
-we'll check them all (it's one HTTP request to get all the commit
-messages, anyway, so it's not like it makes the check take
-significantly longer).
-
-Note that the Ruby Sinatra server listens on localhost port 5000; it
-is not directly accessible to the outside world.
-
-On flatbed.openfabrics.org, the file is located in:
-
-RHEL: /var/www/github-webhooks/ofiwg/verify-signed-off.rb
-Ubuntu: /var/www/html/github-webhooks/ofiwg/verify-signed-off.rb
-
-3. apache-config.txt
-
-This is a snipit from flatbed.openfabrics.org:
-
-RHEL: /etc/httpd/conf.d/20-ssl.conf
-Ubuntu: etc/apache2/mods-enabled/proxy.conf
-
-This snipit is inserted in the SSL version of www.openfabrics.org
-config to enable the reverse proxy (i.e., incoming web requests to
-https://www.openfabrics.org/github-webhooks/ofiwg get routed
-internally to http://localhost:5000/ -- i.e., the Ruby Sinatra
-server).
-
-On Ubuntu, I had to load the mod_proxy Apache module by:
-
-# a2enmod proxy
-# a2enmod proxy_http
-# cd /etc/apache2/mods-enabled
-# ln -s ../mods-available/proxy.conf
- ^^ This is the file to add the apache proxy config
-
-===================
-
-There is one additional file needed on flatbed.openfabrics.org:
-
-RHEL: /etc/sysconfig/github-webhook-ofiwg
-Ubuntu: /etc/default/github-webhook-ofiwg
-
-The init.d script will source this file; it needs to set an
-environment variable that verify-signed-off.rb will look for:
-$GITHUB_AUTH_TOKEN.
-
-Specifically, this file should contain a line like this:
-
-export GITHUB_AUTH_TOKEN="...the token..."
-
-(this file is not here in the git repo, of course -- because then
-everyone could see our auth token!)
-
-A Github auth token can be obtained from
-https://github.com/settings/tokens of a user who has commit access to
-the target Github repo(s). The token needs to have the following
-permissions when you create it:
-
-- repo
-- public_repo
-- repo:status
diff --git a/config/github-webhook/apache-config.txt b/config/github-webhook/apache-config.txt
deleted file mode 100644
index a3bfbd4..0000000
--- a/config/github-webhook/apache-config.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# Github webhooks for the ofiwg project.
-# This is a reverse proxy to a trivial Ruby Sinatra server running on
-# the localhost (and is not otherwise accessible to the outside
-# world).
-ProxyPass /github-webhooks/ofiwg/ http://127.0.0.1:5000/
-<Location /github-webhooks/ofiwg/>
- ProxyPassReverse http://127.0.0.1:5000/
- RequestHeader unset Accept-Encoding
-</Location>
diff --git a/config/github-webhook/github-webhook-ofiwg-init.d-rhel b/config/github-webhook/github-webhook-ofiwg-init.d-rhel
deleted file mode 100755
index 6f0a361..0000000
--- a/config/github-webhook/github-webhook-ofiwg-init.d-rhel
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/bin/bash
-#
-# github-webhook-ofiwg This starts and stops the ofiwg github webhook Ruby Sinatra server
-#
-# chkconfig: 345 56 50
-# description: Simple Ruby Sinatra server to answer Github webhook \
-# pings to check for Signed-off-by lines in Github \
-# pull request commit messages.
-#
-# processname: /usr/bin/ruby
-# pidfile: /var/run/github-webhook-ofiwg.pid
-
-
-### BEGIN INIT INFO
-# Provides:
-# Required-Start: $network
-# Required-Stop:
-# Should-Start:
-# Should-Stop:
-# Default-Start: 3 4 5
-# Default-Stop: 0 1 2 6
-# Short-Description: start and stop github-webhook-ofiwg
-# Description: Simple Ruby Sinatra server to answer Github webhook \
-# pings to check for Signed-off-by lines in Github \
-# pull request commit messages.
-### END INIT INFO
-
-PATH=/sbin:/bin:/usr/bin:/usr/sbin
-
-# Source function library.
-. /etc/init.d/functions
-
-# Get config.
-test -f /etc/sysconfig/network && . /etc/sysconfig/network
-
-# Need to export GITHUB_AUTH_TOKEN in this file. This token should be
-# obtained from https://github.com/settings/tokens for a user who is
-# authorized to set status messages on the target repo(s).
-test -f /etc/sysconfig/github-webhook-ofiwg && . /etc/sysconfig/github-webhook-ofiwg
-
-RETVAL=0
-prog="/var/www/github-webhooks/ofiwg/verify-signed-off.rb"
-desc="Github webhook listener for ofiwg"
-pidfile="/var/run/github-webhook-ofiwg.pid"
-
-start(){
- [ -x $prog ] || exit 5
- # this is suitable way considering SELinux is guarding write
- # access to PID file
- [ $EUID -eq 0 ] || exit 4
-
- echo -n $"Starting $desc: "
-
- pid=`ps -eadf | grep "ruby $prog" | grep -v grep | awk '{ print $2 }'`
- if test -z "$pid"; then
- $prog > /dev/null 2>&1 &
- RETVAL=$?
- else
- RETVAL=1
- fi
- [ $RETVAL -eq 0 ] && echo_success
- [ $RETVAL -ne 0 ] && echo_failure
- echo
- return $RETVAL
-}
-
-stop(){
- [ -x $prog ] || exit 5
- # this is suitable way considering SELinux is guarding write
- # access to PID file
- [ $EUID -eq 0 ] || exit 4
-
- echo -n $"Stopping $desc: "
- pid=`ps -eadf | grep "ruby $prog" | grep -v grep | awk '{ print $2 }'`
- if test -n "$pid"; then
- kill $pid > /dev/null 2>&1
- RETVAL=$?
- else
- RETVAL=1
- fi
- [ $RETVAL -eq 0 ] && echo_success
- [ $RETVAL -ne 0 ] && echo_failure
- echo
- return $RETVAL
-
-}
-
-restart(){
- stop
- sleep 1
- start
-}
-
-# See how we were called.
-case "$1" in
- start)
- start
- RETVAL=$?
- ;;
- stop)
- stop
- RETVAL=$?
- ;;
- status)
- status $prog
- RETVAL=$?
- ;;
- restart)
- restart
- RETVAL=$?
- ;;
- *)
- echo $"Usage: $0 {start|stop|status|restart}"
- RETVAL=2
-esac
-
-exit $RETVAL
diff --git a/config/github-webhook/github-webhook-ofiwg-init.d-ubuntu b/config/github-webhook/github-webhook-ofiwg-init.d-ubuntu
deleted file mode 100755
index 29d0161..0000000
--- a/config/github-webhook/github-webhook-ofiwg-init.d-ubuntu
+++ /dev/null
@@ -1,85 +0,0 @@
-#! /bin/sh
-### BEGIN INIT INFO
-# Provides: github-webhook-ofiwg
-# Required-Start:
-# Required-Stop:
-# Default-Start:
-# Default-Stop: 0
-# Short-Description: Run the Github webhook for ofiwg
-# Description:
-### END INIT INFO
-
-PATH=/sbin:/usr/sbin:/bin:/usr/bin
-[ -f /etc/default/github-webhook-ofiwg ] && . /etc/default/github-webhook-ofiwg
-
-. /lib/lsb/init-functions
-
-RETVAL=0
-prog="/var/www/html/github-webhooks/ofiwg/verify-signed-off.rb"
-desc="Github webhook listener for ofiwg"
-pidfile="/var/run/github-webhook-ofiwg.pid"
-
-do_start() {
- [ -x $prog ] || exit 5
-
- log_action_msg "Starting $desc"
-
- pid=`ps -eadf | grep "ruby $prog" | grep -v grep | awk '{ print $2 }'`
- if test -z "$pid"; then
- $prog > /dev/null 2>&1 &
- RETVAL=$?
- else
- RETVAL=1
- fi
- [ $RETVAL -eq 0 ] && log_success_msg
- [ $RETVAL -ne 0 ] && log_failure_msg
- log_end_msg $RETVAL
- return $RETVAL
-}
-
-do_stop() {
- [ -x $prog ] || exit 5
-
- log_action_msg "Stopping $desc"
- pid=`ps -eadf | grep "ruby $prog" | grep -v grep | awk '{ print $2 }'`
- if test -n "$pid"; then
- kill $pid > /dev/null 2>&1
- RETVAL=$?
- else
- RETVAL=1
- fi
- [ $RETVAL -eq 0 ] && log_success_msg
- [ $RETVAL -ne 0 ] && log_failure_msg
- log_end_msg $RETVAL
- return $RETVAL
-}
-
-do_restart() {
- do_stop
- sleep 1
- do_start
-}
-
-case "$1" in
- start)
- do_start
- ;;
- restart)
- do_restart
- ;;
- reload|force-reload)
- log_action_msg "Error: argument '$1' not supported"
- log_action_end_msg 7
- exit 7
- ;;
- stop)
- do_stop
- ;;
- *)
- log_action_msg "Usage: $0 start|stop|restart"
- log_action_end_msg 3
- exit 3
- ;;
-esac
-
-exit $RETVAL
diff --git a/config/github-webhook/verify-signed-off.rb b/config/github-webhook/verify-signed-off.rb
deleted file mode 100755
index 3ecfba1..0000000
--- a/config/github-webhook/verify-signed-off.rb
+++ /dev/null
@@ -1,127 +0,0 @@
-#!/usr/bin/env ruby
-#
-# Simple(ish) Ruby Sinatra bot to verify that all commits on a Github
-# pull request have a Signed-off-by line in the commit message.
-#
-# Note that this code was heavily inspired by
-# http://git-scm.com/book/en/v2/GitHub-Scripting-GitHub
-#
-##############################################################################
-
-require 'rubygems'
-require 'httparty'
-require 'sinatra'
-require 'json'
-
-# Put Sinatra on flatbed port 5000 (a fairly arbitrary choice)
-set :port, 5000
-
-# Globals
-user_agent = 'ofiwg/signed-off-by-checker'
-
-# Read the Github auth token in from the environment (it wouldn't do
-# the hard-code it where it would show up in a public Github repo!)
-auth_token = ENV['GITHUB_AUTH_TOKEN']
-if auth_token == nil then
- puts "Someone forgot to set \$GITHUB_AUTH_TOKEN before launching me. Aborting!\n"
- exit 1
-end
-
-# Github webhooks are not delivered as HTTP GETs. If we don't have
-# this Sinatra method here, Sinatra displays a goofy error message if
-# someone just visits "https://url_to_this_script".
-get '/' do
- 'Nothing to see here but us chickens'
-end
-
-# Github delivers webhooks via HTTP POSTs.
-post '/' do
- push = JSON.parse(request.body.read)
- repo_name = push['repository']['full_name']
-
- # If this is not a push on a pull request (i.e., if there's no
- # commits to examine, such as if this is a test webhook ping from
- # github), then just return HTTP status 200 (i.e., success) with a
- # handy message that you can see in the Github webhook debug logs.
- if push['action'] == nil || (push['action'] != 'synchronize' &&
- push['action'] != 'opened') then
- return [200, 'This is not a pull request opened or synchronize push; nothing for this bot to do!']
- end
-
- # This webhook will have only delivered the *new* commits on this
- # PR. We need to examine *all* the commits -- so we can discard the
- # commits that were delivered in this webhook push. Instead, do a
- # fetch to get all the commits on this PR (note: putting the
- # Authorization header in this request just in case this is a
- # private repo).
- commits_url = push['pull_request']['commits_url']
- commits = HTTParty.get(commits_url,
- :headers => {
- 'Content-Type' => 'application/json',
- 'User-Agent' => user_agent,
- 'Authorization' => "token #{auth_token}" }
- )
-
- # Sanity check: If we've got no commits, do nothing.
- if commits.length == 0 then
- return [200, 'Somehow there are no commits on this PR... Weird...']
- end
-
- # Ok, we have commits. Go through them.
- happy = true
- targetURL = 'https://github.com/ofiwg/libfabric/wiki#how-to-contribute'
- debug_message = "checking debug URL: #{commits_url}\n\n"
- final_message = ''
-
- commits.each_with_index do |commit, index|
- sha = commit['sha']
- status_url = "https://api.github.com/repos/#{repo_name}/statuses/#{sha}"
- debug_message += "examining commit index #{index} / sha #{sha}:\nstatus url: #{status_url}\n\n#{commit.to_json}\n\n"
-
- status = {
- 'context' => 'Signed-off-by checker'
- }
-
- # Look for a Signed-off-by string in this commit
- if /Signed-off-by/.match commit['commit']['message']
- status['state'] = 'success'
- status['description'] = 'This commit is signed off'
- else
- status['state'] = 'failure'
- status['description'] = 'This commit is not signed off'
- status['target_url'] = targetURL
-
- happy = false
- end
- final_message = status['description']
-
- # If this is the last commit in the array (and there's more than
- # one commit in the array), override its state and description to
- # represent the entire PR (because Github shows the status of the
- # last commit as the status of the overall PR).
- if index == (commits.length - 1) && index > 0 then
- if happy then
- status['state'] = 'success'
- status['description'] = 'All commits were signed off. Yay!'
- else
- status['state'] = 'failure'
- status['description'] = 'At least one commit was not signed off'
- status['target_url'] = targetURL
- end
- final_message = status['description']
- end
-
- # Send the result back to Github for this specific commit
- HTTParty.post(status_url,
- :body => status.to_json,
- :headers => {
- 'Content-Type' => 'application/json',
- 'User-Agent' => user_agent,
- 'Authorization' => "token #{auth_token}" }
- )
- end
-
- # Return HTTP status 200 (success) and a handy message that shows up
- # in the Github webhook debug logs.
- return [200, "#{debug_message} -- #{final_message}"]
-end
diff --git a/config/install-sh b/config/install-sh
new file mode 100755
index 0000000..6781b98
--- /dev/null
+++ b/config/install-sh
@@ -0,0 +1,520 @@
+#!/bin/sh
+# install - install a program, script, or datafile
+
+scriptversion=2009-04-28.21; # UTC
+
+# This originates from X11R5 (mit/util/scripts/install.sh), which was
+# later released in X11R6 (xc/config/util/install.sh) with the
+# following copyright and license.
+#
+# Copyright (C) 1994 X Consortium
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC-
+# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# Except as contained in this notice, the name of the X Consortium shall not
+# be used in advertising or otherwise to promote the sale, use or other deal-
+# ings in this Software without prior written authorization from the X Consor-
+# tium.
+#
+#
+# FSF changes to this file are in the public domain.
+#
+# Calling this script install-sh is preferred over install.sh, to prevent
+# `make' implicit rules from creating a file called install from it
+# when there is no Makefile.
+#
+# This script is compatible with the BSD install script, but was written
+# from scratch.
+
+nl='
+'
+IFS=" "" $nl"
+
+# set DOITPROG to echo to test this script
+
+# Don't use :- since 4.3BSD and earlier shells don't like it.
+doit=${DOITPROG-}
+if test -z "$doit"; then
+ doit_exec=exec
+else
+ doit_exec=$doit
+fi
+
+# Put in absolute file names if you don't have them in your path;
+# or use environment vars.
+
+chgrpprog=${CHGRPPROG-chgrp}
+chmodprog=${CHMODPROG-chmod}
+chownprog=${CHOWNPROG-chown}
+cmpprog=${CMPPROG-cmp}
+cpprog=${CPPROG-cp}
+mkdirprog=${MKDIRPROG-mkdir}
+mvprog=${MVPROG-mv}
+rmprog=${RMPROG-rm}
+stripprog=${STRIPPROG-strip}
+
+posix_glob='?'
+initialize_posix_glob='
+ test "$posix_glob" != "?" || {
+ if (set -f) 2>/dev/null; then
+ posix_glob=
+ else
+ posix_glob=:
+ fi
+ }
+'
+
+posix_mkdir=
+
+# Desired mode of installed file.
+mode=0755
+
+chgrpcmd=
+chmodcmd=$chmodprog
+chowncmd=
+mvcmd=$mvprog
+rmcmd="$rmprog -f"
+stripcmd=
+
+src=
+dst=
+dir_arg=
+dst_arg=
+
+copy_on_change=false
+no_target_directory=
+
+usage="\
+Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE
+ or: $0 [OPTION]... SRCFILES... DIRECTORY
+ or: $0 [OPTION]... -t DIRECTORY SRCFILES...
+ or: $0 [OPTION]... -d DIRECTORIES...
+
+In the 1st form, copy SRCFILE to DSTFILE.
+In the 2nd and 3rd, copy all SRCFILES to DIRECTORY.
+In the 4th, create DIRECTORIES.
+
+Options:
+ --help display this help and exit.
+ --version display version info and exit.
+
+ -c (ignored)
+ -C install only if different (preserve the last data modification time)
+ -d create directories instead of installing files.
+ -g GROUP $chgrpprog installed files to GROUP.
+ -m MODE $chmodprog installed files to MODE.
+ -o USER $chownprog installed files to USER.
+ -s $stripprog installed files.
+ -t DIRECTORY install into DIRECTORY.
+ -T report an error if DSTFILE is a directory.
+
+Environment variables override the default commands:
+ CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG
+ RMPROG STRIPPROG
+"
+
+while test $# -ne 0; do
+ case $1 in
+ -c) ;;
+
+ -C) copy_on_change=true;;
+
+ -d) dir_arg=true;;
+
+ -g) chgrpcmd="$chgrpprog $2"
+ shift;;
+
+ --help) echo "$usage"; exit $?;;
+
+ -m) mode=$2
+ case $mode in
+ *' '* | *' '* | *'
+'* | *'*'* | *'?'* | *'['*)
+ echo "$0: invalid mode: $mode" >&2
+ exit 1;;
+ esac
+ shift;;
+
+ -o) chowncmd="$chownprog $2"
+ shift;;
+
+ -s) stripcmd=$stripprog;;
+
+ -t) dst_arg=$2
+ shift;;
+
+ -T) no_target_directory=true;;
+
+ --version) echo "$0 $scriptversion"; exit $?;;
+
+ --) shift
+ break;;
+
+ -*) echo "$0: invalid option: $1" >&2
+ exit 1;;
+
+ *) break;;
+ esac
+ shift
+done
+
+if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then
+ # When -d is used, all remaining arguments are directories to create.
+ # When -t is used, the destination is already specified.
+ # Otherwise, the last argument is the destination. Remove it from $@.
+ for arg
+ do
+ if test -n "$dst_arg"; then
+ # $@ is not empty: it contains at least $arg.
+ set fnord "$@" "$dst_arg"
+ shift # fnord
+ fi
+ shift # arg
+ dst_arg=$arg
+ done
+fi
+
+if test $# -eq 0; then
+ if test -z "$dir_arg"; then
+ echo "$0: no input file specified." >&2
+ exit 1
+ fi
+ # It's OK to call `install-sh -d' without argument.
+ # This can happen when creating conditional directories.
+ exit 0
+fi
+
+if test -z "$dir_arg"; then
+ trap '(exit $?); exit' 1 2 13 15
+
+ # Set umask so as not to create temps with too-generous modes.
+ # However, 'strip' requires both read and write access to temps.
+ case $mode in
+ # Optimize common cases.
+ *644) cp_umask=133;;
+ *755) cp_umask=22;;
+
+ *[0-7])
+ if test -z "$stripcmd"; then
+ u_plus_rw=
+ else
+ u_plus_rw='% 200'
+ fi
+ cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;;
+ *)
+ if test -z "$stripcmd"; then
+ u_plus_rw=
+ else
+ u_plus_rw=,u+rw
+ fi
+ cp_umask=$mode$u_plus_rw;;
+ esac
+fi
+
+for src
+do
+ # Protect names starting with `-'.
+ case $src in
+ -*) src=./$src;;
+ esac
+
+ if test -n "$dir_arg"; then
+ dst=$src
+ dstdir=$dst
+ test -d "$dstdir"
+ dstdir_status=$?
+ else
+
+ # Waiting for this to be detected by the "$cpprog $src $dsttmp" command
+ # might cause directories to be created, which would be especially bad
+ # if $src (and thus $dsttmp) contains '*'.
+ if test ! -f "$src" && test ! -d "$src"; then
+ echo "$0: $src does not exist." >&2
+ exit 1
+ fi
+
+ if test -z "$dst_arg"; then
+ echo "$0: no destination specified." >&2
+ exit 1
+ fi
+
+ dst=$dst_arg
+ # Protect names starting with `-'.
+ case $dst in
+ -*) dst=./$dst;;
+ esac
+
+ # If destination is a directory, append the input filename; won't work
+ # if double slashes aren't ignored.
+ if test -d "$dst"; then
+ if test -n "$no_target_directory"; then
+ echo "$0: $dst_arg: Is a directory" >&2
+ exit 1
+ fi
+ dstdir=$dst
+ dst=$dstdir/`basename "$src"`
+ dstdir_status=0
+ else
+ # Prefer dirname, but fall back on a substitute if dirname fails.
+ dstdir=`
+ (dirname "$dst") 2>/dev/null ||
+ expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+ X"$dst" : 'X\(//\)[^/]' \| \
+ X"$dst" : 'X\(//\)$' \| \
+ X"$dst" : 'X\(/\)' \| . 2>/dev/null ||
+ echo X"$dst" |
+ sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)[^/].*/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\).*/{
+ s//\1/
+ q
+ }
+ s/.*/./; q'
+ `
+
+ test -d "$dstdir"
+ dstdir_status=$?
+ fi
+ fi
+
+ obsolete_mkdir_used=false
+
+ if test $dstdir_status != 0; then
+ case $posix_mkdir in
+ '')
+ # Create intermediate dirs using mode 755 as modified by the umask.
+ # This is like FreeBSD 'install' as of 1997-10-28.
+ umask=`umask`
+ case $stripcmd.$umask in
+ # Optimize common cases.
+ *[2367][2367]) mkdir_umask=$umask;;
+ .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;;
+
+ *[0-7])
+ mkdir_umask=`expr $umask + 22 \
+ - $umask % 100 % 40 + $umask % 20 \
+ - $umask % 10 % 4 + $umask % 2
+ `;;
+ *) mkdir_umask=$umask,go-w;;
+ esac
+
+ # With -d, create the new directory with the user-specified mode.
+ # Otherwise, rely on $mkdir_umask.
+ if test -n "$dir_arg"; then
+ mkdir_mode=-m$mode
+ else
+ mkdir_mode=
+ fi
+
+ posix_mkdir=false
+ case $umask in
+ *[123567][0-7][0-7])
+ # POSIX mkdir -p sets u+wx bits regardless of umask, which
+ # is incompatible with FreeBSD 'install' when (umask & 300) != 0.
+ ;;
+ *)
+ tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
+ trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0
+
+ if (umask $mkdir_umask &&
+ exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1
+ then
+ if test -z "$dir_arg" || {
+ # Check for POSIX incompatibilities with -m.
+ # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
+ # other-writeable bit of parent directory when it shouldn't.
+ # FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
+ ls_ld_tmpdir=`ls -ld "$tmpdir"`
+ case $ls_ld_tmpdir in
+ d????-?r-*) different_mode=700;;
+ d????-?--*) different_mode=755;;
+ *) false;;
+ esac &&
+ $mkdirprog -m$different_mode -p -- "$tmpdir" && {
+ ls_ld_tmpdir_1=`ls -ld "$tmpdir"`
+ test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
+ }
+ }
+ then posix_mkdir=:
+ fi
+ rmdir "$tmpdir/d" "$tmpdir"
+ else
+ # Remove any dirs left behind by ancient mkdir implementations.
+ rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null
+ fi
+ trap '' 0;;
+ esac;;
+ esac
+
+ if
+ $posix_mkdir && (
+ umask $mkdir_umask &&
+ $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
+ )
+ then :
+ else
+
+ # The umask is ridiculous, or mkdir does not conform to POSIX,
+ # or it failed possibly due to a race condition. Create the
+ # directory the slow way, step by step, checking for races as we go.
+
+ case $dstdir in
+ /*) prefix='/';;
+ -*) prefix='./';;
+ *) prefix='';;
+ esac
+
+ eval "$initialize_posix_glob"
+
+ oIFS=$IFS
+ IFS=/
+ $posix_glob set -f
+ set fnord $dstdir
+ shift
+ $posix_glob set +f
+ IFS=$oIFS
+
+ prefixes=
+
+ for d
+ do
+ test -z "$d" && continue
+
+ prefix=$prefix$d
+ if test -d "$prefix"; then
+ prefixes=
+ else
+ if $posix_mkdir; then
+ (umask=$mkdir_umask &&
+ $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
+ # Don't fail if two instances are running concurrently.
+ test -d "$prefix" || exit 1
+ else
+ case $prefix in
+ *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
+ *) qprefix=$prefix;;
+ esac
+ prefixes="$prefixes '$qprefix'"
+ fi
+ fi
+ prefix=$prefix/
+ done
+
+ if test -n "$prefixes"; then
+ # Don't fail if two instances are running concurrently.
+ (umask $mkdir_umask &&
+ eval "\$doit_exec \$mkdirprog $prefixes") ||
+ test -d "$dstdir" || exit 1
+ obsolete_mkdir_used=true
+ fi
+ fi
+ fi
+
+ if test -n "$dir_arg"; then
+ { test -z "$chowncmd" || $doit $chowncmd "$dst"; } &&
+ { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } &&
+ { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false ||
+ test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1
+ else
+
+ # Make a couple of temp file names in the proper directory.
+ dsttmp=$dstdir/_inst.$$_
+ rmtmp=$dstdir/_rm.$$_
+
+ # Trap to clean up those temp files at exit.
+ trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0
+
+ # Copy the file name to the temp name.
+ (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") &&
+
+ # and set any options; do chmod last to preserve setuid bits.
+ #
+ # If any of these fail, we abort the whole thing. If we want to
+ # ignore errors from any of these, just make sure not to ignore
+ # errors from the above "$doit $cpprog $src $dsttmp" command.
+ #
+ { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } &&
+ { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } &&
+ { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } &&
+ { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } &&
+
+ # If -C, don't bother to copy if it wouldn't change the file.
+ if $copy_on_change &&
+ old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` &&
+ new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` &&
+
+ eval "$initialize_posix_glob" &&
+ $posix_glob set -f &&
+ set X $old && old=:$2:$4:$5:$6 &&
+ set X $new && new=:$2:$4:$5:$6 &&
+ $posix_glob set +f &&
+
+ test "$old" = "$new" &&
+ $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1
+ then
+ rm -f "$dsttmp"
+ else
+ # Rename the file to the real destination.
+ $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null ||
+
+ # The rename failed, perhaps because mv can't rename something else
+ # to itself, or perhaps because mv is so ancient that it does not
+ # support -f.
+ {
+ # Now remove or move aside any old file at destination location.
+ # We try this two ways since rm can't unlink itself on some
+ # systems and the destination file might be busy for other
+ # reasons. In this case, the final cleanup might fail but the new
+ # file should still install successfully.
+ {
+ test ! -f "$dst" ||
+ $doit $rmcmd -f "$dst" 2>/dev/null ||
+ { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null &&
+ { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; }
+ } ||
+ { echo "$0: cannot unlink or rename $dst" >&2
+ (exit 1); exit 1
+ }
+ } &&
+
+ # Now rename the file to the real destination.
+ $doit $mvcmd "$dsttmp" "$dst"
+ }
+ fi || exit 1
+
+ trap '' 0
+ fi
+done
+
+# Local variables:
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "scriptversion="
+# time-stamp-format: "%:y-%02m-%02d.%02H"
+# time-stamp-time-zone: "UTC"
+# time-stamp-end: "; # UTC"
+# End:
diff --git a/config/libtool.m4 b/config/libtool.m4
new file mode 100644
index 0000000..671cde1
--- /dev/null
+++ b/config/libtool.m4
@@ -0,0 +1,7360 @@
+# libtool.m4 - Configure libtool for the host system. -*-Autoconf-*-
+#
+# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, 2005,
+# 2006, 2007, 2008 Free Software Foundation, Inc.
+# Written by Gordon Matzigkeit, 1996
+#
+# This file is free software; the Free Software Foundation gives
+# unlimited permission to copy and/or distribute it, with or without
+# modifications, as long as this notice is preserved.
+
+m4_define([_LT_COPYING], [dnl
+# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, 2005,
+# 2006, 2007, 2008 Free Software Foundation, Inc.
+# Written by Gordon Matzigkeit, 1996
+#
+# This file is part of GNU Libtool.
+#
+# GNU Libtool is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of
+# the License, or (at your option) any later version.
+#
+# As a special exception to the GNU General Public License,
+# if you distribute this file as part of a program or library that
+# is built using GNU Libtool, you may include this file under the
+# same distribution terms that you use for the rest of that program.
+#
+# GNU Libtool is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Libtool; see the file COPYING. If not, a copy
+# can be downloaded from http://www.gnu.org/licenses/gpl.html, or
+# obtained by writing to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+])
+
+# serial 56 LT_INIT
+
+
+# LT_PREREQ(VERSION)
+# ------------------
+# Complain and exit if this libtool version is less that VERSION.
+m4_defun([LT_PREREQ],
+[m4_if(m4_version_compare(m4_defn([LT_PACKAGE_VERSION]), [$1]), -1,
+ [m4_default([$3],
+ [m4_fatal([Libtool version $1 or higher is required],
+ 63)])],
+ [$2])])
+
+
+# _LT_CHECK_BUILDDIR
+# ------------------
+# Complain if the absolute build directory name contains unusual characters
+m4_defun([_LT_CHECK_BUILDDIR],
+[case `pwd` in
+ *\ * | *\ *)
+ AC_MSG_WARN([Libtool does not cope well with whitespace in `pwd`]) ;;
+esac
+])
+
+
+# LT_INIT([OPTIONS])
+# ------------------
+AC_DEFUN([LT_INIT],
+[AC_PREREQ([2.58])dnl We use AC_INCLUDES_DEFAULT
+AC_BEFORE([$0], [LT_LANG])dnl
+AC_BEFORE([$0], [LT_OUTPUT])dnl
+AC_BEFORE([$0], [LTDL_INIT])dnl
+m4_require([_LT_CHECK_BUILDDIR])dnl
+
+dnl Autoconf doesn't catch unexpanded LT_ macros by default:
+m4_pattern_forbid([^_?LT_[A-Z_]+$])dnl
+m4_pattern_allow([^(_LT_EOF|LT_DLGLOBAL|LT_DLLAZY_OR_NOW|LT_MULTI_MODULE)$])dnl
+dnl aclocal doesn't pull ltoptions.m4, ltsugar.m4, or ltversion.m4
+dnl unless we require an AC_DEFUNed macro:
+AC_REQUIRE([LTOPTIONS_VERSION])dnl
+AC_REQUIRE([LTSUGAR_VERSION])dnl
+AC_REQUIRE([LTVERSION_VERSION])dnl
+AC_REQUIRE([LTOBSOLETE_VERSION])dnl
+m4_require([_LT_PROG_LTMAIN])dnl
+
+dnl Parse OPTIONS
+_LT_SET_OPTIONS([$0], [$1])
+
+# This can be used to rebuild libtool when needed
+LIBTOOL_DEPS="$ltmain"
+
+# Always use our own libtool.
+LIBTOOL='$(SHELL) $(top_builddir)/libtool'
+AC_SUBST(LIBTOOL)dnl
+
+_LT_SETUP
+
+# Only expand once:
+m4_define([LT_INIT])
+])# LT_INIT
+
+# Old names:
+AU_ALIAS([AC_PROG_LIBTOOL], [LT_INIT])
+AU_ALIAS([AM_PROG_LIBTOOL], [LT_INIT])
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AC_PROG_LIBTOOL], [])
+dnl AC_DEFUN([AM_PROG_LIBTOOL], [])
+
+
+# _LT_CC_BASENAME(CC)
+# -------------------
+# Calculate cc_basename. Skip known compiler wrappers and cross-prefix.
+m4_defun([_LT_CC_BASENAME],
+[for cc_temp in $1""; do
+ case $cc_temp in
+ compile | *[[\\/]]compile | ccache | *[[\\/]]ccache ) ;;
+ distcc | *[[\\/]]distcc | purify | *[[\\/]]purify ) ;;
+ \-*) ;;
+ *) break;;
+ esac
+done
+cc_basename=`$ECHO "X$cc_temp" | $Xsed -e 's%.*/%%' -e "s%^$host_alias-%%"`
+])
+
+
+# _LT_FILEUTILS_DEFAULTS
+# ----------------------
+# It is okay to use these file commands and assume they have been set
+# sensibly after `m4_require([_LT_FILEUTILS_DEFAULTS])'.
+m4_defun([_LT_FILEUTILS_DEFAULTS],
+[: ${CP="cp -f"}
+: ${MV="mv -f"}
+: ${RM="rm -f"}
+])# _LT_FILEUTILS_DEFAULTS
+
+
+# _LT_SETUP
+# ---------
+m4_defun([_LT_SETUP],
+[AC_REQUIRE([AC_CANONICAL_HOST])dnl
+AC_REQUIRE([AC_CANONICAL_BUILD])dnl
+_LT_DECL([], [host_alias], [0], [The host system])dnl
+_LT_DECL([], [host], [0])dnl
+_LT_DECL([], [host_os], [0])dnl
+dnl
+_LT_DECL([], [build_alias], [0], [The build system])dnl
+_LT_DECL([], [build], [0])dnl
+_LT_DECL([], [build_os], [0])dnl
+dnl
+AC_REQUIRE([AC_PROG_CC])dnl
+AC_REQUIRE([LT_PATH_LD])dnl
+AC_REQUIRE([LT_PATH_NM])dnl
+dnl
+AC_REQUIRE([AC_PROG_LN_S])dnl
+test -z "$LN_S" && LN_S="ln -s"
+_LT_DECL([], [LN_S], [1], [Whether we need soft or hard links])dnl
+dnl
+AC_REQUIRE([LT_CMD_MAX_LEN])dnl
+_LT_DECL([objext], [ac_objext], [0], [Object file suffix (normally "o")])dnl
+_LT_DECL([], [exeext], [0], [Executable file suffix (normally "")])dnl
+dnl
+m4_require([_LT_FILEUTILS_DEFAULTS])dnl
+m4_require([_LT_CHECK_SHELL_FEATURES])dnl
+m4_require([_LT_CMD_RELOAD])dnl
+m4_require([_LT_CHECK_MAGIC_METHOD])dnl
+m4_require([_LT_CMD_OLD_ARCHIVE])dnl
+m4_require([_LT_CMD_GLOBAL_SYMBOLS])dnl
+
+_LT_CONFIG_LIBTOOL_INIT([
+# See if we are running on zsh, and set the options which allow our
+# commands through without removal of \ escapes INIT.
+if test -n "\${ZSH_VERSION+set}" ; then
+ setopt NO_GLOB_SUBST
+fi
+])
+if test -n "${ZSH_VERSION+set}" ; then
+ setopt NO_GLOB_SUBST
+fi
+
+_LT_CHECK_OBJDIR
+
+m4_require([_LT_TAG_COMPILER])dnl
+_LT_PROG_ECHO_BACKSLASH
+
+case $host_os in
+aix3*)
+ # AIX sometimes has problems with the GCC collect2 program. For some
+ # reason, if we set the COLLECT_NAMES environment variable, the problems
+ # vanish in a puff of smoke.
+ if test "X${COLLECT_NAMES+set}" != Xset; then
+ COLLECT_NAMES=
+ export COLLECT_NAMES
+ fi
+ ;;
+esac
+
+# Sed substitution that helps us do robust quoting. It backslashifies
+# metacharacters that are still active within double-quoted strings.
+sed_quote_subst='s/\([["`$\\]]\)/\\\1/g'
+
+# Same as above, but do not quote variable references.
+double_quote_subst='s/\([["`\\]]\)/\\\1/g'
+
+# Sed substitution to delay expansion of an escaped shell variable in a
+# double_quote_subst'ed string.
+delay_variable_subst='s/\\\\\\\\\\\$/\\\\\\$/g'
+
+# Sed substitution to delay expansion of an escaped single quote.
+delay_single_quote_subst='s/'\''/'\'\\\\\\\'\''/g'
+
+# Sed substitution to avoid accidental globbing in evaled expressions
+no_glob_subst='s/\*/\\\*/g'
+
+# Global variables:
+ofile=libtool
+can_build_shared=yes
+
+# All known linkers require a `.a' archive for static linking (except MSVC,
+# which needs '.lib').
+libext=a
+
+with_gnu_ld="$lt_cv_prog_gnu_ld"
+
+old_CC="$CC"
+old_CFLAGS="$CFLAGS"
+
+# Set sane defaults for various variables
+test -z "$CC" && CC=cc
+test -z "$LTCC" && LTCC=$CC
+test -z "$LTCFLAGS" && LTCFLAGS=$CFLAGS
+test -z "$LD" && LD=ld
+test -z "$ac_objext" && ac_objext=o
+
+_LT_CC_BASENAME([$compiler])
+
+# Only perform the check for file, if the check method requires it
+test -z "$MAGIC_CMD" && MAGIC_CMD=file
+case $deplibs_check_method in
+file_magic*)
+ if test "$file_magic_cmd" = '$MAGIC_CMD'; then
+ _LT_PATH_MAGIC
+ fi
+ ;;
+esac
+
+# Use C for the default configuration in the libtool script
+LT_SUPPORTED_TAG([CC])
+_LT_LANG_C_CONFIG
+_LT_LANG_DEFAULT_CONFIG
+_LT_CONFIG_COMMANDS
+])# _LT_SETUP
+
+
+# _LT_PROG_LTMAIN
+# ---------------
+# Note that this code is called both from `configure', and `config.status'
+# now that we use AC_CONFIG_COMMANDS to generate libtool. Notably,
+# `config.status' has no value for ac_aux_dir unless we are using Automake,
+# so we pass a copy along to make sure it has a sensible value anyway.
+m4_defun([_LT_PROG_LTMAIN],
+[m4_ifdef([AC_REQUIRE_AUX_FILE], [AC_REQUIRE_AUX_FILE([ltmain.sh])])dnl
+_LT_CONFIG_LIBTOOL_INIT([ac_aux_dir='$ac_aux_dir'])
+ltmain="$ac_aux_dir/ltmain.sh"
+])# _LT_PROG_LTMAIN
+
+
+## ------------------------------------- ##
+## Accumulate code for creating libtool. ##
+## ------------------------------------- ##
+
+# So that we can recreate a full libtool script including additional
+# tags, we accumulate the chunks of code to send to AC_CONFIG_COMMANDS
+# in macros and then make a single call at the end using the `libtool'
+# label.
+
+
+# _LT_CONFIG_LIBTOOL_INIT([INIT-COMMANDS])
+# ----------------------------------------
+# Register INIT-COMMANDS to be passed to AC_CONFIG_COMMANDS later.
+m4_define([_LT_CONFIG_LIBTOOL_INIT],
+[m4_ifval([$1],
+ [m4_append([_LT_OUTPUT_LIBTOOL_INIT],
+ [$1
+])])])
+
+# Initialize.
+m4_define([_LT_OUTPUT_LIBTOOL_INIT])
+
+
+# _LT_CONFIG_LIBTOOL([COMMANDS])
+# ------------------------------
+# Register COMMANDS to be passed to AC_CONFIG_COMMANDS later.
+m4_define([_LT_CONFIG_LIBTOOL],
+[m4_ifval([$1],
+ [m4_append([_LT_OUTPUT_LIBTOOL_COMMANDS],
+ [$1
+])])])
+
+# Initialize.
+m4_define([_LT_OUTPUT_LIBTOOL_COMMANDS])
+
+
+# _LT_CONFIG_SAVE_COMMANDS([COMMANDS], [INIT_COMMANDS])
+# -----------------------------------------------------
+m4_defun([_LT_CONFIG_SAVE_COMMANDS],
+[_LT_CONFIG_LIBTOOL([$1])
+_LT_CONFIG_LIBTOOL_INIT([$2])
+])
+
+
+# _LT_FORMAT_COMMENT([COMMENT])
+# -----------------------------
+# Add leading comment marks to the start of each line, and a trailing
+# full-stop to the whole comment if one is not present already.
+m4_define([_LT_FORMAT_COMMENT],
+[m4_ifval([$1], [
+m4_bpatsubst([m4_bpatsubst([$1], [^ *], [# ])],
+ [['`$\]], [\\\&])]m4_bmatch([$1], [[!?.]$], [], [.])
+)])
+
+
+
+## ------------------------ ##
+## FIXME: Eliminate VARNAME ##
+## ------------------------ ##
+
+
+# _LT_DECL([CONFIGNAME], VARNAME, VALUE, [DESCRIPTION], [IS-TAGGED?])
+# -------------------------------------------------------------------
+# CONFIGNAME is the name given to the value in the libtool script.
+# VARNAME is the (base) name used in the configure script.
+# VALUE may be 0, 1 or 2 for a computed quote escaped value based on
+# VARNAME. Any other value will be used directly.
+m4_define([_LT_DECL],
+[lt_if_append_uniq([lt_decl_varnames], [$2], [, ],
+ [lt_dict_add_subkey([lt_decl_dict], [$2], [libtool_name],
+ [m4_ifval([$1], [$1], [$2])])
+ lt_dict_add_subkey([lt_decl_dict], [$2], [value], [$3])
+ m4_ifval([$4],
+ [lt_dict_add_subkey([lt_decl_dict], [$2], [description], [$4])])
+ lt_dict_add_subkey([lt_decl_dict], [$2],
+ [tagged?], [m4_ifval([$5], [yes], [no])])])
+])
+
+
+# _LT_TAGDECL([CONFIGNAME], VARNAME, VALUE, [DESCRIPTION])
+# --------------------------------------------------------
+m4_define([_LT_TAGDECL], [_LT_DECL([$1], [$2], [$3], [$4], [yes])])
+
+
+# lt_decl_tag_varnames([SEPARATOR], [VARNAME1...])
+# ------------------------------------------------
+m4_define([lt_decl_tag_varnames],
+[_lt_decl_filter([tagged?], [yes], $@)])
+
+
+# _lt_decl_filter(SUBKEY, VALUE, [SEPARATOR], [VARNAME1..])
+# ---------------------------------------------------------
+m4_define([_lt_decl_filter],
+[m4_case([$#],
+ [0], [m4_fatal([$0: too few arguments: $#])],
+ [1], [m4_fatal([$0: too few arguments: $#: $1])],
+ [2], [lt_dict_filter([lt_decl_dict], [$1], [$2], [], lt_decl_varnames)],
+ [3], [lt_dict_filter([lt_decl_dict], [$1], [$2], [$3], lt_decl_varnames)],
+ [lt_dict_filter([lt_decl_dict], $@)])[]dnl
+])
+
+
+# lt_decl_quote_varnames([SEPARATOR], [VARNAME1...])
+# --------------------------------------------------
+m4_define([lt_decl_quote_varnames],
+[_lt_decl_filter([value], [1], $@)])
+
+
+# lt_decl_dquote_varnames([SEPARATOR], [VARNAME1...])
+# ---------------------------------------------------
+m4_define([lt_decl_dquote_varnames],
+[_lt_decl_filter([value], [2], $@)])
+
+
+# lt_decl_varnames_tagged([SEPARATOR], [VARNAME1...])
+# ---------------------------------------------------
+m4_define([lt_decl_varnames_tagged],
+[m4_assert([$# <= 2])dnl
+_$0(m4_quote(m4_default([$1], [[, ]])),
+ m4_ifval([$2], [[$2]], [m4_dquote(lt_decl_tag_varnames)]),
+ m4_split(m4_normalize(m4_quote(_LT_TAGS)), [ ]))])
+m4_define([_lt_decl_varnames_tagged],
+[m4_ifval([$3], [lt_combine([$1], [$2], [_], $3)])])
+
+
+# lt_decl_all_varnames([SEPARATOR], [VARNAME1...])
+# ------------------------------------------------
+m4_define([lt_decl_all_varnames],
+[_$0(m4_quote(m4_default([$1], [[, ]])),
+ m4_if([$2], [],
+ m4_quote(lt_decl_varnames),
+ m4_quote(m4_shift($@))))[]dnl
+])
+m4_define([_lt_decl_all_varnames],
+[lt_join($@, lt_decl_varnames_tagged([$1],
+ lt_decl_tag_varnames([[, ]], m4_shift($@))))dnl
+])
+
+
+# _LT_CONFIG_STATUS_DECLARE([VARNAME])
+# ------------------------------------
+# Quote a variable value, and forward it to `config.status' so that its
+# declaration there will have the same value as in `configure'. VARNAME
+# must have a single quote delimited value for this to work.
+m4_define([_LT_CONFIG_STATUS_DECLARE],
+[$1='`$ECHO "X$][$1" | $Xsed -e "$delay_single_quote_subst"`'])
+
+
+# _LT_CONFIG_STATUS_DECLARATIONS
+# ------------------------------
+# We delimit libtool config variables with single quotes, so when
+# we write them to config.status, we have to be sure to quote all
+# embedded single quotes properly. In configure, this macro expands
+# each variable declared with _LT_DECL (and _LT_TAGDECL) into:
+#
+# <var>='`$ECHO "X$<var>" | $Xsed -e "$delay_single_quote_subst"`'
+m4_defun([_LT_CONFIG_STATUS_DECLARATIONS],
+[m4_foreach([_lt_var], m4_quote(lt_decl_all_varnames),
+ [m4_n([_LT_CONFIG_STATUS_DECLARE(_lt_var)])])])
+
+
+# _LT_LIBTOOL_TAGS
+# ----------------
+# Output comment and list of tags supported by the script
+m4_defun([_LT_LIBTOOL_TAGS],
+[_LT_FORMAT_COMMENT([The names of the tagged configurations supported by this script])dnl
+available_tags="_LT_TAGS"dnl
+])
+
+
+# _LT_LIBTOOL_DECLARE(VARNAME, [TAG])
+# -----------------------------------
+# Extract the dictionary values for VARNAME (optionally with TAG) and
+# expand to a commented shell variable setting:
+#
+# # Some comment about what VAR is for.
+# visible_name=$lt_internal_name
+m4_define([_LT_LIBTOOL_DECLARE],
+[_LT_FORMAT_COMMENT(m4_quote(lt_dict_fetch([lt_decl_dict], [$1],
+ [description])))[]dnl
+m4_pushdef([_libtool_name],
+ m4_quote(lt_dict_fetch([lt_decl_dict], [$1], [libtool_name])))[]dnl
+m4_case(m4_quote(lt_dict_fetch([lt_decl_dict], [$1], [value])),
+ [0], [_libtool_name=[$]$1],
+ [1], [_libtool_name=$lt_[]$1],
+ [2], [_libtool_name=$lt_[]$1],
+ [_libtool_name=lt_dict_fetch([lt_decl_dict], [$1], [value])])[]dnl
+m4_ifval([$2], [_$2])[]m4_popdef([_libtool_name])[]dnl
+])
+
+
+# _LT_LIBTOOL_CONFIG_VARS
+# -----------------------
+# Produce commented declarations of non-tagged libtool config variables
+# suitable for insertion in the LIBTOOL CONFIG section of the `libtool'
+# script. Tagged libtool config variables (even for the LIBTOOL CONFIG
+# section) are produced by _LT_LIBTOOL_TAG_VARS.
+m4_defun([_LT_LIBTOOL_CONFIG_VARS],
+[m4_foreach([_lt_var],
+ m4_quote(_lt_decl_filter([tagged?], [no], [], lt_decl_varnames)),
+ [m4_n([_LT_LIBTOOL_DECLARE(_lt_var)])])])
+
+
+# _LT_LIBTOOL_TAG_VARS(TAG)
+# -------------------------
+m4_define([_LT_LIBTOOL_TAG_VARS],
+[m4_foreach([_lt_var], m4_quote(lt_decl_tag_varnames),
+ [m4_n([_LT_LIBTOOL_DECLARE(_lt_var, [$1])])])])
+
+
+# _LT_TAGVAR(VARNAME, [TAGNAME])
+# ------------------------------
+m4_define([_LT_TAGVAR], [m4_ifval([$2], [$1_$2], [$1])])
+
+
+# _LT_CONFIG_COMMANDS
+# -------------------
+# Send accumulated output to $CONFIG_STATUS. Thanks to the lists of
+# variables for single and double quote escaping we saved from calls
+# to _LT_DECL, we can put quote escaped variables declarations
+# into `config.status', and then the shell code to quote escape them in
+# for loops in `config.status'. Finally, any additional code accumulated
+# from calls to _LT_CONFIG_LIBTOOL_INIT is expanded.
+m4_defun([_LT_CONFIG_COMMANDS],
+[AC_PROVIDE_IFELSE([LT_OUTPUT],
+ dnl If the libtool generation code has been placed in $CONFIG_LT,
+ dnl instead of duplicating it all over again into config.status,
+ dnl then we will have config.status run $CONFIG_LT later, so it
+ dnl needs to know what name is stored there:
+ [AC_CONFIG_COMMANDS([libtool],
+ [$SHELL $CONFIG_LT || AS_EXIT(1)], [CONFIG_LT='$CONFIG_LT'])],
+ dnl If the libtool generation code is destined for config.status,
+ dnl expand the accumulated commands and init code now:
+ [AC_CONFIG_COMMANDS([libtool],
+ [_LT_OUTPUT_LIBTOOL_COMMANDS], [_LT_OUTPUT_LIBTOOL_COMMANDS_INIT])])
+])#_LT_CONFIG_COMMANDS
+
+
+# Initialize.
+m4_define([_LT_OUTPUT_LIBTOOL_COMMANDS_INIT],
+[
+
+# The HP-UX ksh and POSIX shell print the target directory to stdout
+# if CDPATH is set.
+(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
+
+sed_quote_subst='$sed_quote_subst'
+double_quote_subst='$double_quote_subst'
+delay_variable_subst='$delay_variable_subst'
+_LT_CONFIG_STATUS_DECLARATIONS
+LTCC='$LTCC'
+LTCFLAGS='$LTCFLAGS'
+compiler='$compiler_DEFAULT'
+
+# Quote evaled strings.
+for var in lt_decl_all_varnames([[ \
+]], lt_decl_quote_varnames); do
+ case \`eval \\\\\$ECHO "X\\\\\$\$var"\` in
+ *[[\\\\\\\`\\"\\\$]]*)
+ eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"X\\\$\$var\\" | \\\$Xsed -e \\"\\\$sed_quote_subst\\"\\\`\\\\\\""
+ ;;
+ *)
+ eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\""
+ ;;
+ esac
+done
+
+# Double-quote double-evaled strings.
+for var in lt_decl_all_varnames([[ \
+]], lt_decl_dquote_varnames); do
+ case \`eval \\\\\$ECHO "X\\\\\$\$var"\` in
+ *[[\\\\\\\`\\"\\\$]]*)
+ eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"X\\\$\$var\\" | \\\$Xsed -e \\"\\\$double_quote_subst\\" -e \\"\\\$sed_quote_subst\\" -e \\"\\\$delay_variable_subst\\"\\\`\\\\\\""
+ ;;
+ *)
+ eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\""
+ ;;
+ esac
+done
+
+# Fix-up fallback echo if it was mangled by the above quoting rules.
+case \$lt_ECHO in
+*'\\\[$]0 --fallback-echo"')dnl "
+ lt_ECHO=\`\$ECHO "X\$lt_ECHO" | \$Xsed -e 's/\\\\\\\\\\\\\\\[$]0 --fallback-echo"\[$]/\[$]0 --fallback-echo"/'\`
+ ;;
+esac
+
+_LT_OUTPUT_LIBTOOL_INIT
+])
+
+
+# LT_OUTPUT
+# ---------
+# This macro allows early generation of the libtool script (before
+# AC_OUTPUT is called), incase it is used in configure for compilation
+# tests.
+AC_DEFUN([LT_OUTPUT],
+[: ${CONFIG_LT=./config.lt}
+AC_MSG_NOTICE([creating $CONFIG_LT])
+cat >"$CONFIG_LT" <<_LTEOF
+#! $SHELL
+# Generated by $as_me.
+# Run this file to recreate a libtool stub with the current configuration.
+
+lt_cl_silent=false
+SHELL=\${CONFIG_SHELL-$SHELL}
+_LTEOF
+
+cat >>"$CONFIG_LT" <<\_LTEOF
+AS_SHELL_SANITIZE
+_AS_PREPARE
+
+exec AS_MESSAGE_FD>&1
+exec AS_MESSAGE_LOG_FD>>config.log
+{
+ echo
+ AS_BOX([Running $as_me.])
+} >&AS_MESSAGE_LOG_FD
+
+lt_cl_help="\
+\`$as_me' creates a local libtool stub from the current configuration,
+for use in further configure time tests before the real libtool is
+generated.
+
+Usage: $[0] [[OPTIONS]]
+
+ -h, --help print this help, then exit
+ -V, --version print version number, then exit
+ -q, --quiet do not print progress messages
+ -d, --debug don't remove temporary files
+
+Report bugs to <bug-libtool at gnu.org>."
+
+lt_cl_version="\
+m4_ifset([AC_PACKAGE_NAME], [AC_PACKAGE_NAME ])config.lt[]dnl
+m4_ifset([AC_PACKAGE_VERSION], [ AC_PACKAGE_VERSION])
+configured by $[0], generated by m4_PACKAGE_STRING.
+
+Copyright (C) 2008 Free Software Foundation, Inc.
+This config.lt script is free software; the Free Software Foundation
+gives unlimited permision to copy, distribute and modify it."
+
+while test $[#] != 0
+do
+ case $[1] in
+ --version | --v* | -V )
+ echo "$lt_cl_version"; exit 0 ;;
+ --help | --h* | -h )
+ echo "$lt_cl_help"; exit 0 ;;
+ --debug | --d* | -d )
+ debug=: ;;
+ --quiet | --q* | --silent | --s* | -q )
+ lt_cl_silent=: ;;
+
+ -*) AC_MSG_ERROR([unrecognized option: $[1]
+Try \`$[0] --help' for more information.]) ;;
+
+ *) AC_MSG_ERROR([unrecognized argument: $[1]
+Try \`$[0] --help' for more information.]) ;;
+ esac
+ shift
+done
+
+if $lt_cl_silent; then
+ exec AS_MESSAGE_FD>/dev/null
+fi
+_LTEOF
+
+cat >>"$CONFIG_LT" <<_LTEOF
+_LT_OUTPUT_LIBTOOL_COMMANDS_INIT
+_LTEOF
+
+cat >>"$CONFIG_LT" <<\_LTEOF
+AC_MSG_NOTICE([creating $ofile])
+_LT_OUTPUT_LIBTOOL_COMMANDS
+AS_EXIT(0)
+_LTEOF
+chmod +x "$CONFIG_LT"
+
+# configure is writing to config.log, but config.lt does its own redirection,
+# appending to config.log, which fails on DOS, as config.log is still kept
+# open by configure. Here we exec the FD to /dev/null, effectively closing
+# config.log, so it can be properly (re)opened and appended to by config.lt.
+if test "$no_create" != yes; then
+ lt_cl_success=:
+ test "$silent" = yes &&
+ lt_config_lt_args="$lt_config_lt_args --quiet"
+ exec AS_MESSAGE_LOG_FD>/dev/null
+ $SHELL "$CONFIG_LT" $lt_config_lt_args || lt_cl_success=false
+ exec AS_MESSAGE_LOG_FD>>config.log
+ $lt_cl_success || AS_EXIT(1)
+fi
+])# LT_OUTPUT
+
+
+# _LT_CONFIG(TAG)
+# ---------------
+# If TAG is the built-in tag, create an initial libtool script with a
+# default configuration from the untagged config vars. Otherwise add code
+# to config.status for appending the configuration named by TAG from the
+# matching tagged config vars.
+m4_defun([_LT_CONFIG],
+[m4_require([_LT_FILEUTILS_DEFAULTS])dnl
+_LT_CONFIG_SAVE_COMMANDS([
+ m4_define([_LT_TAG], m4_if([$1], [], [C], [$1]))dnl
+ m4_if(_LT_TAG, [C], [
+ # See if we are running on zsh, and set the options which allow our
+ # commands through without removal of \ escapes.
+ if test -n "${ZSH_VERSION+set}" ; then
+ setopt NO_GLOB_SUBST
+ fi
+
+ cfgfile="${ofile}T"
+ trap "$RM \"$cfgfile\"; exit 1" 1 2 15
+ $RM "$cfgfile"
+
+ cat <<_LT_EOF >> "$cfgfile"
+#! $SHELL
+
+# `$ECHO "$ofile" | sed 's%^.*/%%'` - Provide generalized library-building support services.
+# Generated automatically by $as_me ($PACKAGE$TIMESTAMP) $VERSION
+# Libtool was configured on host `(hostname || uname -n) 2>/dev/null | sed 1q`:
+# NOTE: Changes made to this file will be lost: look at ltmain.sh.
+#
+_LT_COPYING
+_LT_LIBTOOL_TAGS
+
+# ### BEGIN LIBTOOL CONFIG
+_LT_LIBTOOL_CONFIG_VARS
+_LT_LIBTOOL_TAG_VARS
+# ### END LIBTOOL CONFIG
+
+_LT_EOF
+
+ case $host_os in
+ aix3*)
+ cat <<\_LT_EOF >> "$cfgfile"
+# AIX sometimes has problems with the GCC collect2 program. For some
+# reason, if we set the COLLECT_NAMES environment variable, the problems
+# vanish in a puff of smoke.
+if test "X${COLLECT_NAMES+set}" != Xset; then
+ COLLECT_NAMES=
+ export COLLECT_NAMES
+fi
+_LT_EOF
+ ;;
+ esac
+
+ _LT_PROG_LTMAIN
+
+ # We use sed instead of cat because bash on DJGPP gets confused if
+ # if finds mixed CR/LF and LF-only lines. Since sed operates in
+ # text mode, it properly converts lines to CR/LF. This bash problem
+ # is reportedly fixed, but why not run on old versions too?
+ sed '/^# Generated shell functions inserted here/q' "$ltmain" >> "$cfgfile" \
+ || (rm -f "$cfgfile"; exit 1)
+
+ _LT_PROG_XSI_SHELLFNS
+
+ sed -n '/^# Generated shell functions inserted here/,$p' "$ltmain" >> "$cfgfile" \
+ || (rm -f "$cfgfile"; exit 1)
+
+ mv -f "$cfgfile" "$ofile" ||
+ (rm -f "$ofile" && cp "$cfgfile" "$ofile" && rm -f "$cfgfile")
+ chmod +x "$ofile"
+],
+[cat <<_LT_EOF >> "$ofile"
+
+dnl Unfortunately we have to use $1 here, since _LT_TAG is not expanded
+dnl in a comment (ie after a #).
+# ### BEGIN LIBTOOL TAG CONFIG: $1
+_LT_LIBTOOL_TAG_VARS(_LT_TAG)
+# ### END LIBTOOL TAG CONFIG: $1
+_LT_EOF
+])dnl /m4_if
+],
+[m4_if([$1], [], [
+ PACKAGE='$PACKAGE'
+ VERSION='$VERSION'
+ TIMESTAMP='$TIMESTAMP'
+ RM='$RM'
+ ofile='$ofile'], [])
+])dnl /_LT_CONFIG_SAVE_COMMANDS
+])# _LT_CONFIG
+
+
+# LT_SUPPORTED_TAG(TAG)
+# ---------------------
+# Trace this macro to discover what tags are supported by the libtool
+# --tag option, using:
+# autoconf --trace 'LT_SUPPORTED_TAG:$1'
+AC_DEFUN([LT_SUPPORTED_TAG], [])
+
+
+# C support is built-in for now
+m4_define([_LT_LANG_C_enabled], [])
+m4_define([_LT_TAGS], [])
+
+
+# LT_LANG(LANG)
+# -------------
+# Enable libtool support for the given language if not already enabled.
+AC_DEFUN([LT_LANG],
+[AC_BEFORE([$0], [LT_OUTPUT])dnl
+m4_case([$1],
+ [C], [_LT_LANG(C)],
+ [C++], [_LT_LANG(CXX)],
+ [Java], [_LT_LANG(GCJ)],
+ [Fortran 77], [_LT_LANG(F77)],
+ [Fortran], [_LT_LANG(FC)],
+ [Windows Resource], [_LT_LANG(RC)],
+ [m4_ifdef([_LT_LANG_]$1[_CONFIG],
+ [_LT_LANG($1)],
+ [m4_fatal([$0: unsupported language: "$1"])])])dnl
+])# LT_LANG
+
+
+# _LT_LANG(LANGNAME)
+# ------------------
+m4_defun([_LT_LANG],
+[m4_ifdef([_LT_LANG_]$1[_enabled], [],
+ [LT_SUPPORTED_TAG([$1])dnl
+ m4_append([_LT_TAGS], [$1 ])dnl
+ m4_define([_LT_LANG_]$1[_enabled], [])dnl
+ _LT_LANG_$1_CONFIG($1)])dnl
+])# _LT_LANG
+
+
+# _LT_LANG_DEFAULT_CONFIG
+# -----------------------
+m4_defun([_LT_LANG_DEFAULT_CONFIG],
+[AC_PROVIDE_IFELSE([AC_PROG_CXX],
+ [LT_LANG(CXX)],
+ [m4_define([AC_PROG_CXX], defn([AC_PROG_CXX])[LT_LANG(CXX)])])
+
+AC_PROVIDE_IFELSE([AC_PROG_F77],
+ [LT_LANG(F77)],
+ [m4_define([AC_PROG_F77], defn([AC_PROG_F77])[LT_LANG(F77)])])
+
+AC_PROVIDE_IFELSE([AC_PROG_FC],
+ [LT_LANG(FC)],
+ [m4_define([AC_PROG_FC], defn([AC_PROG_FC])[LT_LANG(FC)])])
+
+dnl The call to [A][M_PROG_GCJ] is quoted like that to stop aclocal
+dnl pulling things in needlessly.
+AC_PROVIDE_IFELSE([AC_PROG_GCJ],
+ [LT_LANG(GCJ)],
+ [AC_PROVIDE_IFELSE([A][M_PROG_GCJ],
+ [LT_LANG(GCJ)],
+ [AC_PROVIDE_IFELSE([LT_PROG_GCJ],
+ [LT_LANG(GCJ)],
+ [m4_ifdef([AC_PROG_GCJ],
+ [m4_define([AC_PROG_GCJ], defn([AC_PROG_GCJ])[LT_LANG(GCJ)])])
+ m4_ifdef([A][M_PROG_GCJ],
+ [m4_define([A][M_PROG_GCJ], defn([A][M_PROG_GCJ])[LT_LANG(GCJ)])])
+ m4_ifdef([LT_PROG_GCJ],
+ [m4_define([LT_PROG_GCJ], defn([LT_PROG_GCJ])[LT_LANG(GCJ)])])])])])
+
+AC_PROVIDE_IFELSE([LT_PROG_RC],
+ [LT_LANG(RC)],
+ [m4_define([LT_PROG_RC], defn([LT_PROG_RC])[LT_LANG(RC)])])
+])# _LT_LANG_DEFAULT_CONFIG
+
+# Obsolete macros:
+AU_DEFUN([AC_LIBTOOL_CXX], [LT_LANG(C++)])
+AU_DEFUN([AC_LIBTOOL_F77], [LT_LANG(Fortran 77)])
+AU_DEFUN([AC_LIBTOOL_FC], [LT_LANG(Fortran)])
+AU_DEFUN([AC_LIBTOOL_GCJ], [LT_LANG(Java)])
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AC_LIBTOOL_CXX], [])
+dnl AC_DEFUN([AC_LIBTOOL_F77], [])
+dnl AC_DEFUN([AC_LIBTOOL_FC], [])
+dnl AC_DEFUN([AC_LIBTOOL_GCJ], [])
+
+
+# _LT_TAG_COMPILER
+# ----------------
+m4_defun([_LT_TAG_COMPILER],
+[AC_REQUIRE([AC_PROG_CC])dnl
+
+_LT_DECL([LTCC], [CC], [1], [A C compiler])dnl
+_LT_DECL([LTCFLAGS], [CFLAGS], [1], [LTCC compiler flags])dnl
+_LT_TAGDECL([CC], [compiler], [1], [A language specific compiler])dnl
+_LT_TAGDECL([with_gcc], [GCC], [0], [Is the compiler the GNU compiler?])dnl
+
+# If no C compiler was specified, use CC.
+LTCC=${LTCC-"$CC"}
+
+# If no C compiler flags were specified, use CFLAGS.
+LTCFLAGS=${LTCFLAGS-"$CFLAGS"}
+
+# Allow CC to be a program name with arguments.
+compiler=$CC
+])# _LT_TAG_COMPILER
+
+
+# _LT_COMPILER_BOILERPLATE
+# ------------------------
+# Check for compiler boilerplate output or warnings with
+# the simple compiler test code.
+m4_defun([_LT_COMPILER_BOILERPLATE],
+[m4_require([_LT_DECL_SED])dnl
+ac_outfile=conftest.$ac_objext
+echo "$lt_simple_compile_test_code" >conftest.$ac_ext
+eval "$ac_compile" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err
+_lt_compiler_boilerplate=`cat conftest.err`
+$RM conftest*
+])# _LT_COMPILER_BOILERPLATE
+
+
+# _LT_LINKER_BOILERPLATE
+# ----------------------
+# Check for linker boilerplate output or warnings with
+# the simple link test code.
+m4_defun([_LT_LINKER_BOILERPLATE],
+[m4_require([_LT_DECL_SED])dnl
+ac_outfile=conftest.$ac_objext
+echo "$lt_simple_link_test_code" >conftest.$ac_ext
+eval "$ac_link" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err
+_lt_linker_boilerplate=`cat conftest.err`
+$RM -r conftest*
+])# _LT_LINKER_BOILERPLATE
+
+# _LT_REQUIRED_DARWIN_CHECKS
+# -------------------------
+m4_defun_once([_LT_REQUIRED_DARWIN_CHECKS],[
+ case $host_os in
+ rhapsody* | darwin*)
+ AC_CHECK_TOOL([DSYMUTIL], [dsymutil], [:])
+ AC_CHECK_TOOL([NMEDIT], [nmedit], [:])
+ AC_CHECK_TOOL([LIPO], [lipo], [:])
+ AC_CHECK_TOOL([OTOOL], [otool], [:])
+ AC_CHECK_TOOL([OTOOL64], [otool64], [:])
+ _LT_DECL([], [DSYMUTIL], [1],
+ [Tool to manipulate archived DWARF debug symbol files on Mac OS X])
+ _LT_DECL([], [NMEDIT], [1],
+ [Tool to change global to local symbols on Mac OS X])
+ _LT_DECL([], [LIPO], [1],
+ [Tool to manipulate fat objects and archives on Mac OS X])
+ _LT_DECL([], [OTOOL], [1],
+ [ldd/readelf like tool for Mach-O binaries on Mac OS X])
+ _LT_DECL([], [OTOOL64], [1],
+ [ldd/readelf like tool for 64 bit Mach-O binaries on Mac OS X 10.4])
+
+ AC_CACHE_CHECK([for -single_module linker flag],[lt_cv_apple_cc_single_mod],
+ [lt_cv_apple_cc_single_mod=no
+ if test -z "${LT_MULTI_MODULE}"; then
+ # By default we will add the -single_module flag. You can override
+ # by either setting the environment variable LT_MULTI_MODULE
+ # non-empty at configure time, or by adding -multi_module to the
+ # link flags.
+ rm -rf libconftest.dylib*
+ echo "int foo(void){return 1;}" > conftest.c
+ echo "$LTCC $LTCFLAGS $LDFLAGS -o libconftest.dylib \
+-dynamiclib -Wl,-single_module conftest.c" >&AS_MESSAGE_LOG_FD
+ $LTCC $LTCFLAGS $LDFLAGS -o libconftest.dylib \
+ -dynamiclib -Wl,-single_module conftest.c 2>conftest.err
+ _lt_result=$?
+ if test -f libconftest.dylib && test ! -s conftest.err && test $_lt_result = 0; then
+ lt_cv_apple_cc_single_mod=yes
+ else
+ cat conftest.err >&AS_MESSAGE_LOG_FD
+ fi
+ rm -rf libconftest.dylib*
+ rm -f conftest.*
+ fi])
+ AC_CACHE_CHECK([for -exported_symbols_list linker flag],
+ [lt_cv_ld_exported_symbols_list],
+ [lt_cv_ld_exported_symbols_list=no
+ save_LDFLAGS=$LDFLAGS
+ echo "_main" > conftest.sym
+ LDFLAGS="$LDFLAGS -Wl,-exported_symbols_list,conftest.sym"
+ AC_LINK_IFELSE([AC_LANG_PROGRAM([],[])],
+ [lt_cv_ld_exported_symbols_list=yes],
+ [lt_cv_ld_exported_symbols_list=no])
+ LDFLAGS="$save_LDFLAGS"
+ ])
+ case $host_os in
+ rhapsody* | darwin1.[[012]])
+ _lt_dar_allow_undefined='${wl}-undefined ${wl}suppress' ;;
+ darwin1.*)
+ _lt_dar_allow_undefined='${wl}-flat_namespace ${wl}-undefined ${wl}suppress' ;;
+ darwin*) # darwin 5.x on
+ # if running on 10.5 or later, the deployment target defaults
+ # to the OS version, if on x86, and 10.4, the deployment
+ # target defaults to 10.4. Don't you love it?
+ case ${MACOSX_DEPLOYMENT_TARGET-10.0},$host in
+ 10.0,*86*-darwin8*|10.0,*-darwin[[91]]*)
+ _lt_dar_allow_undefined='${wl}-undefined ${wl}dynamic_lookup' ;;
+ 10.[[012]]*)
+ _lt_dar_allow_undefined='${wl}-flat_namespace ${wl}-undefined ${wl}suppress' ;;
+ 10.*)
+ _lt_dar_allow_undefined='${wl}-undefined ${wl}dynamic_lookup' ;;
+ esac
+ ;;
+ esac
+ if test "$lt_cv_apple_cc_single_mod" = "yes"; then
+ _lt_dar_single_mod='$single_module'
+ fi
+ if test "$lt_cv_ld_exported_symbols_list" = "yes"; then
+ _lt_dar_export_syms=' ${wl}-exported_symbols_list,$output_objdir/${libname}-symbols.expsym'
+ else
+ _lt_dar_export_syms='~$NMEDIT -s $output_objdir/${libname}-symbols.expsym ${lib}'
+ fi
+ if test "$DSYMUTIL" != ":"; then
+ _lt_dsymutil='~$DSYMUTIL $lib || :'
+ else
+ _lt_dsymutil=
+ fi
+ ;;
+ esac
+])
+
+
+# _LT_DARWIN_LINKER_FEATURES
+# --------------------------
+# Checks for linker and compiler features on darwin
+m4_defun([_LT_DARWIN_LINKER_FEATURES],
+[
+ m4_require([_LT_REQUIRED_DARWIN_CHECKS])
+ _LT_TAGVAR(archive_cmds_need_lc, $1)=no
+ _LT_TAGVAR(hardcode_direct, $1)=no
+ _LT_TAGVAR(hardcode_automatic, $1)=yes
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=unsupported
+ _LT_TAGVAR(whole_archive_flag_spec, $1)=''
+ _LT_TAGVAR(link_all_deplibs, $1)=yes
+ _LT_TAGVAR(allow_undefined_flag, $1)="$_lt_dar_allow_undefined"
+ case $cc_basename in
+ ifort*) _lt_dar_can_shared=yes ;;
+ *) _lt_dar_can_shared=$GCC ;;
+ esac
+ if test "$_lt_dar_can_shared" = "yes"; then
+ output_verbose_link_cmd=echo
+ _LT_TAGVAR(archive_cmds, $1)="\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod${_lt_dsymutil}"
+ _LT_TAGVAR(module_cmds, $1)="\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags${_lt_dsymutil}"
+ _LT_TAGVAR(archive_expsym_cmds, $1)="sed 's,^,_,' < \$export_symbols > \$output_objdir/\${libname}-symbols.expsym~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring ${_lt_dar_single_mod}${_lt_dar_export_syms}${_lt_dsymutil}"
+ _LT_TAGVAR(module_expsym_cmds, $1)="sed -e 's,^,_,' < \$export_symbols > \$output_objdir/\${libname}-symbols.expsym~\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags${_lt_dar_export_syms}${_lt_dsymutil}"
+ m4_if([$1], [CXX],
+[ if test "$lt_cv_apple_cc_single_mod" != "yes"; then
+ _LT_TAGVAR(archive_cmds, $1)="\$CC -r -keep_private_externs -nostdlib -o \${lib}-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \${lib}-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring${_lt_dsymutil}"
+ _LT_TAGVAR(archive_expsym_cmds, $1)="sed 's,^,_,' < \$export_symbols > \$output_objdir/\${libname}-symbols.expsym~\$CC -r -keep_private_externs -nostdlib -o \${lib}-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \${lib}-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring${_lt_dar_export_syms}${_lt_dsymutil}"
+ fi
+],[])
+ else
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+])
+
+# _LT_SYS_MODULE_PATH_AIX
+# -----------------------
+# Links a minimal program and checks the executable
+# for the system default hardcoded library path. In most cases,
+# this is /usr/lib:/lib, but when the MPI compilers are used
+# the location of the communication and MPI libs are included too.
+# If we don't find anything, use the default library path according
+# to the aix ld manual.
+m4_defun([_LT_SYS_MODULE_PATH_AIX],
+[m4_require([_LT_DECL_SED])dnl
+AC_LINK_IFELSE(AC_LANG_PROGRAM,[
+lt_aix_libpath_sed='
+ /Import File Strings/,/^$/ {
+ /^0/ {
+ s/^0 *\(.*\)$/\1/
+ p
+ }
+ }'
+aix_libpath=`dump -H conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"`
+# Check for a 64-bit object if we didn't find anything.
+if test -z "$aix_libpath"; then
+ aix_libpath=`dump -HX64 conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"`
+fi],[])
+if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi
+])# _LT_SYS_MODULE_PATH_AIX
+
+
+# _LT_SHELL_INIT(ARG)
+# -------------------
+m4_define([_LT_SHELL_INIT],
+[ifdef([AC_DIVERSION_NOTICE],
+ [AC_DIVERT_PUSH(AC_DIVERSION_NOTICE)],
+ [AC_DIVERT_PUSH(NOTICE)])
+$1
+AC_DIVERT_POP
+])# _LT_SHELL_INIT
+
+
+# _LT_PROG_ECHO_BACKSLASH
+# -----------------------
+# Add some code to the start of the generated configure script which
+# will find an echo command which doesn't interpret backslashes.
+m4_defun([_LT_PROG_ECHO_BACKSLASH],
+[_LT_SHELL_INIT([
+# Check that we are running under the correct shell.
+SHELL=${CONFIG_SHELL-/bin/sh}
+
+case X$lt_ECHO in
+X*--fallback-echo)
+ # Remove one level of quotation (which was required for Make).
+ ECHO=`echo "$lt_ECHO" | sed 's,\\\\\[$]\\[$]0,'[$]0','`
+ ;;
+esac
+
+ECHO=${lt_ECHO-echo}
+if test "X[$]1" = X--no-reexec; then
+ # Discard the --no-reexec flag, and continue.
+ shift
+elif test "X[$]1" = X--fallback-echo; then
+ # Avoid inline document here, it may be left over
+ :
+elif test "X`{ $ECHO '\t'; } 2>/dev/null`" = 'X\t' ; then
+ # Yippee, $ECHO works!
+ :
+else
+ # Restart under the correct shell.
+ exec $SHELL "[$]0" --no-reexec ${1+"[$]@"}
+fi
+
+if test "X[$]1" = X--fallback-echo; then
+ # used as fallback echo
+ shift
+ cat <<_LT_EOF
+[$]*
+_LT_EOF
+ exit 0
+fi
+
+# The HP-UX ksh and POSIX shell print the target directory to stdout
+# if CDPATH is set.
+(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
+
+if test -z "$lt_ECHO"; then
+ if test "X${echo_test_string+set}" != Xset; then
+ # find a string as large as possible, as long as the shell can cope with it
+ for cmd in 'sed 50q "[$]0"' 'sed 20q "[$]0"' 'sed 10q "[$]0"' 'sed 2q "[$]0"' 'echo test'; do
+ # expected sizes: less than 2Kb, 1Kb, 512 bytes, 16 bytes, ...
+ if { echo_test_string=`eval $cmd`; } 2>/dev/null &&
+ { test "X$echo_test_string" = "X$echo_test_string"; } 2>/dev/null
+ then
+ break
+ fi
+ done
+ fi
+
+ if test "X`{ $ECHO '\t'; } 2>/dev/null`" = 'X\t' &&
+ echo_testing_string=`{ $ECHO "$echo_test_string"; } 2>/dev/null` &&
+ test "X$echo_testing_string" = "X$echo_test_string"; then
+ :
+ else
+ # The Solaris, AIX, and Digital Unix default echo programs unquote
+ # backslashes. This makes it impossible to quote backslashes using
+ # echo "$something" | sed 's/\\/\\\\/g'
+ #
+ # So, first we look for a working echo in the user's PATH.
+
+ lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR
+ for dir in $PATH /usr/ucb; do
+ IFS="$lt_save_ifs"
+ if (test -f $dir/echo || test -f $dir/echo$ac_exeext) &&
+ test "X`($dir/echo '\t') 2>/dev/null`" = 'X\t' &&
+ echo_testing_string=`($dir/echo "$echo_test_string") 2>/dev/null` &&
+ test "X$echo_testing_string" = "X$echo_test_string"; then
+ ECHO="$dir/echo"
+ break
+ fi
+ done
+ IFS="$lt_save_ifs"
+
+ if test "X$ECHO" = Xecho; then
+ # We didn't find a better echo, so look for alternatives.
+ if test "X`{ print -r '\t'; } 2>/dev/null`" = 'X\t' &&
+ echo_testing_string=`{ print -r "$echo_test_string"; } 2>/dev/null` &&
+ test "X$echo_testing_string" = "X$echo_test_string"; then
+ # This shell has a builtin print -r that does the trick.
+ ECHO='print -r'
+ elif { test -f /bin/ksh || test -f /bin/ksh$ac_exeext; } &&
+ test "X$CONFIG_SHELL" != X/bin/ksh; then
+ # If we have ksh, try running configure again with it.
+ ORIGINAL_CONFIG_SHELL=${CONFIG_SHELL-/bin/sh}
+ export ORIGINAL_CONFIG_SHELL
+ CONFIG_SHELL=/bin/ksh
+ export CONFIG_SHELL
+ exec $CONFIG_SHELL "[$]0" --no-reexec ${1+"[$]@"}
+ else
+ # Try using printf.
+ ECHO='printf %s\n'
+ if test "X`{ $ECHO '\t'; } 2>/dev/null`" = 'X\t' &&
+ echo_testing_string=`{ $ECHO "$echo_test_string"; } 2>/dev/null` &&
+ test "X$echo_testing_string" = "X$echo_test_string"; then
+ # Cool, printf works
+ :
+ elif echo_testing_string=`($ORIGINAL_CONFIG_SHELL "[$]0" --fallback-echo '\t') 2>/dev/null` &&
+ test "X$echo_testing_string" = 'X\t' &&
+ echo_testing_string=`($ORIGINAL_CONFIG_SHELL "[$]0" --fallback-echo "$echo_test_string") 2>/dev/null` &&
+ test "X$echo_testing_string" = "X$echo_test_string"; then
+ CONFIG_SHELL=$ORIGINAL_CONFIG_SHELL
+ export CONFIG_SHELL
+ SHELL="$CONFIG_SHELL"
+ export SHELL
+ ECHO="$CONFIG_SHELL [$]0 --fallback-echo"
+ elif echo_testing_string=`($CONFIG_SHELL "[$]0" --fallback-echo '\t') 2>/dev/null` &&
+ test "X$echo_testing_string" = 'X\t' &&
+ echo_testing_string=`($CONFIG_SHELL "[$]0" --fallback-echo "$echo_test_string") 2>/dev/null` &&
+ test "X$echo_testing_string" = "X$echo_test_string"; then
+ ECHO="$CONFIG_SHELL [$]0 --fallback-echo"
+ else
+ # maybe with a smaller string...
+ prev=:
+
+ for cmd in 'echo test' 'sed 2q "[$]0"' 'sed 10q "[$]0"' 'sed 20q "[$]0"' 'sed 50q "[$]0"'; do
+ if { test "X$echo_test_string" = "X`eval $cmd`"; } 2>/dev/null
+ then
+ break
+ fi
+ prev="$cmd"
+ done
+
+ if test "$prev" != 'sed 50q "[$]0"'; then
+ echo_test_string=`eval $prev`
+ export echo_test_string
+ exec ${ORIGINAL_CONFIG_SHELL-${CONFIG_SHELL-/bin/sh}} "[$]0" ${1+"[$]@"}
+ else
+ # Oops. We lost completely, so just stick with echo.
+ ECHO=echo
+ fi
+ fi
+ fi
+ fi
+ fi
+fi
+
+# Copy echo and quote the copy suitably for passing to libtool from
+# the Makefile, instead of quoting the original, which is used later.
+lt_ECHO=$ECHO
+if test "X$lt_ECHO" = "X$CONFIG_SHELL [$]0 --fallback-echo"; then
+ lt_ECHO="$CONFIG_SHELL \\\$\[$]0 --fallback-echo"
+fi
+
+AC_SUBST(lt_ECHO)
+])
+_LT_DECL([], [SHELL], [1], [Shell to use when invoking shell scripts])
+_LT_DECL([], [ECHO], [1],
+ [An echo program that does not interpret backslashes])
+])# _LT_PROG_ECHO_BACKSLASH
+
+
+# _LT_ENABLE_LOCK
+# ---------------
+m4_defun([_LT_ENABLE_LOCK],
+[AC_ARG_ENABLE([libtool-lock],
+ [AS_HELP_STRING([--disable-libtool-lock],
+ [avoid locking (might break parallel builds)])])
+test "x$enable_libtool_lock" != xno && enable_libtool_lock=yes
+
+# Some flags need to be propagated to the compiler or linker for good
+# libtool support.
+case $host in
+ia64-*-hpux*)
+ # Find out which ABI we are using.
+ echo 'int i;' > conftest.$ac_ext
+ if AC_TRY_EVAL(ac_compile); then
+ case `/usr/bin/file conftest.$ac_objext` in
+ *ELF-32*)
+ HPUX_IA64_MODE="32"
+ ;;
+ *ELF-64*)
+ HPUX_IA64_MODE="64"
+ ;;
+ esac
+ fi
+ rm -rf conftest*
+ ;;
+*-*-irix6*)
+ # Find out which ABI we are using.
+ echo '[#]line __oline__ "configure"' > conftest.$ac_ext
+ if AC_TRY_EVAL(ac_compile); then
+ if test "$lt_cv_prog_gnu_ld" = yes; then
+ case `/usr/bin/file conftest.$ac_objext` in
+ *32-bit*)
+ LD="${LD-ld} -melf32bsmip"
+ ;;
+ *N32*)
+ LD="${LD-ld} -melf32bmipn32"
+ ;;
+ *64-bit*)
+ LD="${LD-ld} -melf64bmip"
+ ;;
+ esac
+ else
+ case `/usr/bin/file conftest.$ac_objext` in
+ *32-bit*)
+ LD="${LD-ld} -32"
+ ;;
+ *N32*)
+ LD="${LD-ld} -n32"
+ ;;
+ *64-bit*)
+ LD="${LD-ld} -64"
+ ;;
+ esac
+ fi
+ fi
+ rm -rf conftest*
+ ;;
+
+x86_64-*kfreebsd*-gnu|x86_64-*linux*|ppc*-*linux*|powerpc*-*linux*| \
+s390*-*linux*|s390*-*tpf*|sparc*-*linux*)
+ # Find out which ABI we are using.
+ echo 'int i;' > conftest.$ac_ext
+ if AC_TRY_EVAL(ac_compile); then
+ case `/usr/bin/file conftest.o` in
+ *32-bit*)
+ case $host in
+ x86_64-*kfreebsd*-gnu)
+ LD="${LD-ld} -m elf_i386_fbsd"
+ ;;
+ x86_64-*linux*)
+ LD="${LD-ld} -m elf_i386"
+ ;;
+ ppc64-*linux*|powerpc64-*linux*)
+ LD="${LD-ld} -m elf32ppclinux"
+ ;;
+ s390x-*linux*)
+ LD="${LD-ld} -m elf_s390"
+ ;;
+ sparc64-*linux*)
+ LD="${LD-ld} -m elf32_sparc"
+ ;;
+ esac
+ ;;
+ *64-bit*)
+ case $host in
+ x86_64-*kfreebsd*-gnu)
+ LD="${LD-ld} -m elf_x86_64_fbsd"
+ ;;
+ x86_64-*linux*)
+ LD="${LD-ld} -m elf_x86_64"
+ ;;
+ ppc*-*linux*|powerpc*-*linux*)
+ LD="${LD-ld} -m elf64ppc"
+ ;;
+ s390*-*linux*|s390*-*tpf*)
+ LD="${LD-ld} -m elf64_s390"
+ ;;
+ sparc*-*linux*)
+ LD="${LD-ld} -m elf64_sparc"
+ ;;
+ esac
+ ;;
+ esac
+ fi
+ rm -rf conftest*
+ ;;
+
+*-*-sco3.2v5*)
+ # On SCO OpenServer 5, we need -belf to get full-featured binaries.
+ SAVE_CFLAGS="$CFLAGS"
+ CFLAGS="$CFLAGS -belf"
+ AC_CACHE_CHECK([whether the C compiler needs -belf], lt_cv_cc_needs_belf,
+ [AC_LANG_PUSH(C)
+ AC_LINK_IFELSE([AC_LANG_PROGRAM([[]],[[]])],[lt_cv_cc_needs_belf=yes],[lt_cv_cc_needs_belf=no])
+ AC_LANG_POP])
+ if test x"$lt_cv_cc_needs_belf" != x"yes"; then
+ # this is probably gcc 2.8.0, egcs 1.0 or newer; no need for -belf
+ CFLAGS="$SAVE_CFLAGS"
+ fi
+ ;;
+sparc*-*solaris*)
+ # Find out which ABI we are using.
+ echo 'int i;' > conftest.$ac_ext
+ if AC_TRY_EVAL(ac_compile); then
+ case `/usr/bin/file conftest.o` in
+ *64-bit*)
+ case $lt_cv_prog_gnu_ld in
+ yes*) LD="${LD-ld} -m elf64_sparc" ;;
+ *)
+ if ${LD-ld} -64 -r -o conftest2.o conftest.o >/dev/null 2>&1; then
+ LD="${LD-ld} -64"
+ fi
+ ;;
+ esac
+ ;;
+ esac
+ fi
+ rm -rf conftest*
+ ;;
+esac
+
+need_locks="$enable_libtool_lock"
+])# _LT_ENABLE_LOCK
+
+
+# _LT_CMD_OLD_ARCHIVE
+# -------------------
+m4_defun([_LT_CMD_OLD_ARCHIVE],
+[AC_CHECK_TOOL(AR, ar, false)
+test -z "$AR" && AR=ar
+test -z "$AR_FLAGS" && AR_FLAGS=cru
+_LT_DECL([], [AR], [1], [The archiver])
+_LT_DECL([], [AR_FLAGS], [1])
+
+AC_CHECK_TOOL(STRIP, strip, :)
+test -z "$STRIP" && STRIP=:
+_LT_DECL([], [STRIP], [1], [A symbol stripping program])
+
+AC_CHECK_TOOL(RANLIB, ranlib, :)
+test -z "$RANLIB" && RANLIB=:
+_LT_DECL([], [RANLIB], [1],
+ [Commands used to install an old-style archive])
+
+# Determine commands to create old-style static archives.
+old_archive_cmds='$AR $AR_FLAGS $oldlib$oldobjs'
+old_postinstall_cmds='chmod 644 $oldlib'
+old_postuninstall_cmds=
+
+if test -n "$RANLIB"; then
+ case $host_os in
+ openbsd*)
+ old_postinstall_cmds="$old_postinstall_cmds~\$RANLIB -t \$oldlib"
+ ;;
+ *)
+ old_postinstall_cmds="$old_postinstall_cmds~\$RANLIB \$oldlib"
+ ;;
+ esac
+ old_archive_cmds="$old_archive_cmds~\$RANLIB \$oldlib"
+fi
+_LT_DECL([], [old_postinstall_cmds], [2])
+_LT_DECL([], [old_postuninstall_cmds], [2])
+_LT_TAGDECL([], [old_archive_cmds], [2],
+ [Commands used to build an old-style archive])
+])# _LT_CMD_OLD_ARCHIVE
+
+
+# _LT_COMPILER_OPTION(MESSAGE, VARIABLE-NAME, FLAGS,
+# [OUTPUT-FILE], [ACTION-SUCCESS], [ACTION-FAILURE])
+# ----------------------------------------------------------------
+# Check whether the given compiler option works
+AC_DEFUN([_LT_COMPILER_OPTION],
+[m4_require([_LT_FILEUTILS_DEFAULTS])dnl
+m4_require([_LT_DECL_SED])dnl
+AC_CACHE_CHECK([$1], [$2],
+ [$2=no
+ m4_if([$4], , [ac_outfile=conftest.$ac_objext], [ac_outfile=$4])
+ echo "$lt_simple_compile_test_code" > conftest.$ac_ext
+ lt_compiler_flag="$3"
+ # Insert the option either (1) after the last *FLAGS variable, or
+ # (2) before a word containing "conftest.", or (3) at the end.
+ # Note that $ac_compile itself does not contain backslashes and begins
+ # with a dollar sign (not a hyphen), so the echo should work correctly.
+ # The option is referenced via a variable to avoid confusing sed.
+ lt_compile=`echo "$ac_compile" | $SED \
+ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
+ -e 's: [[^ ]]*conftest\.: $lt_compiler_flag&:; t' \
+ -e 's:$: $lt_compiler_flag:'`
+ (eval echo "\"\$as_me:__oline__: $lt_compile\"" >&AS_MESSAGE_LOG_FD)
+ (eval "$lt_compile" 2>conftest.err)
+ ac_status=$?
+ cat conftest.err >&AS_MESSAGE_LOG_FD
+ echo "$as_me:__oline__: \$? = $ac_status" >&AS_MESSAGE_LOG_FD
+ if (exit $ac_status) && test -s "$ac_outfile"; then
+ # The compiler can only warn and ignore the option if not recognized
+ # So say no if there are warnings other than the usual output.
+ $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp
+ $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2
+ if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then
+ $2=yes
+ fi
+ fi
+ $RM conftest*
+])
+
+if test x"[$]$2" = xyes; then
+ m4_if([$5], , :, [$5])
+else
+ m4_if([$6], , :, [$6])
+fi
+])# _LT_COMPILER_OPTION
+
+# Old name:
+AU_ALIAS([AC_LIBTOOL_COMPILER_OPTION], [_LT_COMPILER_OPTION])
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AC_LIBTOOL_COMPILER_OPTION], [])
+
+
+# _LT_LINKER_OPTION(MESSAGE, VARIABLE-NAME, FLAGS,
+# [ACTION-SUCCESS], [ACTION-FAILURE])
+# ----------------------------------------------------
+# Check whether the given linker option works
+AC_DEFUN([_LT_LINKER_OPTION],
+[m4_require([_LT_FILEUTILS_DEFAULTS])dnl
+m4_require([_LT_DECL_SED])dnl
+AC_CACHE_CHECK([$1], [$2],
+ [$2=no
+ save_LDFLAGS="$LDFLAGS"
+ LDFLAGS="$LDFLAGS $3"
+ echo "$lt_simple_link_test_code" > conftest.$ac_ext
+ if (eval $ac_link 2>conftest.err) && test -s conftest$ac_exeext; then
+ # The linker can only warn and ignore the option if not recognized
+ # So say no if there are warnings
+ if test -s conftest.err; then
+ # Append any errors to the config.log.
+ cat conftest.err 1>&AS_MESSAGE_LOG_FD
+ $ECHO "X$_lt_linker_boilerplate" | $Xsed -e '/^$/d' > conftest.exp
+ $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2
+ if diff conftest.exp conftest.er2 >/dev/null; then
+ $2=yes
+ fi
+ else
+ $2=yes
+ fi
+ fi
+ $RM -r conftest*
+ LDFLAGS="$save_LDFLAGS"
+])
+
+if test x"[$]$2" = xyes; then
+ m4_if([$4], , :, [$4])
+else
+ m4_if([$5], , :, [$5])
+fi
+])# _LT_LINKER_OPTION
+
+# Old name:
+AU_ALIAS([AC_LIBTOOL_LINKER_OPTION], [_LT_LINKER_OPTION])
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AC_LIBTOOL_LINKER_OPTION], [])
+
+
+# LT_CMD_MAX_LEN
+#---------------
+AC_DEFUN([LT_CMD_MAX_LEN],
+[AC_REQUIRE([AC_CANONICAL_HOST])dnl
+# find the maximum length of command line arguments
+AC_MSG_CHECKING([the maximum length of command line arguments])
+AC_CACHE_VAL([lt_cv_sys_max_cmd_len], [dnl
+ i=0
+ teststring="ABCD"
+
+ case $build_os in
+ msdosdjgpp*)
+ # On DJGPP, this test can blow up pretty badly due to problems in libc
+ # (any single argument exceeding 2000 bytes causes a buffer overrun
+ # during glob expansion). Even if it were fixed, the result of this
+ # check would be larger than it should be.
+ lt_cv_sys_max_cmd_len=12288; # 12K is about right
+ ;;
+
+ gnu*)
+ # Under GNU Hurd, this test is not required because there is
+ # no limit to the length of command line arguments.
+ # Libtool will interpret -1 as no limit whatsoever
+ lt_cv_sys_max_cmd_len=-1;
+ ;;
+
+ cygwin* | mingw* | cegcc*)
+ # On Win9x/ME, this test blows up -- it succeeds, but takes
+ # about 5 minutes as the teststring grows exponentially.
+ # Worse, since 9x/ME are not pre-emptively multitasking,
+ # you end up with a "frozen" computer, even though with patience
+ # the test eventually succeeds (with a max line length of 256k).
+ # Instead, let's just punt: use the minimum linelength reported by
+ # all of the supported platforms: 8192 (on NT/2K/XP).
+ lt_cv_sys_max_cmd_len=8192;
+ ;;
+
+ amigaos*)
+ # On AmigaOS with pdksh, this test takes hours, literally.
+ # So we just punt and use a minimum line length of 8192.
+ lt_cv_sys_max_cmd_len=8192;
+ ;;
+
+ netbsd* | freebsd* | openbsd* | darwin* | dragonfly*)
+ # This has been around since 386BSD, at least. Likely further.
+ if test -x /sbin/sysctl; then
+ lt_cv_sys_max_cmd_len=`/sbin/sysctl -n kern.argmax`
+ elif test -x /usr/sbin/sysctl; then
+ lt_cv_sys_max_cmd_len=`/usr/sbin/sysctl -n kern.argmax`
+ else
+ lt_cv_sys_max_cmd_len=65536 # usable default for all BSDs
+ fi
+ # And add a safety zone
+ lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 4`
+ lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \* 3`
+ ;;
+
+ interix*)
+ # We know the value 262144 and hardcode it with a safety zone (like BSD)
+ lt_cv_sys_max_cmd_len=196608
+ ;;
+
+ osf*)
+ # Dr. Hans Ekkehard Plesser reports seeing a kernel panic running configure
+ # due to this test when exec_disable_arg_limit is 1 on Tru64. It is not
+ # nice to cause kernel panics so lets avoid the loop below.
+ # First set a reasonable default.
+ lt_cv_sys_max_cmd_len=16384
+ #
+ if test -x /sbin/sysconfig; then
+ case `/sbin/sysconfig -q proc exec_disable_arg_limit` in
+ *1*) lt_cv_sys_max_cmd_len=-1 ;;
+ esac
+ fi
+ ;;
+ sco3.2v5*)
+ lt_cv_sys_max_cmd_len=102400
+ ;;
+ sysv5* | sco5v6* | sysv4.2uw2*)
+ kargmax=`grep ARG_MAX /etc/conf/cf.d/stune 2>/dev/null`
+ if test -n "$kargmax"; then
+ lt_cv_sys_max_cmd_len=`echo $kargmax | sed 's/.*[[ ]]//'`
+ else
+ lt_cv_sys_max_cmd_len=32768
+ fi
+ ;;
+ *)
+ lt_cv_sys_max_cmd_len=`(getconf ARG_MAX) 2> /dev/null`
+ if test -n "$lt_cv_sys_max_cmd_len"; then
+ lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 4`
+ lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \* 3`
+ else
+ # Make teststring a little bigger before we do anything with it.
+ # a 1K string should be a reasonable start.
+ for i in 1 2 3 4 5 6 7 8 ; do
+ teststring=$teststring$teststring
+ done
+ SHELL=${SHELL-${CONFIG_SHELL-/bin/sh}}
+ # If test is not a shell built-in, we'll probably end up computing a
+ # maximum length that is only half of the actual maximum length, but
+ # we can't tell.
+ while { test "X"`$SHELL [$]0 --fallback-echo "X$teststring$teststring" 2>/dev/null` \
+ = "XX$teststring$teststring"; } >/dev/null 2>&1 &&
+ test $i != 17 # 1/2 MB should be enough
+ do
+ i=`expr $i + 1`
+ teststring=$teststring$teststring
+ done
+ # Only check the string length outside the loop.
+ lt_cv_sys_max_cmd_len=`expr "X$teststring" : ".*" 2>&1`
+ teststring=
+ # Add a significant safety factor because C++ compilers can tack on
+ # massive amounts of additional arguments before passing them to the
+ # linker. It appears as though 1/2 is a usable value.
+ lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 2`
+ fi
+ ;;
+ esac
+])
+if test -n $lt_cv_sys_max_cmd_len ; then
+ AC_MSG_RESULT($lt_cv_sys_max_cmd_len)
+else
+ AC_MSG_RESULT(none)
+fi
+max_cmd_len=$lt_cv_sys_max_cmd_len
+_LT_DECL([], [max_cmd_len], [0],
+ [What is the maximum length of a command?])
+])# LT_CMD_MAX_LEN
+
+# Old name:
+AU_ALIAS([AC_LIBTOOL_SYS_MAX_CMD_LEN], [LT_CMD_MAX_LEN])
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AC_LIBTOOL_SYS_MAX_CMD_LEN], [])
+
+
+# _LT_HEADER_DLFCN
+# ----------------
+m4_defun([_LT_HEADER_DLFCN],
+[AC_CHECK_HEADERS([dlfcn.h], [], [], [AC_INCLUDES_DEFAULT])dnl
+])# _LT_HEADER_DLFCN
+
+
+# _LT_TRY_DLOPEN_SELF (ACTION-IF-TRUE, ACTION-IF-TRUE-W-USCORE,
+# ACTION-IF-FALSE, ACTION-IF-CROSS-COMPILING)
+# ----------------------------------------------------------------
+m4_defun([_LT_TRY_DLOPEN_SELF],
+[m4_require([_LT_HEADER_DLFCN])dnl
+if test "$cross_compiling" = yes; then :
+ [$4]
+else
+ lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
+ lt_status=$lt_dlunknown
+ cat > conftest.$ac_ext <<_LT_EOF
+[#line __oline__ "configure"
+#include "confdefs.h"
+
+#if HAVE_DLFCN_H
+#include <dlfcn.h>
+#endif
+
+#include <stdio.h>
+
+#ifdef RTLD_GLOBAL
+# define LT_DLGLOBAL RTLD_GLOBAL
+#else
+# ifdef DL_GLOBAL
+# define LT_DLGLOBAL DL_GLOBAL
+# else
+# define LT_DLGLOBAL 0
+# endif
+#endif
+
+/* We may have to define LT_DLLAZY_OR_NOW in the command line if we
+ find out it does not work in some platform. */
+#ifndef LT_DLLAZY_OR_NOW
+# ifdef RTLD_LAZY
+# define LT_DLLAZY_OR_NOW RTLD_LAZY
+# else
+# ifdef DL_LAZY
+# define LT_DLLAZY_OR_NOW DL_LAZY
+# else
+# ifdef RTLD_NOW
+# define LT_DLLAZY_OR_NOW RTLD_NOW
+# else
+# ifdef DL_NOW
+# define LT_DLLAZY_OR_NOW DL_NOW
+# else
+# define LT_DLLAZY_OR_NOW 0
+# endif
+# endif
+# endif
+# endif
+#endif
+
+void fnord() { int i=42;}
+int main ()
+{
+ void *self = dlopen (0, LT_DLGLOBAL|LT_DLLAZY_OR_NOW);
+ int status = $lt_dlunknown;
+
+ if (self)
+ {
+ if (dlsym (self,"fnord")) status = $lt_dlno_uscore;
+ else if (dlsym( self,"_fnord")) status = $lt_dlneed_uscore;
+ /* dlclose (self); */
+ }
+ else
+ puts (dlerror ());
+
+ return status;
+}]
+_LT_EOF
+ if AC_TRY_EVAL(ac_link) && test -s conftest${ac_exeext} 2>/dev/null; then
+ (./conftest; exit; ) >&AS_MESSAGE_LOG_FD 2>/dev/null
+ lt_status=$?
+ case x$lt_status in
+ x$lt_dlno_uscore) $1 ;;
+ x$lt_dlneed_uscore) $2 ;;
+ x$lt_dlunknown|x*) $3 ;;
+ esac
+ else :
+ # compilation failed
+ $3
+ fi
+fi
+rm -fr conftest*
+])# _LT_TRY_DLOPEN_SELF
+
+
+# LT_SYS_DLOPEN_SELF
+# ------------------
+AC_DEFUN([LT_SYS_DLOPEN_SELF],
+[m4_require([_LT_HEADER_DLFCN])dnl
+if test "x$enable_dlopen" != xyes; then
+ enable_dlopen=unknown
+ enable_dlopen_self=unknown
+ enable_dlopen_self_static=unknown
+else
+ lt_cv_dlopen=no
+ lt_cv_dlopen_libs=
+
+ case $host_os in
+ beos*)
+ lt_cv_dlopen="load_add_on"
+ lt_cv_dlopen_libs=
+ lt_cv_dlopen_self=yes
+ ;;
+
+ mingw* | pw32* | cegcc*)
+ lt_cv_dlopen="LoadLibrary"
+ lt_cv_dlopen_libs=
+ ;;
+
+ cygwin*)
+ lt_cv_dlopen="dlopen"
+ lt_cv_dlopen_libs=
+ ;;
+
+ darwin*)
+ # if libdl is installed we need to link against it
+ AC_CHECK_LIB([dl], [dlopen],
+ [lt_cv_dlopen="dlopen" lt_cv_dlopen_libs="-ldl"],[
+ lt_cv_dlopen="dyld"
+ lt_cv_dlopen_libs=
+ lt_cv_dlopen_self=yes
+ ])
+ ;;
+
+ *)
+ AC_CHECK_FUNC([shl_load],
+ [lt_cv_dlopen="shl_load"],
+ [AC_CHECK_LIB([dld], [shl_load],
+ [lt_cv_dlopen="shl_load" lt_cv_dlopen_libs="-ldld"],
+ [AC_CHECK_FUNC([dlopen],
+ [lt_cv_dlopen="dlopen"],
+ [AC_CHECK_LIB([dl], [dlopen],
+ [lt_cv_dlopen="dlopen" lt_cv_dlopen_libs="-ldl"],
+ [AC_CHECK_LIB([svld], [dlopen],
+ [lt_cv_dlopen="dlopen" lt_cv_dlopen_libs="-lsvld"],
+ [AC_CHECK_LIB([dld], [dld_link],
+ [lt_cv_dlopen="dld_link" lt_cv_dlopen_libs="-ldld"])
+ ])
+ ])
+ ])
+ ])
+ ])
+ ;;
+ esac
+
+ if test "x$lt_cv_dlopen" != xno; then
+ enable_dlopen=yes
+ else
+ enable_dlopen=no
+ fi
+
+ case $lt_cv_dlopen in
+ dlopen)
+ save_CPPFLAGS="$CPPFLAGS"
+ test "x$ac_cv_header_dlfcn_h" = xyes && CPPFLAGS="$CPPFLAGS -DHAVE_DLFCN_H"
+
+ save_LDFLAGS="$LDFLAGS"
+ wl=$lt_prog_compiler_wl eval LDFLAGS=\"\$LDFLAGS $export_dynamic_flag_spec\"
+
+ save_LIBS="$LIBS"
+ LIBS="$lt_cv_dlopen_libs $LIBS"
+
+ AC_CACHE_CHECK([whether a program can dlopen itself],
+ lt_cv_dlopen_self, [dnl
+ _LT_TRY_DLOPEN_SELF(
+ lt_cv_dlopen_self=yes, lt_cv_dlopen_self=yes,
+ lt_cv_dlopen_self=no, lt_cv_dlopen_self=cross)
+ ])
+
+ if test "x$lt_cv_dlopen_self" = xyes; then
+ wl=$lt_prog_compiler_wl eval LDFLAGS=\"\$LDFLAGS $lt_prog_compiler_static\"
+ AC_CACHE_CHECK([whether a statically linked program can dlopen itself],
+ lt_cv_dlopen_self_static, [dnl
+ _LT_TRY_DLOPEN_SELF(
+ lt_cv_dlopen_self_static=yes, lt_cv_dlopen_self_static=yes,
+ lt_cv_dlopen_self_static=no, lt_cv_dlopen_self_static=cross)
+ ])
+ fi
+
+ CPPFLAGS="$save_CPPFLAGS"
+ LDFLAGS="$save_LDFLAGS"
+ LIBS="$save_LIBS"
+ ;;
+ esac
+
+ case $lt_cv_dlopen_self in
+ yes|no) enable_dlopen_self=$lt_cv_dlopen_self ;;
+ *) enable_dlopen_self=unknown ;;
+ esac
+
+ case $lt_cv_dlopen_self_static in
+ yes|no) enable_dlopen_self_static=$lt_cv_dlopen_self_static ;;
+ *) enable_dlopen_self_static=unknown ;;
+ esac
+fi
+_LT_DECL([dlopen_support], [enable_dlopen], [0],
+ [Whether dlopen is supported])
+_LT_DECL([dlopen_self], [enable_dlopen_self], [0],
+ [Whether dlopen of programs is supported])
+_LT_DECL([dlopen_self_static], [enable_dlopen_self_static], [0],
+ [Whether dlopen of statically linked programs is supported])
+])# LT_SYS_DLOPEN_SELF
+
+# Old name:
+AU_ALIAS([AC_LIBTOOL_DLOPEN_SELF], [LT_SYS_DLOPEN_SELF])
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AC_LIBTOOL_DLOPEN_SELF], [])
+
+
+# _LT_COMPILER_C_O([TAGNAME])
+# ---------------------------
+# Check to see if options -c and -o are simultaneously supported by compiler.
+# This macro does not hard code the compiler like AC_PROG_CC_C_O.
+m4_defun([_LT_COMPILER_C_O],
+[m4_require([_LT_DECL_SED])dnl
+m4_require([_LT_FILEUTILS_DEFAULTS])dnl
+m4_require([_LT_TAG_COMPILER])dnl
+AC_CACHE_CHECK([if $compiler supports -c -o file.$ac_objext],
+ [_LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)],
+ [_LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)=no
+ $RM -r conftest 2>/dev/null
+ mkdir conftest
+ cd conftest
+ mkdir out
+ echo "$lt_simple_compile_test_code" > conftest.$ac_ext
+
+ lt_compiler_flag="-o out/conftest2.$ac_objext"
+ # Insert the option either (1) after the last *FLAGS variable, or
+ # (2) before a word containing "conftest.", or (3) at the end.
+ # Note that $ac_compile itself does not contain backslashes and begins
+ # with a dollar sign (not a hyphen), so the echo should work correctly.
+ lt_compile=`echo "$ac_compile" | $SED \
+ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
+ -e 's: [[^ ]]*conftest\.: $lt_compiler_flag&:; t' \
+ -e 's:$: $lt_compiler_flag:'`
+ (eval echo "\"\$as_me:__oline__: $lt_compile\"" >&AS_MESSAGE_LOG_FD)
+ (eval "$lt_compile" 2>out/conftest.err)
+ ac_status=$?
+ cat out/conftest.err >&AS_MESSAGE_LOG_FD
+ echo "$as_me:__oline__: \$? = $ac_status" >&AS_MESSAGE_LOG_FD
+ if (exit $ac_status) && test -s out/conftest2.$ac_objext
+ then
+ # The compiler can only warn and ignore the option if not recognized
+ # So say no if there are warnings
+ $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp
+ $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2
+ if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then
+ _LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)=yes
+ fi
+ fi
+ chmod u+w . 2>&AS_MESSAGE_LOG_FD
+ $RM conftest*
+ # SGI C++ compiler will create directory out/ii_files/ for
+ # template instantiation
+ test -d out/ii_files && $RM out/ii_files/* && rmdir out/ii_files
+ $RM out/* && rmdir out
+ cd ..
+ $RM -r conftest
+ $RM conftest*
+])
+_LT_TAGDECL([compiler_c_o], [lt_cv_prog_compiler_c_o], [1],
+ [Does compiler simultaneously support -c and -o options?])
+])# _LT_COMPILER_C_O
+
+
+# _LT_COMPILER_FILE_LOCKS([TAGNAME])
+# ----------------------------------
+# Check to see if we can do hard links to lock some files if needed
+m4_defun([_LT_COMPILER_FILE_LOCKS],
+[m4_require([_LT_ENABLE_LOCK])dnl
+m4_require([_LT_FILEUTILS_DEFAULTS])dnl
+_LT_COMPILER_C_O([$1])
+
+hard_links="nottested"
+if test "$_LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)" = no && test "$need_locks" != no; then
+ # do not overwrite the value of need_locks provided by the user
+ AC_MSG_CHECKING([if we can lock with hard links])
+ hard_links=yes
+ $RM conftest*
+ ln conftest.a conftest.b 2>/dev/null && hard_links=no
+ touch conftest.a
+ ln conftest.a conftest.b 2>&5 || hard_links=no
+ ln conftest.a conftest.b 2>/dev/null && hard_links=no
+ AC_MSG_RESULT([$hard_links])
+ if test "$hard_links" = no; then
+ AC_MSG_WARN([`$CC' does not support `-c -o', so `make -j' may be unsafe])
+ need_locks=warn
+ fi
+else
+ need_locks=no
+fi
+_LT_DECL([], [need_locks], [1], [Must we lock files when doing compilation?])
+])# _LT_COMPILER_FILE_LOCKS
+
+
+# _LT_CHECK_OBJDIR
+# ----------------
+m4_defun([_LT_CHECK_OBJDIR],
+[AC_CACHE_CHECK([for objdir], [lt_cv_objdir],
+[rm -f .libs 2>/dev/null
+mkdir .libs 2>/dev/null
+if test -d .libs; then
+ lt_cv_objdir=.libs
+else
+ # MS-DOS does not allow filenames that begin with a dot.
+ lt_cv_objdir=_libs
+fi
+rmdir .libs 2>/dev/null])
+objdir=$lt_cv_objdir
+_LT_DECL([], [objdir], [0],
+ [The name of the directory that contains temporary libtool files])dnl
+m4_pattern_allow([LT_OBJDIR])dnl
+AC_DEFINE_UNQUOTED(LT_OBJDIR, "$lt_cv_objdir/",
+ [Define to the sub-directory in which libtool stores uninstalled libraries.])
+])# _LT_CHECK_OBJDIR
+
+
+# _LT_LINKER_HARDCODE_LIBPATH([TAGNAME])
+# --------------------------------------
+# Check hardcoding attributes.
+m4_defun([_LT_LINKER_HARDCODE_LIBPATH],
+[AC_MSG_CHECKING([how to hardcode library paths into programs])
+_LT_TAGVAR(hardcode_action, $1)=
+if test -n "$_LT_TAGVAR(hardcode_libdir_flag_spec, $1)" ||
+ test -n "$_LT_TAGVAR(runpath_var, $1)" ||
+ test "X$_LT_TAGVAR(hardcode_automatic, $1)" = "Xyes" ; then
+
+ # We can hardcode non-existent directories.
+ if test "$_LT_TAGVAR(hardcode_direct, $1)" != no &&
+ # If the only mechanism to avoid hardcoding is shlibpath_var, we
+ # have to relink, otherwise we might link with an installed library
+ # when we should be linking with a yet-to-be-installed one
+ ## test "$_LT_TAGVAR(hardcode_shlibpath_var, $1)" != no &&
+ test "$_LT_TAGVAR(hardcode_minus_L, $1)" != no; then
+ # Linking always hardcodes the temporary library directory.
+ _LT_TAGVAR(hardcode_action, $1)=relink
+ else
+ # We can link without hardcoding, and we can hardcode nonexisting dirs.
+ _LT_TAGVAR(hardcode_action, $1)=immediate
+ fi
+else
+ # We cannot hardcode anything, or else we can only hardcode existing
+ # directories.
+ _LT_TAGVAR(hardcode_action, $1)=unsupported
+fi
+AC_MSG_RESULT([$_LT_TAGVAR(hardcode_action, $1)])
+
+if test "$_LT_TAGVAR(hardcode_action, $1)" = relink ||
+ test "$_LT_TAGVAR(inherit_rpath, $1)" = yes; then
+ # Fast installation is not supported
+ enable_fast_install=no
+elif test "$shlibpath_overrides_runpath" = yes ||
+ test "$enable_shared" = no; then
+ # Fast installation is not necessary
+ enable_fast_install=needless
+fi
+_LT_TAGDECL([], [hardcode_action], [0],
+ [How to hardcode a shared library path into an executable])
+])# _LT_LINKER_HARDCODE_LIBPATH
+
+
+# _LT_CMD_STRIPLIB
+# ----------------
+m4_defun([_LT_CMD_STRIPLIB],
+[m4_require([_LT_DECL_EGREP])
+striplib=
+old_striplib=
+AC_MSG_CHECKING([whether stripping libraries is possible])
+if test -n "$STRIP" && $STRIP -V 2>&1 | $GREP "GNU strip" >/dev/null; then
+ test -z "$old_striplib" && old_striplib="$STRIP --strip-debug"
+ test -z "$striplib" && striplib="$STRIP --strip-unneeded"
+ AC_MSG_RESULT([yes])
+else
+# FIXME - insert some real tests, host_os isn't really good enough
+ case $host_os in
+ darwin*)
+ if test -n "$STRIP" ; then
+ striplib="$STRIP -x"
+ old_striplib="$STRIP -S"
+ AC_MSG_RESULT([yes])
+ else
+ AC_MSG_RESULT([no])
+ fi
+ ;;
+ *)
+ AC_MSG_RESULT([no])
+ ;;
+ esac
+fi
+_LT_DECL([], [old_striplib], [1], [Commands to strip libraries])
+_LT_DECL([], [striplib], [1])
+])# _LT_CMD_STRIPLIB
+
+
+# _LT_SYS_DYNAMIC_LINKER([TAG])
+# -----------------------------
+# PORTME Fill in your ld.so characteristics
+m4_defun([_LT_SYS_DYNAMIC_LINKER],
+[AC_REQUIRE([AC_CANONICAL_HOST])dnl
+m4_require([_LT_DECL_EGREP])dnl
+m4_require([_LT_FILEUTILS_DEFAULTS])dnl
+m4_require([_LT_DECL_OBJDUMP])dnl
+m4_require([_LT_DECL_SED])dnl
+AC_MSG_CHECKING([dynamic linker characteristics])
+m4_if([$1],
+ [], [
+if test "$GCC" = yes; then
+ case $host_os in
+ darwin*) lt_awk_arg="/^libraries:/,/LR/" ;;
+ *) lt_awk_arg="/^libraries:/" ;;
+ esac
+ lt_search_path_spec=`$CC -print-search-dirs | awk $lt_awk_arg | $SED -e "s/^libraries://" -e "s,=/,/,g"`
+ if $ECHO "$lt_search_path_spec" | $GREP ';' >/dev/null ; then
+ # if the path contains ";" then we assume it to be the separator
+ # otherwise default to the standard path separator (i.e. ":") - it is
+ # assumed that no part of a normal pathname contains ";" but that should
+ # okay in the real world where ";" in dirpaths is itself problematic.
+ lt_search_path_spec=`$ECHO "$lt_search_path_spec" | $SED -e 's/;/ /g'`
+ else
+ lt_search_path_spec=`$ECHO "$lt_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"`
+ fi
+ # Ok, now we have the path, separated by spaces, we can step through it
+ # and add multilib dir if necessary.
+ lt_tmp_lt_search_path_spec=
+ lt_multi_os_dir=`$CC $CPPFLAGS $CFLAGS $LDFLAGS -print-multi-os-directory 2>/dev/null`
+ for lt_sys_path in $lt_search_path_spec; do
+ if test -d "$lt_sys_path/$lt_multi_os_dir"; then
+ lt_tmp_lt_search_path_spec="$lt_tmp_lt_search_path_spec $lt_sys_path/$lt_multi_os_dir"
+ else
+ test -d "$lt_sys_path" && \
+ lt_tmp_lt_search_path_spec="$lt_tmp_lt_search_path_spec $lt_sys_path"
+ fi
+ done
+ lt_search_path_spec=`$ECHO $lt_tmp_lt_search_path_spec | awk '
+BEGIN {RS=" "; FS="/|\n";} {
+ lt_foo="";
+ lt_count=0;
+ for (lt_i = NF; lt_i > 0; lt_i--) {
+ if ($lt_i != "" && $lt_i != ".") {
+ if ($lt_i == "..") {
+ lt_count++;
+ } else {
+ if (lt_count == 0) {
+ lt_foo="/" $lt_i lt_foo;
+ } else {
+ lt_count--;
+ }
+ }
+ }
+ }
+ if (lt_foo != "") { lt_freq[[lt_foo]]++; }
+ if (lt_freq[[lt_foo]] == 1) { print lt_foo; }
+}'`
+ sys_lib_search_path_spec=`$ECHO $lt_search_path_spec`
+else
+ sys_lib_search_path_spec="/lib /usr/lib /usr/local/lib"
+fi])
+library_names_spec=
+libname_spec='lib$name'
+soname_spec=
+shrext_cmds=".so"
+postinstall_cmds=
+postuninstall_cmds=
+finish_cmds=
+finish_eval=
+shlibpath_var=
+shlibpath_overrides_runpath=unknown
+version_type=none
+dynamic_linker="$host_os ld.so"
+sys_lib_dlsearch_path_spec="/lib /usr/lib"
+need_lib_prefix=unknown
+hardcode_into_libs=no
+
+# when you set need_version to no, make sure it does not cause -set_version
+# flags to be left without arguments
+need_version=unknown
+
+case $host_os in
+aix3*)
+ version_type=linux
+ library_names_spec='${libname}${release}${shared_ext}$versuffix $libname.a'
+ shlibpath_var=LIBPATH
+
+ # AIX 3 has no versioning support, so we append a major version to the name.
+ soname_spec='${libname}${release}${shared_ext}$major'
+ ;;
+
+aix[[4-9]]*)
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ hardcode_into_libs=yes
+ if test "$host_cpu" = ia64; then
+ # AIX 5 supports IA64
+ library_names_spec='${libname}${release}${shared_ext}$major ${libname}${release}${shared_ext}$versuffix $libname${shared_ext}'
+ shlibpath_var=LD_LIBRARY_PATH
+ else
+ # With GCC up to 2.95.x, collect2 would create an import file
+ # for dependence libraries. The import file would start with
+ # the line `#! .'. This would cause the generated library to
+ # depend on `.', always an invalid library. This was fixed in
+ # development snapshots of GCC prior to 3.0.
+ case $host_os in
+ aix4 | aix4.[[01]] | aix4.[[01]].*)
+ if { echo '#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 97)'
+ echo ' yes '
+ echo '#endif'; } | ${CC} -E - | $GREP yes > /dev/null; then
+ :
+ else
+ can_build_shared=no
+ fi
+ ;;
+ esac
+ # AIX (on Power*) has no versioning support, so currently we can not hardcode correct
+ # soname into executable. Probably we can add versioning support to
+ # collect2, so additional links can be useful in future.
+ if test "$aix_use_runtimelinking" = yes; then
+ # If using run time linking (on AIX 4.2 or later) use lib<name>.so
+ # instead of lib<name>.a to let people know that these are not
+ # typical AIX shared libraries.
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ else
+ # We preserve .a as extension for shared libraries through AIX4.2
+ # and later when we are not doing run time linking.
+ library_names_spec='${libname}${release}.a $libname.a'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ fi
+ shlibpath_var=LIBPATH
+ fi
+ ;;
+
+amigaos*)
+ case $host_cpu in
+ powerpc)
+ # Since July 2007 AmigaOS4 officially supports .so libraries.
+ # When compiling the executable, add -use-dynld -Lsobjs: to the compileline.
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ ;;
+ m68k)
+ library_names_spec='$libname.ixlibrary $libname.a'
+ # Create ${libname}_ixlibrary.a entries in /sys/libs.
+ finish_eval='for lib in `ls $libdir/*.ixlibrary 2>/dev/null`; do libname=`$ECHO "X$lib" | $Xsed -e '\''s%^.*/\([[^/]]*\)\.ixlibrary$%\1%'\''`; test $RM /sys/libs/${libname}_ixlibrary.a; $show "cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a"; cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a || exit 1; done'
+ ;;
+ esac
+ ;;
+
+beos*)
+ library_names_spec='${libname}${shared_ext}'
+ dynamic_linker="$host_os ld.so"
+ shlibpath_var=LIBRARY_PATH
+ ;;
+
+bsdi[[45]]*)
+ version_type=linux
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ finish_cmds='PATH="\$PATH:/sbin" ldconfig $libdir'
+ shlibpath_var=LD_LIBRARY_PATH
+ sys_lib_search_path_spec="/shlib /usr/lib /usr/X11/lib /usr/contrib/lib /lib /usr/local/lib"
+ sys_lib_dlsearch_path_spec="/shlib /usr/lib /usr/local/lib"
+ # the default ld.so.conf also contains /usr/contrib/lib and
+ # /usr/X11R6/lib (/usr/X11 is a link to /usr/X11R6), but let us allow
+ # libtool to hard-code these into programs
+ ;;
+
+cygwin* | mingw* | pw32* | cegcc*)
+ version_type=windows
+ shrext_cmds=".dll"
+ need_version=no
+ need_lib_prefix=no
+
+ case $GCC,$host_os in
+ yes,cygwin* | yes,mingw* | yes,pw32* | yes,cegcc*)
+ library_names_spec='$libname.dll.a'
+ # DLL is installed to $(libdir)/../bin by postinstall_cmds
+ postinstall_cmds='base_file=`basename \${file}`~
+ dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\${base_file}'\''i; echo \$dlname'\''`~
+ dldir=$destdir/`dirname \$dlpath`~
+ test -d \$dldir || mkdir -p \$dldir~
+ $install_prog $dir/$dlname \$dldir/$dlname~
+ chmod a+x \$dldir/$dlname~
+ if test -n '\''$stripme'\'' && test -n '\''$striplib'\''; then
+ eval '\''$striplib \$dldir/$dlname'\'' || exit \$?;
+ fi'
+ postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; echo \$dlname'\''`~
+ dlpath=$dir/\$dldll~
+ $RM \$dlpath'
+ shlibpath_overrides_runpath=yes
+
+ case $host_os in
+ cygwin*)
+ # Cygwin DLLs use 'cyg' prefix rather than 'lib'
+ soname_spec='`echo ${libname} | sed -e 's/^lib/cyg/'``echo ${release} | $SED -e 's/[[.]]/-/g'`${versuffix}${shared_ext}'
+ sys_lib_search_path_spec="/usr/lib /lib/w32api /lib /usr/local/lib"
+ ;;
+ mingw* | cegcc*)
+ # MinGW DLLs use traditional 'lib' prefix
+ soname_spec='${libname}`echo ${release} | $SED -e 's/[[.]]/-/g'`${versuffix}${shared_ext}'
+ sys_lib_search_path_spec=`$CC -print-search-dirs | $GREP "^libraries:" | $SED -e "s/^libraries://" -e "s,=/,/,g"`
+ if $ECHO "$sys_lib_search_path_spec" | [$GREP ';[c-zC-Z]:/' >/dev/null]; then
+ # It is most probably a Windows format PATH printed by
+ # mingw gcc, but we are running on Cygwin. Gcc prints its search
+ # path with ; separators, and with drive letters. We can handle the
+ # drive letters (cygwin fileutils understands them), so leave them,
+ # especially as we might pass files found there to a mingw objdump,
+ # which wouldn't understand a cygwinified path. Ahh.
+ sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e 's/;/ /g'`
+ else
+ sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"`
+ fi
+ ;;
+ pw32*)
+ # pw32 DLLs use 'pw' prefix rather than 'lib'
+ library_names_spec='`echo ${libname} | sed -e 's/^lib/pw/'``echo ${release} | $SED -e 's/[[.]]/-/g'`${versuffix}${shared_ext}'
+ ;;
+ esac
+ ;;
+
+ *)
+ library_names_spec='${libname}`echo ${release} | $SED -e 's/[[.]]/-/g'`${versuffix}${shared_ext} $libname.lib'
+ ;;
+ esac
+ dynamic_linker='Win32 ld.exe'
+ # FIXME: first we should search . and the directory the executable is in
+ shlibpath_var=PATH
+ ;;
+
+darwin* | rhapsody*)
+ dynamic_linker="$host_os dyld"
+ version_type=darwin
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${major}$shared_ext ${libname}$shared_ext'
+ soname_spec='${libname}${release}${major}$shared_ext'
+ shlibpath_overrides_runpath=yes
+ shlibpath_var=DYLD_LIBRARY_PATH
+ shrext_cmds='`test .$module = .yes && echo .so || echo .dylib`'
+m4_if([$1], [],[
+ sys_lib_search_path_spec="$sys_lib_search_path_spec /usr/local/lib"])
+ sys_lib_dlsearch_path_spec='/usr/local/lib /lib /usr/lib'
+ ;;
+
+dgux*)
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname$shared_ext'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ ;;
+
+freebsd1*)
+ dynamic_linker=no
+ ;;
+
+freebsd* | dragonfly*)
+ # DragonFly does not have aout. When/if they implement a new
+ # versioning mechanism, adjust this.
+ if test -x /usr/bin/objformat; then
+ objformat=`/usr/bin/objformat`
+ else
+ case $host_os in
+ freebsd[[123]]*) objformat=aout ;;
+ *) objformat=elf ;;
+ esac
+ fi
+ version_type=freebsd-$objformat
+ case $version_type in
+ freebsd-elf*)
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext} $libname${shared_ext}'
+ need_version=no
+ need_lib_prefix=no
+ ;;
+ freebsd-*)
+ library_names_spec='${libname}${release}${shared_ext}$versuffix $libname${shared_ext}$versuffix'
+ need_version=yes
+ ;;
+ esac
+ shlibpath_var=LD_LIBRARY_PATH
+ case $host_os in
+ freebsd2*)
+ shlibpath_overrides_runpath=yes
+ ;;
+ freebsd3.[[01]]* | freebsdelf3.[[01]]*)
+ shlibpath_overrides_runpath=yes
+ hardcode_into_libs=yes
+ ;;
+ freebsd3.[[2-9]]* | freebsdelf3.[[2-9]]* | \
+ freebsd4.[[0-5]] | freebsdelf4.[[0-5]] | freebsd4.1.1 | freebsdelf4.1.1)
+ shlibpath_overrides_runpath=no
+ hardcode_into_libs=yes
+ ;;
+ *) # from 4.6 on, and DragonFly
+ shlibpath_overrides_runpath=yes
+ hardcode_into_libs=yes
+ ;;
+ esac
+ ;;
+
+gnu*)
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}${major} ${libname}${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ hardcode_into_libs=yes
+ ;;
+
+hpux9* | hpux10* | hpux11*)
+ # Give a soname corresponding to the major version so that dld.sl refuses to
+ # link against other versions.
+ version_type=sunos
+ need_lib_prefix=no
+ need_version=no
+ case $host_cpu in
+ ia64*)
+ shrext_cmds='.so'
+ hardcode_into_libs=yes
+ dynamic_linker="$host_os dld.so"
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=yes # Unless +noenvvar is specified.
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ if test "X$HPUX_IA64_MODE" = X32; then
+ sys_lib_search_path_spec="/usr/lib/hpux32 /usr/local/lib/hpux32 /usr/local/lib"
+ else
+ sys_lib_search_path_spec="/usr/lib/hpux64 /usr/local/lib/hpux64"
+ fi
+ sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec
+ ;;
+ hppa*64*)
+ shrext_cmds='.sl'
+ hardcode_into_libs=yes
+ dynamic_linker="$host_os dld.sl"
+ shlibpath_var=LD_LIBRARY_PATH # How should we handle SHLIB_PATH
+ shlibpath_overrides_runpath=yes # Unless +noenvvar is specified.
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ sys_lib_search_path_spec="/usr/lib/pa20_64 /usr/ccs/lib/pa20_64"
+ sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec
+ ;;
+ *)
+ shrext_cmds='.sl'
+ dynamic_linker="$host_os dld.sl"
+ shlibpath_var=SHLIB_PATH
+ shlibpath_overrides_runpath=no # +s is required to enable SHLIB_PATH
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ ;;
+ esac
+ # HP-UX runs *really* slowly unless shared libraries are mode 555.
+ postinstall_cmds='chmod 555 $lib'
+ ;;
+
+interix[[3-9]]*)
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ dynamic_linker='Interix 3.x ld.so.1 (PE, like ELF)'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=no
+ hardcode_into_libs=yes
+ ;;
+
+irix5* | irix6* | nonstopux*)
+ case $host_os in
+ nonstopux*) version_type=nonstopux ;;
+ *)
+ if test "$lt_cv_prog_gnu_ld" = yes; then
+ version_type=linux
+ else
+ version_type=irix
+ fi ;;
+ esac
+ need_lib_prefix=no
+ need_version=no
+ soname_spec='${libname}${release}${shared_ext}$major'
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${release}${shared_ext} $libname${shared_ext}'
+ case $host_os in
+ irix5* | nonstopux*)
+ libsuff= shlibsuff=
+ ;;
+ *)
+ case $LD in # libtool.m4 will add one of these switches to LD
+ *-32|*"-32 "|*-melf32bsmip|*"-melf32bsmip ")
+ libsuff= shlibsuff= libmagic=32-bit;;
+ *-n32|*"-n32 "|*-melf32bmipn32|*"-melf32bmipn32 ")
+ libsuff=32 shlibsuff=N32 libmagic=N32;;
+ *-64|*"-64 "|*-melf64bmip|*"-melf64bmip ")
+ libsuff=64 shlibsuff=64 libmagic=64-bit;;
+ *) libsuff= shlibsuff= libmagic=never-match;;
+ esac
+ ;;
+ esac
+ shlibpath_var=LD_LIBRARY${shlibsuff}_PATH
+ shlibpath_overrides_runpath=no
+ sys_lib_search_path_spec="/usr/lib${libsuff} /lib${libsuff} /usr/local/lib${libsuff}"
+ sys_lib_dlsearch_path_spec="/usr/lib${libsuff} /lib${libsuff}"
+ hardcode_into_libs=yes
+ ;;
+
+# No shared lib support for Linux oldld, aout, or coff.
+linux*oldld* | linux*aout* | linux*coff*)
+ dynamic_linker=no
+ ;;
+
+# This must be Linux ELF.
+linux* | k*bsd*-gnu)
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ finish_cmds='PATH="\$PATH:/sbin" ldconfig -n $libdir'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=no
+ # Some binutils ld are patched to set DT_RUNPATH
+ save_LDFLAGS=$LDFLAGS
+ save_libdir=$libdir
+ eval "libdir=/foo; wl=\"$_LT_TAGVAR(lt_prog_compiler_wl, $1)\"; \
+ LDFLAGS=\"\$LDFLAGS $_LT_TAGVAR(hardcode_libdir_flag_spec, $1)\""
+ AC_LINK_IFELSE([AC_LANG_PROGRAM([],[])],
+ [AS_IF([ ($OBJDUMP -p conftest$ac_exeext) 2>/dev/null | grep "RUNPATH.*$libdir" >/dev/null],
+ [shlibpath_overrides_runpath=yes])])
+ LDFLAGS=$save_LDFLAGS
+ libdir=$save_libdir
+
+ # This implies no fast_install, which is unacceptable.
+ # Some rework will be needed to allow for fast_install
+ # before this can be enabled.
+ hardcode_into_libs=yes
+
+ # Add ABI-specific directories to the system library path.
+ sys_lib_dlsearch_path_spec="/lib64 /usr/lib64 /lib /usr/lib"
+
+ # Append ld.so.conf contents to the search path
+ if test -f /etc/ld.so.conf; then
+ lt_ld_extra=`awk '/^include / { system(sprintf("cd /etc; cat %s 2>/dev/null", \[$]2)); skip = 1; } { if (!skip) print \[$]0; skip = 0; }' < /etc/ld.so.conf | $SED -e 's/#.*//;/^[ ]*hwcap[ ]/d;s/[:, ]/ /g;s/=[^=]*$//;s/=[^= ]* / /g;/^$/d' | tr '\n' ' '`
+ sys_lib_dlsearch_path_spec="$sys_lib_dlsearch_path_spec $lt_ld_extra"
+ fi
+
+ # We used to test for /lib/ld.so.1 and disable shared libraries on
+ # powerpc, because MkLinux only supported shared libraries with the
+ # GNU dynamic linker. Since this was broken with cross compilers,
+ # most powerpc-linux boxes support dynamic linking these days and
+ # people can always --disable-shared, the test was removed, and we
+ # assume the GNU/Linux dynamic linker is in use.
+ dynamic_linker='GNU/Linux ld.so'
+ ;;
+
+netbsd*)
+ version_type=sunos
+ need_lib_prefix=no
+ need_version=no
+ if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix'
+ finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir'
+ dynamic_linker='NetBSD (a.out) ld.so'
+ else
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ dynamic_linker='NetBSD ld.elf_so'
+ fi
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=yes
+ hardcode_into_libs=yes
+ ;;
+
+newsos6)
+ version_type=linux
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=yes
+ ;;
+
+*nto* | *qnx*)
+ version_type=qnx
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=no
+ hardcode_into_libs=yes
+ dynamic_linker='ldqnx.so'
+ ;;
+
+openbsd*)
+ version_type=sunos
+ sys_lib_dlsearch_path_spec="/usr/lib"
+ need_lib_prefix=no
+ # Some older versions of OpenBSD (3.3 at least) *do* need versioned libs.
+ case $host_os in
+ openbsd3.3 | openbsd3.3.*) need_version=yes ;;
+ *) need_version=no ;;
+ esac
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix'
+ finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir'
+ shlibpath_var=LD_LIBRARY_PATH
+ if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then
+ case $host_os in
+ openbsd2.[[89]] | openbsd2.[[89]].*)
+ shlibpath_overrides_runpath=no
+ ;;
+ *)
+ shlibpath_overrides_runpath=yes
+ ;;
+ esac
+ else
+ shlibpath_overrides_runpath=yes
+ fi
+ ;;
+
+os2*)
+ libname_spec='$name'
+ shrext_cmds=".dll"
+ need_lib_prefix=no
+ library_names_spec='$libname${shared_ext} $libname.a'
+ dynamic_linker='OS/2 ld.exe'
+ shlibpath_var=LIBPATH
+ ;;
+
+osf3* | osf4* | osf5*)
+ version_type=osf
+ need_lib_prefix=no
+ need_version=no
+ soname_spec='${libname}${release}${shared_ext}$major'
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ shlibpath_var=LD_LIBRARY_PATH
+ sys_lib_search_path_spec="/usr/shlib /usr/ccs/lib /usr/lib/cmplrs/cc /usr/lib /usr/local/lib /var/shlib"
+ sys_lib_dlsearch_path_spec="$sys_lib_search_path_spec"
+ ;;
+
+rdos*)
+ dynamic_linker=no
+ ;;
+
+solaris*)
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=yes
+ hardcode_into_libs=yes
+ # ldd complains unless libraries are executable
+ postinstall_cmds='chmod +x $lib'
+ ;;
+
+sunos4*)
+ version_type=sunos
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix'
+ finish_cmds='PATH="\$PATH:/usr/etc" ldconfig $libdir'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=yes
+ if test "$with_gnu_ld" = yes; then
+ need_lib_prefix=no
+ fi
+ need_version=yes
+ ;;
+
+sysv4 | sysv4.3*)
+ version_type=linux
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ case $host_vendor in
+ sni)
+ shlibpath_overrides_runpath=no
+ need_lib_prefix=no
+ runpath_var=LD_RUN_PATH
+ ;;
+ siemens)
+ need_lib_prefix=no
+ ;;
+ motorola)
+ need_lib_prefix=no
+ need_version=no
+ shlibpath_overrides_runpath=no
+ sys_lib_search_path_spec='/lib /usr/lib /usr/ccs/lib'
+ ;;
+ esac
+ ;;
+
+sysv4*MP*)
+ if test -d /usr/nec ;then
+ version_type=linux
+ library_names_spec='$libname${shared_ext}.$versuffix $libname${shared_ext}.$major $libname${shared_ext}'
+ soname_spec='$libname${shared_ext}.$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ fi
+ ;;
+
+sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX* | sysv4*uw2*)
+ version_type=freebsd-elf
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext} $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=yes
+ hardcode_into_libs=yes
+ if test "$with_gnu_ld" = yes; then
+ sys_lib_search_path_spec='/usr/local/lib /usr/gnu/lib /usr/ccs/lib /usr/lib /lib'
+ else
+ sys_lib_search_path_spec='/usr/ccs/lib /usr/lib'
+ case $host_os in
+ sco3.2v5*)
+ sys_lib_search_path_spec="$sys_lib_search_path_spec /lib"
+ ;;
+ esac
+ fi
+ sys_lib_dlsearch_path_spec='/usr/lib'
+ ;;
+
+tpf*)
+ # TPF is a cross-target only. Preferred cross-host = GNU/Linux.
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=no
+ hardcode_into_libs=yes
+ ;;
+
+uts4*)
+ version_type=linux
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ ;;
+
+*)
+ dynamic_linker=no
+ ;;
+esac
+AC_MSG_RESULT([$dynamic_linker])
+test "$dynamic_linker" = no && can_build_shared=no
+
+variables_saved_for_relink="PATH $shlibpath_var $runpath_var"
+if test "$GCC" = yes; then
+ variables_saved_for_relink="$variables_saved_for_relink GCC_EXEC_PREFIX COMPILER_PATH LIBRARY_PATH"
+fi
+
+if test "${lt_cv_sys_lib_search_path_spec+set}" = set; then
+ sys_lib_search_path_spec="$lt_cv_sys_lib_search_path_spec"
+fi
+if test "${lt_cv_sys_lib_dlsearch_path_spec+set}" = set; then
+ sys_lib_dlsearch_path_spec="$lt_cv_sys_lib_dlsearch_path_spec"
+fi
+
+_LT_DECL([], [variables_saved_for_relink], [1],
+ [Variables whose values should be saved in libtool wrapper scripts and
+ restored at link time])
+_LT_DECL([], [need_lib_prefix], [0],
+ [Do we need the "lib" prefix for modules?])
+_LT_DECL([], [need_version], [0], [Do we need a version for libraries?])
+_LT_DECL([], [version_type], [0], [Library versioning type])
+_LT_DECL([], [runpath_var], [0], [Shared library runtime path variable])
+_LT_DECL([], [shlibpath_var], [0],[Shared library path variable])
+_LT_DECL([], [shlibpath_overrides_runpath], [0],
+ [Is shlibpath searched before the hard-coded library search path?])
+_LT_DECL([], [libname_spec], [1], [Format of library name prefix])
+_LT_DECL([], [library_names_spec], [1],
+ [[List of archive names. First name is the real one, the rest are links.
+ The last name is the one that the linker finds with -lNAME]])
+_LT_DECL([], [soname_spec], [1],
+ [[The coded name of the library, if different from the real name]])
+_LT_DECL([], [postinstall_cmds], [2],
+ [Command to use after installation of a shared archive])
+_LT_DECL([], [postuninstall_cmds], [2],
+ [Command to use after uninstallation of a shared archive])
+_LT_DECL([], [finish_cmds], [2],
+ [Commands used to finish a libtool library installation in a directory])
+_LT_DECL([], [finish_eval], [1],
+ [[As "finish_cmds", except a single script fragment to be evaled but
+ not shown]])
+_LT_DECL([], [hardcode_into_libs], [0],
+ [Whether we should hardcode library paths into libraries])
+_LT_DECL([], [sys_lib_search_path_spec], [2],
+ [Compile-time system search path for libraries])
+_LT_DECL([], [sys_lib_dlsearch_path_spec], [2],
+ [Run-time system search path for libraries])
+])# _LT_SYS_DYNAMIC_LINKER
+
+
+# _LT_PATH_TOOL_PREFIX(TOOL)
+# --------------------------
+# find a file program which can recognize shared library
+AC_DEFUN([_LT_PATH_TOOL_PREFIX],
+[m4_require([_LT_DECL_EGREP])dnl
+AC_MSG_CHECKING([for $1])
+AC_CACHE_VAL(lt_cv_path_MAGIC_CMD,
+[case $MAGIC_CMD in
+[[\\/*] | ?:[\\/]*])
+ lt_cv_path_MAGIC_CMD="$MAGIC_CMD" # Let the user override the test with a path.
+ ;;
+*)
+ lt_save_MAGIC_CMD="$MAGIC_CMD"
+ lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR
+dnl $ac_dummy forces splitting on constant user-supplied paths.
+dnl POSIX.2 word splitting is done only on the output of word expansions,
+dnl not every word. This closes a longstanding sh security hole.
+ ac_dummy="m4_if([$2], , $PATH, [$2])"
+ for ac_dir in $ac_dummy; do
+ IFS="$lt_save_ifs"
+ test -z "$ac_dir" && ac_dir=.
+ if test -f $ac_dir/$1; then
+ lt_cv_path_MAGIC_CMD="$ac_dir/$1"
+ if test -n "$file_magic_test_file"; then
+ case $deplibs_check_method in
+ "file_magic "*)
+ file_magic_regex=`expr "$deplibs_check_method" : "file_magic \(.*\)"`
+ MAGIC_CMD="$lt_cv_path_MAGIC_CMD"
+ if eval $file_magic_cmd \$file_magic_test_file 2> /dev/null |
+ $EGREP "$file_magic_regex" > /dev/null; then
+ :
+ else
+ cat <<_LT_EOF 1>&2
+
+*** Warning: the command libtool uses to detect shared libraries,
+*** $file_magic_cmd, produces output that libtool cannot recognize.
+*** The result is that libtool may fail to recognize shared libraries
+*** as such. This will affect the creation of libtool libraries that
+*** depend on shared libraries, but programs linked with such libtool
+*** libraries will work regardless of this problem. Nevertheless, you
+*** may want to report the problem to your system manager and/or to
+*** bug-libtool at gnu.org
+
+_LT_EOF
+ fi ;;
+ esac
+ fi
+ break
+ fi
+ done
+ IFS="$lt_save_ifs"
+ MAGIC_CMD="$lt_save_MAGIC_CMD"
+ ;;
+esac])
+MAGIC_CMD="$lt_cv_path_MAGIC_CMD"
+if test -n "$MAGIC_CMD"; then
+ AC_MSG_RESULT($MAGIC_CMD)
+else
+ AC_MSG_RESULT(no)
+fi
+_LT_DECL([], [MAGIC_CMD], [0],
+ [Used to examine libraries when file_magic_cmd begins with "file"])dnl
+])# _LT_PATH_TOOL_PREFIX
+
+# Old name:
+AU_ALIAS([AC_PATH_TOOL_PREFIX], [_LT_PATH_TOOL_PREFIX])
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AC_PATH_TOOL_PREFIX], [])
+
+
+# _LT_PATH_MAGIC
+# --------------
+# find a file program which can recognize a shared library
+m4_defun([_LT_PATH_MAGIC],
+[_LT_PATH_TOOL_PREFIX(${ac_tool_prefix}file, /usr/bin$PATH_SEPARATOR$PATH)
+if test -z "$lt_cv_path_MAGIC_CMD"; then
+ if test -n "$ac_tool_prefix"; then
+ _LT_PATH_TOOL_PREFIX(file, /usr/bin$PATH_SEPARATOR$PATH)
+ else
+ MAGIC_CMD=:
+ fi
+fi
+])# _LT_PATH_MAGIC
+
+
+# LT_PATH_LD
+# ----------
+# find the pathname to the GNU or non-GNU linker
+AC_DEFUN([LT_PATH_LD],
+[AC_REQUIRE([AC_PROG_CC])dnl
+AC_REQUIRE([AC_CANONICAL_HOST])dnl
+AC_REQUIRE([AC_CANONICAL_BUILD])dnl
+m4_require([_LT_DECL_SED])dnl
+m4_require([_LT_DECL_EGREP])dnl
+
+AC_ARG_WITH([gnu-ld],
+ [AS_HELP_STRING([--with-gnu-ld],
+ [assume the C compiler uses GNU ld @<:@default=no@:>@])],
+ [test "$withval" = no || with_gnu_ld=yes],
+ [with_gnu_ld=no])dnl
+
+ac_prog=ld
+if test "$GCC" = yes; then
+ # Check if gcc -print-prog-name=ld gives a path.
+ AC_MSG_CHECKING([for ld used by $CC])
+ case $host in
+ *-*-mingw*)
+ # gcc leaves a trailing carriage return which upsets mingw
+ ac_prog=`($CC -print-prog-name=ld) 2>&5 | tr -d '\015'` ;;
+ *)
+ ac_prog=`($CC -print-prog-name=ld) 2>&5` ;;
+ esac
+ case $ac_prog in
+ # Accept absolute paths.
+ [[\\/]]* | ?:[[\\/]]*)
+ re_direlt='/[[^/]][[^/]]*/\.\./'
+ # Canonicalize the pathname of ld
+ ac_prog=`$ECHO "$ac_prog"| $SED 's%\\\\%/%g'`
+ while $ECHO "$ac_prog" | $GREP "$re_direlt" > /dev/null 2>&1; do
+ ac_prog=`$ECHO $ac_prog| $SED "s%$re_direlt%/%"`
+ done
+ test -z "$LD" && LD="$ac_prog"
+ ;;
+ "")
+ # If it fails, then pretend we aren't using GCC.
+ ac_prog=ld
+ ;;
+ *)
+ # If it is relative, then search for the first ld in PATH.
+ with_gnu_ld=unknown
+ ;;
+ esac
+elif test "$with_gnu_ld" = yes; then
+ AC_MSG_CHECKING([for GNU ld])
+else
+ AC_MSG_CHECKING([for non-GNU ld])
+fi
+AC_CACHE_VAL(lt_cv_path_LD,
+[if test -z "$LD"; then
+ lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR
+ for ac_dir in $PATH; do
+ IFS="$lt_save_ifs"
+ test -z "$ac_dir" && ac_dir=.
+ if test -f "$ac_dir/$ac_prog" || test -f "$ac_dir/$ac_prog$ac_exeext"; then
+ lt_cv_path_LD="$ac_dir/$ac_prog"
+ # Check to see if the program is GNU ld. I'd rather use --version,
+ # but apparently some variants of GNU ld only accept -v.
+ # Break only if it was the GNU/non-GNU ld that we prefer.
+ case `"$lt_cv_path_LD" -v 2>&1 </dev/null` in
+ *GNU* | *'with BFD'*)
+ test "$with_gnu_ld" != no && break
+ ;;
+ *)
+ test "$with_gnu_ld" != yes && break
+ ;;
+ esac
+ fi
+ done
+ IFS="$lt_save_ifs"
+else
+ lt_cv_path_LD="$LD" # Let the user override the test with a path.
+fi])
+LD="$lt_cv_path_LD"
+if test -n "$LD"; then
+ AC_MSG_RESULT($LD)
+else
+ AC_MSG_RESULT(no)
+fi
+test -z "$LD" && AC_MSG_ERROR([no acceptable ld found in \$PATH])
+_LT_PATH_LD_GNU
+AC_SUBST([LD])
+
+_LT_TAGDECL([], [LD], [1], [The linker used to build libraries])
+])# LT_PATH_LD
+
+# Old names:
+AU_ALIAS([AM_PROG_LD], [LT_PATH_LD])
+AU_ALIAS([AC_PROG_LD], [LT_PATH_LD])
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AM_PROG_LD], [])
+dnl AC_DEFUN([AC_PROG_LD], [])
+
+
+# _LT_PATH_LD_GNU
+#- --------------
+m4_defun([_LT_PATH_LD_GNU],
+[AC_CACHE_CHECK([if the linker ($LD) is GNU ld], lt_cv_prog_gnu_ld,
+[# I'd rather use --version here, but apparently some GNU lds only accept -v.
+case `$LD -v 2>&1 </dev/null` in
+*GNU* | *'with BFD'*)
+ lt_cv_prog_gnu_ld=yes
+ ;;
+*)
+ lt_cv_prog_gnu_ld=no
+ ;;
+esac])
+with_gnu_ld=$lt_cv_prog_gnu_ld
+])# _LT_PATH_LD_GNU
+
+
+# _LT_CMD_RELOAD
+# --------------
+# find reload flag for linker
+# -- PORTME Some linkers may need a different reload flag.
+m4_defun([_LT_CMD_RELOAD],
+[AC_CACHE_CHECK([for $LD option to reload object files],
+ lt_cv_ld_reload_flag,
+ [lt_cv_ld_reload_flag='-r'])
+reload_flag=$lt_cv_ld_reload_flag
+case $reload_flag in
+"" | " "*) ;;
+*) reload_flag=" $reload_flag" ;;
+esac
+reload_cmds='$LD$reload_flag -o $output$reload_objs'
+case $host_os in
+ darwin*)
+ if test "$GCC" = yes; then
+ reload_cmds='$LTCC $LTCFLAGS -nostdlib ${wl}-r -o $output$reload_objs'
+ else
+ reload_cmds='$LD$reload_flag -o $output$reload_objs'
+ fi
+ ;;
+esac
+_LT_DECL([], [reload_flag], [1], [How to create reloadable object files])dnl
+_LT_DECL([], [reload_cmds], [2])dnl
+])# _LT_CMD_RELOAD
+
+
+# _LT_CHECK_MAGIC_METHOD
+# ----------------------
+# how to check for library dependencies
+# -- PORTME fill in with the dynamic library characteristics
+m4_defun([_LT_CHECK_MAGIC_METHOD],
+[m4_require([_LT_DECL_EGREP])
+m4_require([_LT_DECL_OBJDUMP])
+AC_CACHE_CHECK([how to recognize dependent libraries],
+lt_cv_deplibs_check_method,
+[lt_cv_file_magic_cmd='$MAGIC_CMD'
+lt_cv_file_magic_test_file=
+lt_cv_deplibs_check_method='unknown'
+# Need to set the preceding variable on all platforms that support
+# interlibrary dependencies.
+# 'none' -- dependencies not supported.
+# `unknown' -- same as none, but documents that we really don't know.
+# 'pass_all' -- all dependencies passed with no checks.
+# 'test_compile' -- check by making test program.
+# 'file_magic [[regex]]' -- check by looking for files in library path
+# which responds to the $file_magic_cmd with a given extended regex.
+# If you have `file' or equivalent on your system and you're not sure
+# whether `pass_all' will *always* work, you probably want this one.
+
+case $host_os in
+aix[[4-9]]*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+beos*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+bsdi[[45]]*)
+ lt_cv_deplibs_check_method='file_magic ELF [[0-9]][[0-9]]*-bit [[ML]]SB (shared object|dynamic lib)'
+ lt_cv_file_magic_cmd='/usr/bin/file -L'
+ lt_cv_file_magic_test_file=/shlib/libc.so
+ ;;
+
+cygwin*)
+ # func_win32_libid is a shell function defined in ltmain.sh
+ lt_cv_deplibs_check_method='file_magic ^x86 archive import|^x86 DLL'
+ lt_cv_file_magic_cmd='func_win32_libid'
+ ;;
+
+mingw* | pw32*)
+ # Base MSYS/MinGW do not provide the 'file' command needed by
+ # func_win32_libid shell function, so use a weaker test based on 'objdump',
+ # unless we find 'file', for example because we are cross-compiling.
+ if ( file / ) >/dev/null 2>&1; then
+ lt_cv_deplibs_check_method='file_magic ^x86 archive import|^x86 DLL'
+ lt_cv_file_magic_cmd='func_win32_libid'
+ else
+ lt_cv_deplibs_check_method='file_magic file format pei*-i386(.*architecture: i386)?'
+ lt_cv_file_magic_cmd='$OBJDUMP -f'
+ fi
+ ;;
+
+cegcc)
+ # use the weaker test based on 'objdump'. See mingw*.
+ lt_cv_deplibs_check_method='file_magic file format pe-arm-.*little(.*architecture: arm)?'
+ lt_cv_file_magic_cmd='$OBJDUMP -f'
+ ;;
+
+darwin* | rhapsody*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+freebsd* | dragonfly*)
+ if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then
+ case $host_cpu in
+ i*86 )
+ # Not sure whether the presence of OpenBSD here was a mistake.
+ # Let's accept both of them until this is cleared up.
+ lt_cv_deplibs_check_method='file_magic (FreeBSD|OpenBSD|DragonFly)/i[[3-9]]86 (compact )?demand paged shared library'
+ lt_cv_file_magic_cmd=/usr/bin/file
+ lt_cv_file_magic_test_file=`echo /usr/lib/libc.so.*`
+ ;;
+ esac
+ else
+ lt_cv_deplibs_check_method=pass_all
+ fi
+ ;;
+
+gnu*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+hpux10.20* | hpux11*)
+ lt_cv_file_magic_cmd=/usr/bin/file
+ case $host_cpu in
+ ia64*)
+ lt_cv_deplibs_check_method='file_magic (s[[0-9]][[0-9]][[0-9]]|ELF-[[0-9]][[0-9]]) shared object file - IA64'
+ lt_cv_file_magic_test_file=/usr/lib/hpux32/libc.so
+ ;;
+ hppa*64*)
+ [lt_cv_deplibs_check_method='file_magic (s[0-9][0-9][0-9]|ELF-[0-9][0-9]) shared object file - PA-RISC [0-9].[0-9]']
+ lt_cv_file_magic_test_file=/usr/lib/pa20_64/libc.sl
+ ;;
+ *)
+ lt_cv_deplibs_check_method='file_magic (s[[0-9]][[0-9]][[0-9]]|PA-RISC[[0-9]].[[0-9]]) shared library'
+ lt_cv_file_magic_test_file=/usr/lib/libc.sl
+ ;;
+ esac
+ ;;
+
+interix[[3-9]]*)
+ # PIC code is broken on Interix 3.x, that's why |\.a not |_pic\.a here
+ lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so|\.a)$'
+ ;;
+
+irix5* | irix6* | nonstopux*)
+ case $LD in
+ *-32|*"-32 ") libmagic=32-bit;;
+ *-n32|*"-n32 ") libmagic=N32;;
+ *-64|*"-64 ") libmagic=64-bit;;
+ *) libmagic=never-match;;
+ esac
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+# This must be Linux ELF.
+linux* | k*bsd*-gnu)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+netbsd*)
+ if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then
+ lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so\.[[0-9]]+\.[[0-9]]+|_pic\.a)$'
+ else
+ lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so|_pic\.a)$'
+ fi
+ ;;
+
+newos6*)
+ lt_cv_deplibs_check_method='file_magic ELF [[0-9]][[0-9]]*-bit [[ML]]SB (executable|dynamic lib)'
+ lt_cv_file_magic_cmd=/usr/bin/file
+ lt_cv_file_magic_test_file=/usr/lib/libnls.so
+ ;;
+
+*nto* | *qnx*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+openbsd*)
+ if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then
+ lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so\.[[0-9]]+\.[[0-9]]+|\.so|_pic\.a)$'
+ else
+ lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so\.[[0-9]]+\.[[0-9]]+|_pic\.a)$'
+ fi
+ ;;
+
+osf3* | osf4* | osf5*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+rdos*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+solaris*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX* | sysv4*uw2*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+sysv4 | sysv4.3*)
+ case $host_vendor in
+ motorola)
+ lt_cv_deplibs_check_method='file_magic ELF [[0-9]][[0-9]]*-bit [[ML]]SB (shared object|dynamic lib) M[[0-9]][[0-9]]* Version [[0-9]]'
+ lt_cv_file_magic_test_file=`echo /usr/lib/libc.so*`
+ ;;
+ ncr)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+ sequent)
+ lt_cv_file_magic_cmd='/bin/file'
+ lt_cv_deplibs_check_method='file_magic ELF [[0-9]][[0-9]]*-bit [[LM]]SB (shared object|dynamic lib )'
+ ;;
+ sni)
+ lt_cv_file_magic_cmd='/bin/file'
+ lt_cv_deplibs_check_method="file_magic ELF [[0-9]][[0-9]]*-bit [[LM]]SB dynamic lib"
+ lt_cv_file_magic_test_file=/lib/libc.so
+ ;;
+ siemens)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+ pc)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+ esac
+ ;;
+
+tpf*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+esac
+])
+file_magic_cmd=$lt_cv_file_magic_cmd
+deplibs_check_method=$lt_cv_deplibs_check_method
+test -z "$deplibs_check_method" && deplibs_check_method=unknown
+
+_LT_DECL([], [deplibs_check_method], [1],
+ [Method to check whether dependent libraries are shared objects])
+_LT_DECL([], [file_magic_cmd], [1],
+ [Command to use when deplibs_check_method == "file_magic"])
+])# _LT_CHECK_MAGIC_METHOD
+
+
+# LT_PATH_NM
+# ----------
+# find the pathname to a BSD- or MS-compatible name lister
+AC_DEFUN([LT_PATH_NM],
+[AC_REQUIRE([AC_PROG_CC])dnl
+AC_CACHE_CHECK([for BSD- or MS-compatible name lister (nm)], lt_cv_path_NM,
+[if test -n "$NM"; then
+ # Let the user override the test.
+ lt_cv_path_NM="$NM"
+else
+ lt_nm_to_check="${ac_tool_prefix}nm"
+ if test -n "$ac_tool_prefix" && test "$build" = "$host"; then
+ lt_nm_to_check="$lt_nm_to_check nm"
+ fi
+ for lt_tmp_nm in $lt_nm_to_check; do
+ lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR
+ for ac_dir in $PATH /usr/ccs/bin/elf /usr/ccs/bin /usr/ucb /bin; do
+ IFS="$lt_save_ifs"
+ test -z "$ac_dir" && ac_dir=.
+ tmp_nm="$ac_dir/$lt_tmp_nm"
+ if test -f "$tmp_nm" || test -f "$tmp_nm$ac_exeext" ; then
+ # Check to see if the nm accepts a BSD-compat flag.
+ # Adding the `sed 1q' prevents false positives on HP-UX, which says:
+ # nm: unknown option "B" ignored
+ # Tru64's nm complains that /dev/null is an invalid object file
+ case `"$tmp_nm" -B /dev/null 2>&1 | sed '1q'` in
+ */dev/null* | *'Invalid file or object type'*)
+ lt_cv_path_NM="$tmp_nm -B"
+ break
+ ;;
+ *)
+ case `"$tmp_nm" -p /dev/null 2>&1 | sed '1q'` in
+ */dev/null*)
+ lt_cv_path_NM="$tmp_nm -p"
+ break
+ ;;
+ *)
+ lt_cv_path_NM=${lt_cv_path_NM="$tmp_nm"} # keep the first match, but
+ continue # so that we can try to find one that supports BSD flags
+ ;;
+ esac
+ ;;
+ esac
+ fi
+ done
+ IFS="$lt_save_ifs"
+ done
+ : ${lt_cv_path_NM=no}
+fi])
+if test "$lt_cv_path_NM" != "no"; then
+ NM="$lt_cv_path_NM"
+else
+ # Didn't find any BSD compatible name lister, look for dumpbin.
+ AC_CHECK_TOOLS(DUMPBIN, ["dumpbin -symbols" "link -dump -symbols"], :)
+ AC_SUBST([DUMPBIN])
+ if test "$DUMPBIN" != ":"; then
+ NM="$DUMPBIN"
+ fi
+fi
+test -z "$NM" && NM=nm
+AC_SUBST([NM])
+_LT_DECL([], [NM], [1], [A BSD- or MS-compatible name lister])dnl
+
+AC_CACHE_CHECK([the name lister ($NM) interface], [lt_cv_nm_interface],
+ [lt_cv_nm_interface="BSD nm"
+ echo "int some_variable = 0;" > conftest.$ac_ext
+ (eval echo "\"\$as_me:__oline__: $ac_compile\"" >&AS_MESSAGE_LOG_FD)
+ (eval "$ac_compile" 2>conftest.err)
+ cat conftest.err >&AS_MESSAGE_LOG_FD
+ (eval echo "\"\$as_me:__oline__: $NM \\\"conftest.$ac_objext\\\"\"" >&AS_MESSAGE_LOG_FD)
+ (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
+ cat conftest.err >&AS_MESSAGE_LOG_FD
+ (eval echo "\"\$as_me:__oline__: output\"" >&AS_MESSAGE_LOG_FD)
+ cat conftest.out >&AS_MESSAGE_LOG_FD
+ if $GREP 'External.*some_variable' conftest.out > /dev/null; then
+ lt_cv_nm_interface="MS dumpbin"
+ fi
+ rm -f conftest*])
+])# LT_PATH_NM
+
+# Old names:
+AU_ALIAS([AM_PROG_NM], [LT_PATH_NM])
+AU_ALIAS([AC_PROG_NM], [LT_PATH_NM])
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AM_PROG_NM], [])
+dnl AC_DEFUN([AC_PROG_NM], [])
+
+
+# LT_LIB_M
+# --------
+# check for math library
+AC_DEFUN([LT_LIB_M],
+[AC_REQUIRE([AC_CANONICAL_HOST])dnl
+LIBM=
+case $host in
+*-*-beos* | *-*-cygwin* | *-*-pw32* | *-*-darwin*)
+ # These system don't have libm, or don't need it
+ ;;
+*-ncr-sysv4.3*)
+ AC_CHECK_LIB(mw, _mwvalidcheckl, LIBM="-lmw")
+ AC_CHECK_LIB(m, cos, LIBM="$LIBM -lm")
+ ;;
+*)
+ AC_CHECK_LIB(m, cos, LIBM="-lm")
+ ;;
+esac
+AC_SUBST([LIBM])
+])# LT_LIB_M
+
+# Old name:
+AU_ALIAS([AC_CHECK_LIBM], [LT_LIB_M])
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AC_CHECK_LIBM], [])
+
+
+# _LT_COMPILER_NO_RTTI([TAGNAME])
+# -------------------------------
+m4_defun([_LT_COMPILER_NO_RTTI],
+[m4_require([_LT_TAG_COMPILER])dnl
+
+_LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)=
+
+if test "$GCC" = yes; then
+ _LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)=' -fno-builtin'
+
+ _LT_COMPILER_OPTION([if $compiler supports -fno-rtti -fno-exceptions],
+ lt_cv_prog_compiler_rtti_exceptions,
+ [-fno-rtti -fno-exceptions], [],
+ [_LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)="$_LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1) -fno-rtti -fno-exceptions"])
+fi
+_LT_TAGDECL([no_builtin_flag], [lt_prog_compiler_no_builtin_flag], [1],
+ [Compiler flag to turn off builtin functions])
+])# _LT_COMPILER_NO_RTTI
+
+
+# _LT_CMD_GLOBAL_SYMBOLS
+# ----------------------
+m4_defun([_LT_CMD_GLOBAL_SYMBOLS],
+[AC_REQUIRE([AC_CANONICAL_HOST])dnl
+AC_REQUIRE([AC_PROG_CC])dnl
+AC_REQUIRE([LT_PATH_NM])dnl
+AC_REQUIRE([LT_PATH_LD])dnl
+m4_require([_LT_DECL_SED])dnl
+m4_require([_LT_DECL_EGREP])dnl
+m4_require([_LT_TAG_COMPILER])dnl
+
+# Check for command to grab the raw symbol name followed by C symbol from nm.
+AC_MSG_CHECKING([command to parse $NM output from $compiler object])
+AC_CACHE_VAL([lt_cv_sys_global_symbol_pipe],
+[
+# These are sane defaults that work on at least a few old systems.
+# [They come from Ultrix. What could be older than Ultrix?!! ;)]
+
+# Character class describing NM global symbol codes.
+symcode='[[BCDEGRST]]'
+
+# Regexp to match symbols that can be accessed directly from C.
+sympat='\([[_A-Za-z]][[_A-Za-z0-9]]*\)'
+
+# Define system-specific variables.
+case $host_os in
+aix*)
+ symcode='[[BCDT]]'
+ ;;
+cygwin* | mingw* | pw32* | cegcc*)
+ symcode='[[ABCDGISTW]]'
+ ;;
+hpux*)
+ if test "$host_cpu" = ia64; then
+ symcode='[[ABCDEGRST]]'
+ fi
+ ;;
+irix* | nonstopux*)
+ symcode='[[BCDEGRST]]'
+ ;;
+osf*)
+ symcode='[[BCDEGQRST]]'
+ ;;
+solaris*)
+ symcode='[[BDRT]]'
+ ;;
+sco3.2v5*)
+ symcode='[[DT]]'
+ ;;
+sysv4.2uw2*)
+ symcode='[[DT]]'
+ ;;
+sysv5* | sco5v6* | unixware* | OpenUNIX*)
+ symcode='[[ABDT]]'
+ ;;
+sysv4)
+ symcode='[[DFNSTU]]'
+ ;;
+esac
+
+# If we're using GNU nm, then use its standard symbol codes.
+case `$NM -V 2>&1` in
+*GNU* | *'with BFD'*)
+ symcode='[[ABCDGIRSTW]]' ;;
+esac
+
+# Transform an extracted symbol line into a proper C declaration.
+# Some systems (esp. on ia64) link data and code symbols differently,
+# so use this general approach.
+lt_cv_sys_global_symbol_to_cdecl="sed -n -e 's/^T .* \(.*\)$/extern int \1();/p' -e 's/^$symcode* .* \(.*\)$/extern char \1;/p'"
+
+# Transform an extracted symbol line into symbol name and symbol address
+lt_cv_sys_global_symbol_to_c_name_address="sed -n -e 's/^: \([[^ ]]*\) $/ {\\\"\1\\\", (void *) 0},/p' -e 's/^$symcode* \([[^ ]]*\) \([[^ ]]*\)$/ {\"\2\", (void *) \&\2},/p'"
+lt_cv_sys_global_symbol_to_c_name_address_lib_prefix="sed -n -e 's/^: \([[^ ]]*\) $/ {\\\"\1\\\", (void *) 0},/p' -e 's/^$symcode* \([[^ ]]*\) \(lib[[^ ]]*\)$/ {\"\2\", (void *) \&\2},/p' -e 's/^$symcode* \([[^ ]]*\) \([[^ ]]*\)$/ {\"lib\2\", (void *) \&\2},/p'"
+
+# Handle CRLF in mingw tool chain
+opt_cr=
+case $build_os in
+mingw*)
+ opt_cr=`$ECHO 'x\{0,1\}' | tr x '\015'` # option cr in regexp
+ ;;
+esac
+
+# Try without a prefix underscore, then with it.
+for ac_symprfx in "" "_"; do
+
+ # Transform symcode, sympat, and symprfx into a raw symbol and a C symbol.
+ symxfrm="\\1 $ac_symprfx\\2 \\2"
+
+ # Write the raw and C identifiers.
+ if test "$lt_cv_nm_interface" = "MS dumpbin"; then
+ # Fake it for dumpbin and say T for any non-static function
+ # and D for any global variable.
+ # Also find C++ and __fastcall symbols from MSVC++,
+ # which start with @ or ?.
+ lt_cv_sys_global_symbol_pipe="$AWK ['"\
+" {last_section=section; section=\$ 3};"\
+" /Section length .*#relocs.*(pick any)/{hide[last_section]=1};"\
+" \$ 0!~/External *\|/{next};"\
+" / 0+ UNDEF /{next}; / UNDEF \([^|]\)*()/{next};"\
+" {if(hide[section]) next};"\
+" {f=0}; \$ 0~/\(\).*\|/{f=1}; {printf f ? \"T \" : \"D \"};"\
+" {split(\$ 0, a, /\||\r/); split(a[2], s)};"\
+" s[1]~/^[@?]/{print s[1], s[1]; next};"\
+" s[1]~prfx {split(s[1],t,\"@\"); print t[1], substr(t[1],length(prfx))}"\
+" ' prfx=^$ac_symprfx]"
+ else
+ lt_cv_sys_global_symbol_pipe="sed -n -e 's/^.*[[ ]]\($symcode$symcode*\)[[ ]][[ ]]*$ac_symprfx$sympat$opt_cr$/$symxfrm/p'"
+ fi
+
+ # Check to see that the pipe works correctly.
+ pipe_works=no
+
+ rm -f conftest*
+ cat > conftest.$ac_ext <<_LT_EOF
+#ifdef __cplusplus
+extern "C" {
+#endif
+char nm_test_var;
+void nm_test_func(void);
+void nm_test_func(void){}
+#ifdef __cplusplus
+}
+#endif
+int main(){nm_test_var='a';nm_test_func();return(0);}
+_LT_EOF
+
+ if AC_TRY_EVAL(ac_compile); then
+ # Now try to grab the symbols.
+ nlist=conftest.nm
+ if AC_TRY_EVAL(NM conftest.$ac_objext \| $lt_cv_sys_global_symbol_pipe \> $nlist) && test -s "$nlist"; then
+ # Try sorting and uniquifying the output.
+ if sort "$nlist" | uniq > "$nlist"T; then
+ mv -f "$nlist"T "$nlist"
+ else
+ rm -f "$nlist"T
+ fi
+
+ # Make sure that we snagged all the symbols we need.
+ if $GREP ' nm_test_var$' "$nlist" >/dev/null; then
+ if $GREP ' nm_test_func$' "$nlist" >/dev/null; then
+ cat <<_LT_EOF > conftest.$ac_ext
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+_LT_EOF
+ # Now generate the symbol file.
+ eval "$lt_cv_sys_global_symbol_to_cdecl"' < "$nlist" | $GREP -v main >> conftest.$ac_ext'
+
+ cat <<_LT_EOF >> conftest.$ac_ext
+
+/* The mapping between symbol names and symbols. */
+const struct {
+ const char *name;
+ void *address;
+}
+lt__PROGRAM__LTX_preloaded_symbols[[]] =
+{
+ { "@PROGRAM@", (void *) 0 },
+_LT_EOF
+ $SED "s/^$symcode$symcode* \(.*\) \(.*\)$/ {\"\2\", (void *) \&\2},/" < "$nlist" | $GREP -v main >> conftest.$ac_ext
+ cat <<\_LT_EOF >> conftest.$ac_ext
+ {0, (void *) 0}
+};
+
+/* This works around a problem in FreeBSD linker */
+#ifdef FREEBSD_WORKAROUND
+static const void *lt_preloaded_setup() {
+ return lt__PROGRAM__LTX_preloaded_symbols;
+}
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+_LT_EOF
+ # Now try linking the two files.
+ mv conftest.$ac_objext conftstm.$ac_objext
+ lt_save_LIBS="$LIBS"
+ lt_save_CFLAGS="$CFLAGS"
+ LIBS="conftstm.$ac_objext"
+ CFLAGS="$CFLAGS$_LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)"
+ if AC_TRY_EVAL(ac_link) && test -s conftest${ac_exeext}; then
+ pipe_works=yes
+ fi
+ LIBS="$lt_save_LIBS"
+ CFLAGS="$lt_save_CFLAGS"
+ else
+ echo "cannot find nm_test_func in $nlist" >&AS_MESSAGE_LOG_FD
+ fi
+ else
+ echo "cannot find nm_test_var in $nlist" >&AS_MESSAGE_LOG_FD
+ fi
+ else
+ echo "cannot run $lt_cv_sys_global_symbol_pipe" >&AS_MESSAGE_LOG_FD
+ fi
+ else
+ echo "$progname: failed program was:" >&AS_MESSAGE_LOG_FD
+ cat conftest.$ac_ext >&5
+ fi
+ rm -rf conftest* conftst*
+
+ # Do not use the global_symbol_pipe unless it works.
+ if test "$pipe_works" = yes; then
+ break
+ else
+ lt_cv_sys_global_symbol_pipe=
+ fi
+done
+])
+if test -z "$lt_cv_sys_global_symbol_pipe"; then
+ lt_cv_sys_global_symbol_to_cdecl=
+fi
+if test -z "$lt_cv_sys_global_symbol_pipe$lt_cv_sys_global_symbol_to_cdecl"; then
+ AC_MSG_RESULT(failed)
+else
+ AC_MSG_RESULT(ok)
+fi
+
+_LT_DECL([global_symbol_pipe], [lt_cv_sys_global_symbol_pipe], [1],
+ [Take the output of nm and produce a listing of raw symbols and C names])
+_LT_DECL([global_symbol_to_cdecl], [lt_cv_sys_global_symbol_to_cdecl], [1],
+ [Transform the output of nm in a proper C declaration])
+_LT_DECL([global_symbol_to_c_name_address],
+ [lt_cv_sys_global_symbol_to_c_name_address], [1],
+ [Transform the output of nm in a C name address pair])
+_LT_DECL([global_symbol_to_c_name_address_lib_prefix],
+ [lt_cv_sys_global_symbol_to_c_name_address_lib_prefix], [1],
+ [Transform the output of nm in a C name address pair when lib prefix is needed])
+]) # _LT_CMD_GLOBAL_SYMBOLS
+
+
+# _LT_COMPILER_PIC([TAGNAME])
+# ---------------------------
+m4_defun([_LT_COMPILER_PIC],
+[m4_require([_LT_TAG_COMPILER])dnl
+_LT_TAGVAR(lt_prog_compiler_wl, $1)=
+_LT_TAGVAR(lt_prog_compiler_pic, $1)=
+_LT_TAGVAR(lt_prog_compiler_static, $1)=
+
+AC_MSG_CHECKING([for $compiler option to produce PIC])
+m4_if([$1], [CXX], [
+ # C++ specific cases for pic, static, wl, etc.
+ if test "$GXX" = yes; then
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-static'
+
+ case $host_os in
+ aix*)
+ # All AIX code is PIC.
+ if test "$host_cpu" = ia64; then
+ # AIX 5 now supports IA64 processor
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ fi
+ ;;
+
+ amigaos*)
+ case $host_cpu in
+ powerpc)
+ # see comment about AmigaOS4 .so support
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC'
+ ;;
+ m68k)
+ # FIXME: we need at least 68020 code to build shared libraries, but
+ # adding the `-m68020' flag to GCC prevents building anything better,
+ # like `-m68040'.
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-m68020 -resident32 -malways-restore-a4'
+ ;;
+ esac
+ ;;
+
+ beos* | irix5* | irix6* | nonstopux* | osf3* | osf4* | osf5*)
+ # PIC is the default for these OSes.
+ ;;
+ mingw* | cygwin* | os2* | pw32* | cegcc*)
+ # This hack is so that the source file can tell whether it is being
+ # built for inclusion in a dll (and should export symbols for example).
+ # Although the cygwin gcc ignores -fPIC, still need this for old-style
+ # (--disable-auto-import) libraries
+ m4_if([$1], [GCJ], [],
+ [_LT_TAGVAR(lt_prog_compiler_pic, $1)='-DDLL_EXPORT'])
+ ;;
+ darwin* | rhapsody*)
+ # PIC is the default on this platform
+ # Common symbols not allowed in MH_DYLIB files
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fno-common'
+ ;;
+ *djgpp*)
+ # DJGPP does not support shared libraries at all
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)=
+ ;;
+ interix[[3-9]]*)
+ # Interix 3.x gcc -fpic/-fPIC options generate broken code.
+ # Instead, we relocate shared libraries at runtime.
+ ;;
+ sysv4*MP*)
+ if test -d /usr/nec; then
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)=-Kconform_pic
+ fi
+ ;;
+ hpux*)
+ # PIC is the default for 64-bit PA HP-UX, but not for 32-bit
+ # PA HP-UX. On IA64 HP-UX, PIC is the default but the pic flag
+ # sets the default TLS model and affects inlining.
+ case $host_cpu in
+ hppa*64*)
+ ;;
+ *)
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC'
+ ;;
+ esac
+ ;;
+ *qnx* | *nto*)
+ # QNX uses GNU C++, but need to define -shared option too, otherwise
+ # it will coredump.
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC -shared'
+ ;;
+ *)
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC'
+ ;;
+ esac
+ else
+ case $host_os in
+ aix[[4-9]]*)
+ # All AIX code is PIC.
+ if test "$host_cpu" = ia64; then
+ # AIX 5 now supports IA64 processor
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ else
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-bnso -bI:/lib/syscalls.exp'
+ fi
+ ;;
+ chorus*)
+ case $cc_basename in
+ cxch68*)
+ # Green Hills C++ Compiler
+ # _LT_TAGVAR(lt_prog_compiler_static, $1)="--no_auto_instantiation -u __main -u __premain -u _abort -r $COOL_DIR/lib/libOrb.a $MVME_DIR/lib/CC/libC.a $MVME_DIR/lib/classix/libcx.s.a"
+ ;;
+ esac
+ ;;
+ dgux*)
+ case $cc_basename in
+ ec++*)
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC'
+ ;;
+ ghcx*)
+ # Green Hills C++ Compiler
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic'
+ ;;
+ *)
+ ;;
+ esac
+ ;;
+ freebsd* | dragonfly*)
+ # FreeBSD uses GNU C++
+ ;;
+ hpux9* | hpux10* | hpux11*)
+ case $cc_basename in
+ CC*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='${wl}-a ${wl}archive'
+ if test "$host_cpu" != ia64; then
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='+Z'
+ fi
+ ;;
+ aCC*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='${wl}-a ${wl}archive'
+ case $host_cpu in
+ hppa*64*|ia64*)
+ # +Z the default
+ ;;
+ *)
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='+Z'
+ ;;
+ esac
+ ;;
+ *)
+ ;;
+ esac
+ ;;
+ interix*)
+ # This is c89, which is MS Visual C++ (no shared libs)
+ # Anyone wants to do a port?
+ ;;
+ irix5* | irix6* | nonstopux*)
+ case $cc_basename in
+ CC*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared'
+ # CC pic flag -KPIC is the default.
+ ;;
+ *)
+ ;;
+ esac
+ ;;
+ linux* | k*bsd*-gnu)
+ case $cc_basename in
+ KCC*)
+ # KAI C++ Compiler
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='--backend -Wl,'
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC'
+ ;;
+ ecpc* )
+ # old Intel C++ for x86_64 which still supported -KPIC.
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-static'
+ ;;
+ icpc* )
+ # Intel C++, used to be incompatible with GCC.
+ # ICC 10 doesn't accept -KPIC any more.
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-static'
+ ;;
+ pgCC* | pgcpp*)
+ # Portland Group C++ compiler
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fpic'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ ;;
+ cxx*)
+ # Compaq C++
+ # Make sure the PIC flag is empty. It appears that all Alpha
+ # Linux and Compaq Tru64 Unix objects are PIC.
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)=
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared'
+ ;;
+ xlc* | xlC*)
+ # IBM XL 8.0 on PPC
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-qpic'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-qstaticlink'
+ ;;
+ *)
+ case `$CC -V 2>&1 | sed 5q` in
+ *Sun\ C*)
+ # Sun C++ 5.9
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld '
+ ;;
+ esac
+ ;;
+ esac
+ ;;
+ lynxos*)
+ ;;
+ m88k*)
+ ;;
+ mvs*)
+ case $cc_basename in
+ cxx*)
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-W c,exportall'
+ ;;
+ *)
+ ;;
+ esac
+ ;;
+ netbsd*)
+ ;;
+ *qnx* | *nto*)
+ # QNX uses GNU C++, but need to define -shared option too, otherwise
+ # it will coredump.
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC -shared'
+ ;;
+ osf3* | osf4* | osf5*)
+ case $cc_basename in
+ KCC*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='--backend -Wl,'
+ ;;
+ RCC*)
+ # Rational C++ 2.4.1
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic'
+ ;;
+ cxx*)
+ # Digital/Compaq C++
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ # Make sure the PIC flag is empty. It appears that all Alpha
+ # Linux and Compaq Tru64 Unix objects are PIC.
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)=
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared'
+ ;;
+ *)
+ ;;
+ esac
+ ;;
+ psos*)
+ ;;
+ solaris*)
+ case $cc_basename in
+ CC*)
+ # Sun C++ 4.2, 5.x and Centerline C++
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld '
+ ;;
+ gcx*)
+ # Green Hills C++ Compiler
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-PIC'
+ ;;
+ *)
+ ;;
+ esac
+ ;;
+ sunos4*)
+ case $cc_basename in
+ CC*)
+ # Sun C++ 4.x
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ ;;
+ lcc*)
+ # Lucid
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic'
+ ;;
+ *)
+ ;;
+ esac
+ ;;
+ sysv5* | unixware* | sco3.2v5* | sco5v6* | OpenUNIX*)
+ case $cc_basename in
+ CC*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ ;;
+ esac
+ ;;
+ tandem*)
+ case $cc_basename in
+ NCC*)
+ # NonStop-UX NCC 3.20
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC'
+ ;;
+ *)
+ ;;
+ esac
+ ;;
+ vxworks*)
+ ;;
+ *)
+ _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no
+ ;;
+ esac
+ fi
+],
+[
+ if test "$GCC" = yes; then
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-static'
+
+ case $host_os in
+ aix*)
+ # All AIX code is PIC.
+ if test "$host_cpu" = ia64; then
+ # AIX 5 now supports IA64 processor
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ fi
+ ;;
+
+ amigaos*)
+ case $host_cpu in
+ powerpc)
+ # see comment about AmigaOS4 .so support
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC'
+ ;;
+ m68k)
+ # FIXME: we need at least 68020 code to build shared libraries, but
+ # adding the `-m68020' flag to GCC prevents building anything better,
+ # like `-m68040'.
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-m68020 -resident32 -malways-restore-a4'
+ ;;
+ esac
+ ;;
+
+ beos* | irix5* | irix6* | nonstopux* | osf3* | osf4* | osf5*)
+ # PIC is the default for these OSes.
+ ;;
+
+ mingw* | cygwin* | pw32* | os2* | cegcc*)
+ # This hack is so that the source file can tell whether it is being
+ # built for inclusion in a dll (and should export symbols for example).
+ # Although the cygwin gcc ignores -fPIC, still need this for old-style
+ # (--disable-auto-import) libraries
+ m4_if([$1], [GCJ], [],
+ [_LT_TAGVAR(lt_prog_compiler_pic, $1)='-DDLL_EXPORT'])
+ ;;
+
+ darwin* | rhapsody*)
+ # PIC is the default on this platform
+ # Common symbols not allowed in MH_DYLIB files
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fno-common'
+ ;;
+
+ hpux*)
+ # PIC is the default for 64-bit PA HP-UX, but not for 32-bit
+ # PA HP-UX. On IA64 HP-UX, PIC is the default but the pic flag
+ # sets the default TLS model and affects inlining.
+ case $host_cpu in
+ hppa*64*)
+ # +Z the default
+ ;;
+ *)
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC'
+ ;;
+ esac
+ ;;
+
+ interix[[3-9]]*)
+ # Interix 3.x gcc -fpic/-fPIC options generate broken code.
+ # Instead, we relocate shared libraries at runtime.
+ ;;
+
+ msdosdjgpp*)
+ # Just because we use GCC doesn't mean we suddenly get shared libraries
+ # on systems that don't support them.
+ _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no
+ enable_shared=no
+ ;;
+
+ *nto* | *qnx*)
+ # QNX uses GNU C++, but need to define -shared option too, otherwise
+ # it will coredump.
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC -shared'
+ ;;
+
+ sysv4*MP*)
+ if test -d /usr/nec; then
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)=-Kconform_pic
+ fi
+ ;;
+
+ *)
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC'
+ ;;
+ esac
+ else
+ # PORTME Check for flag to pass linker flags through the system compiler.
+ case $host_os in
+ aix*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ if test "$host_cpu" = ia64; then
+ # AIX 5 now supports IA64 processor
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ else
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-bnso -bI:/lib/syscalls.exp'
+ fi
+ ;;
+
+ mingw* | cygwin* | pw32* | os2* | cegcc*)
+ # This hack is so that the source file can tell whether it is being
+ # built for inclusion in a dll (and should export symbols for example).
+ m4_if([$1], [GCJ], [],
+ [_LT_TAGVAR(lt_prog_compiler_pic, $1)='-DDLL_EXPORT'])
+ ;;
+
+ hpux9* | hpux10* | hpux11*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ # PIC is the default for IA64 HP-UX and 64-bit HP-UX, but
+ # not for PA HP-UX.
+ case $host_cpu in
+ hppa*64*|ia64*)
+ # +Z the default
+ ;;
+ *)
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='+Z'
+ ;;
+ esac
+ # Is there a better lt_prog_compiler_static that works with the bundled CC?
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='${wl}-a ${wl}archive'
+ ;;
+
+ irix5* | irix6* | nonstopux*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ # PIC (with -KPIC) is the default.
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared'
+ ;;
+
+ linux* | k*bsd*-gnu)
+ case $cc_basename in
+ # old Intel for x86_64 which still supported -KPIC.
+ ecc*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-static'
+ ;;
+ # icc used to be incompatible with GCC.
+ # ICC 10 doesn't accept -KPIC any more.
+ icc* | ifort*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-static'
+ ;;
+ # Lahey Fortran 8.1.
+ lf95*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='--shared'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='--static'
+ ;;
+ pgcc* | pgf77* | pgf90* | pgf95*)
+ # Portland Group compilers (*not* the Pentium gcc compiler,
+ # which looks to be a dead project)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fpic'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ ;;
+ ccc*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ # All Alpha code is PIC.
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared'
+ ;;
+ xl*)
+ # IBM XL C 8.0/Fortran 10.1 on PPC
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-qpic'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-qstaticlink'
+ ;;
+ *)
+ case `$CC -V 2>&1 | sed 5q` in
+ *Sun\ C*)
+ # Sun C 5.9
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ ;;
+ *Sun\ F*)
+ # Sun Fortran 8.3 passes all unrecognized flags to the linker
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)=''
+ ;;
+ esac
+ ;;
+ esac
+ ;;
+
+ newsos6)
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ ;;
+
+ *nto* | *qnx*)
+ # QNX uses GNU C++, but need to define -shared option too, otherwise
+ # it will coredump.
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC -shared'
+ ;;
+
+ osf3* | osf4* | osf5*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ # All OSF/1 code is PIC.
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared'
+ ;;
+
+ rdos*)
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared'
+ ;;
+
+ solaris*)
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ case $cc_basename in
+ f77* | f90* | f95*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld ';;
+ *)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,';;
+ esac
+ ;;
+
+ sunos4*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld '
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-PIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ ;;
+
+ sysv4 | sysv4.2uw2* | sysv4.3*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ ;;
+
+ sysv4*MP*)
+ if test -d /usr/nec ;then
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-Kconform_pic'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ fi
+ ;;
+
+ sysv5* | unixware* | sco3.2v5* | sco5v6* | OpenUNIX*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ ;;
+
+ unicos*)
+ _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,'
+ _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no
+ ;;
+
+ uts4*)
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic'
+ _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic'
+ ;;
+
+ *)
+ _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no
+ ;;
+ esac
+ fi
+])
+case $host_os in
+ # For platforms which do not support PIC, -DPIC is meaningless:
+ *djgpp*)
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)=
+ ;;
+ *)
+ _LT_TAGVAR(lt_prog_compiler_pic, $1)="$_LT_TAGVAR(lt_prog_compiler_pic, $1)@&t at m4_if([$1],[],[ -DPIC],[m4_if([$1],[CXX],[ -DPIC],[])])"
+ ;;
+esac
+AC_MSG_RESULT([$_LT_TAGVAR(lt_prog_compiler_pic, $1)])
+_LT_TAGDECL([wl], [lt_prog_compiler_wl], [1],
+ [How to pass a linker flag through the compiler])
+
+#
+# Check to make sure the PIC flag actually works.
+#
+if test -n "$_LT_TAGVAR(lt_prog_compiler_pic, $1)"; then
+ _LT_COMPILER_OPTION([if $compiler PIC flag $_LT_TAGVAR(lt_prog_compiler_pic, $1) works],
+ [_LT_TAGVAR(lt_cv_prog_compiler_pic_works, $1)],
+ [$_LT_TAGVAR(lt_prog_compiler_pic, $1)@&t at m4_if([$1],[],[ -DPIC],[m4_if([$1],[CXX],[ -DPIC],[])])], [],
+ [case $_LT_TAGVAR(lt_prog_compiler_pic, $1) in
+ "" | " "*) ;;
+ *) _LT_TAGVAR(lt_prog_compiler_pic, $1)=" $_LT_TAGVAR(lt_prog_compiler_pic, $1)" ;;
+ esac],
+ [_LT_TAGVAR(lt_prog_compiler_pic, $1)=
+ _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no])
+fi
+_LT_TAGDECL([pic_flag], [lt_prog_compiler_pic], [1],
+ [Additional compiler flags for building library objects])
+
+#
+# Check to make sure the static flag actually works.
+#
+wl=$_LT_TAGVAR(lt_prog_compiler_wl, $1) eval lt_tmp_static_flag=\"$_LT_TAGVAR(lt_prog_compiler_static, $1)\"
+_LT_LINKER_OPTION([if $compiler static flag $lt_tmp_static_flag works],
+ _LT_TAGVAR(lt_cv_prog_compiler_static_works, $1),
+ $lt_tmp_static_flag,
+ [],
+ [_LT_TAGVAR(lt_prog_compiler_static, $1)=])
+_LT_TAGDECL([link_static_flag], [lt_prog_compiler_static], [1],
+ [Compiler flag to prevent dynamic linking])
+])# _LT_COMPILER_PIC
+
+
+# _LT_LINKER_SHLIBS([TAGNAME])
+# ----------------------------
+# See if the linker supports building shared libraries.
+m4_defun([_LT_LINKER_SHLIBS],
+[AC_REQUIRE([LT_PATH_LD])dnl
+AC_REQUIRE([LT_PATH_NM])dnl
+m4_require([_LT_FILEUTILS_DEFAULTS])dnl
+m4_require([_LT_DECL_EGREP])dnl
+m4_require([_LT_DECL_SED])dnl
+m4_require([_LT_CMD_GLOBAL_SYMBOLS])dnl
+m4_require([_LT_TAG_COMPILER])dnl
+AC_MSG_CHECKING([whether the $compiler linker ($LD) supports shared libraries])
+m4_if([$1], [CXX], [
+ _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols'
+ case $host_os in
+ aix[[4-9]]*)
+ # If we're using GNU nm, then we don't want the "-C" option.
+ # -C means demangle to AIX nm, but means don't demangle with GNU nm
+ if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then
+ _LT_TAGVAR(export_symbols_cmds, $1)='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B")) && ([substr](\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols'
+ else
+ _LT_TAGVAR(export_symbols_cmds, $1)='$NM -BCpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B")) && ([substr](\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols'
+ fi
+ ;;
+ pw32*)
+ _LT_TAGVAR(export_symbols_cmds, $1)="$ltdll_cmds"
+ ;;
+ cygwin* | mingw* | cegcc*)
+ _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[[BCDGRS]][[ ]]/s/.*[[ ]]\([[^ ]]*\)/\1 DATA/;/^.*[[ ]]__nm__/s/^.*[[ ]]__nm__\([[^ ]]*\)[[ ]][[^ ]]*/\1 DATA/;/^I[[ ]]/d;/^[[AITW]][[ ]]/s/.* //'\'' | sort | uniq > $export_symbols'
+ ;;
+ *)
+ _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols'
+ ;;
+ esac
+ _LT_TAGVAR(exclude_expsyms, $1)=['_GLOBAL_OFFSET_TABLE_|_GLOBAL__F[ID]_.*']
+], [
+ runpath_var=
+ _LT_TAGVAR(allow_undefined_flag, $1)=
+ _LT_TAGVAR(always_export_symbols, $1)=no
+ _LT_TAGVAR(archive_cmds, $1)=
+ _LT_TAGVAR(archive_expsym_cmds, $1)=
+ _LT_TAGVAR(compiler_needs_object, $1)=no
+ _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=no
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)=
+ _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols'
+ _LT_TAGVAR(hardcode_automatic, $1)=no
+ _LT_TAGVAR(hardcode_direct, $1)=no
+ _LT_TAGVAR(hardcode_direct_absolute, $1)=no
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)=
+ _LT_TAGVAR(hardcode_libdir_flag_spec_ld, $1)=
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=
+ _LT_TAGVAR(hardcode_minus_L, $1)=no
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=unsupported
+ _LT_TAGVAR(inherit_rpath, $1)=no
+ _LT_TAGVAR(link_all_deplibs, $1)=unknown
+ _LT_TAGVAR(module_cmds, $1)=
+ _LT_TAGVAR(module_expsym_cmds, $1)=
+ _LT_TAGVAR(old_archive_from_new_cmds, $1)=
+ _LT_TAGVAR(old_archive_from_expsyms_cmds, $1)=
+ _LT_TAGVAR(thread_safe_flag_spec, $1)=
+ _LT_TAGVAR(whole_archive_flag_spec, $1)=
+ # include_expsyms should be a list of space-separated symbols to be *always*
+ # included in the symbol list
+ _LT_TAGVAR(include_expsyms, $1)=
+ # exclude_expsyms can be an extended regexp of symbols to exclude
+ # it will be wrapped by ` (' and `)$', so one must not match beginning or
+ # end of line. Example: `a|bc|.*d.*' will exclude the symbols `a' and `bc',
+ # as well as any symbol that contains `d'.
+ _LT_TAGVAR(exclude_expsyms, $1)=['_GLOBAL_OFFSET_TABLE_|_GLOBAL__F[ID]_.*']
+ # Although _GLOBAL_OFFSET_TABLE_ is a valid symbol C name, most a.out
+ # platforms (ab)use it in PIC code, but their linkers get confused if
+ # the symbol is explicitly referenced. Since portable code cannot
+ # rely on this symbol name, it's probably fine to never include it in
+ # preloaded symbol tables.
+ # Exclude shared library initialization/finalization symbols.
+dnl Note also adjust exclude_expsyms for C++ above.
+ extract_expsyms_cmds=
+
+ case $host_os in
+ cygwin* | mingw* | pw32* | cegcc*)
+ # FIXME: the MSVC++ port hasn't been tested in a loooong time
+ # When not using gcc, we currently assume that we are using
+ # Microsoft Visual C++.
+ if test "$GCC" != yes; then
+ with_gnu_ld=no
+ fi
+ ;;
+ interix*)
+ # we just hope/assume this is gcc and not c89 (= MSVC++)
+ with_gnu_ld=yes
+ ;;
+ openbsd*)
+ with_gnu_ld=no
+ ;;
+ esac
+
+ _LT_TAGVAR(ld_shlibs, $1)=yes
+ if test "$with_gnu_ld" = yes; then
+ # If archive_cmds runs LD, not CC, wlarc should be empty
+ wlarc='${wl}'
+
+ # Set some defaults for GNU ld with shared library support. These
+ # are reset later if shared libraries are not supported. Putting them
+ # here allows them to be overridden if necessary.
+ runpath_var=LD_RUN_PATH
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir'
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-dynamic'
+ # ancient GNU ld didn't support --whole-archive et. al.
+ if $LD --help 2>&1 | $GREP 'no-whole-archive' > /dev/null; then
+ _LT_TAGVAR(whole_archive_flag_spec, $1)="$wlarc"'--whole-archive$convenience '"$wlarc"'--no-whole-archive'
+ else
+ _LT_TAGVAR(whole_archive_flag_spec, $1)=
+ fi
+ supports_anon_versioning=no
+ case `$LD -v 2>&1` in
+ *\ [[01]].* | *\ 2.[[0-9]].* | *\ 2.10.*) ;; # catch versions < 2.11
+ *\ 2.11.93.0.2\ *) supports_anon_versioning=yes ;; # RH7.3 ...
+ *\ 2.11.92.0.12\ *) supports_anon_versioning=yes ;; # Mandrake 8.2 ...
+ *\ 2.11.*) ;; # other 2.11 versions
+ *) supports_anon_versioning=yes ;;
+ esac
+
+ # See if GNU ld supports shared libraries.
+ case $host_os in
+ aix[[3-9]]*)
+ # On AIX/PPC, the GNU linker is very broken
+ if test "$host_cpu" != ia64; then
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ cat <<_LT_EOF 1>&2
+
+*** Warning: the GNU linker, at least up to release 2.9.1, is reported
+*** to be unable to reliably create shared libraries on AIX.
+*** Therefore, libtool is disabling shared libraries support. If you
+*** really care for shared libraries, you may want to modify your PATH
+*** so that a non-GNU linker is found, and then restart.
+
+_LT_EOF
+ fi
+ ;;
+
+ amigaos*)
+ case $host_cpu in
+ powerpc)
+ # see comment about AmigaOS4 .so support
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)=''
+ ;;
+ m68k)
+ _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/a2ixlibrary.data~$ECHO "#define NAME $libname" > $output_objdir/a2ixlibrary.data~$ECHO "#define LIBRARY_ID 1" >> $output_objdir/a2ixlibrary.data~$ECHO "#define VERSION $major" >> $output_objdir/a2ixlibrary.data~$ECHO "#define REVISION $revision" >> $output_objdir/a2ixlibrary.data~$AR $AR_FLAGS $lib $libobjs~$RANLIB $lib~(cd $output_objdir && a2ixlibrary -32)'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir'
+ _LT_TAGVAR(hardcode_minus_L, $1)=yes
+ ;;
+ esac
+ ;;
+
+ beos*)
+ if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then
+ _LT_TAGVAR(allow_undefined_flag, $1)=unsupported
+ # Joseph Beckenbach <jrb3 at best.com> says some releases of gcc
+ # support --undefined. This deserves some investigation. FIXME
+ _LT_TAGVAR(archive_cmds, $1)='$CC -nostart $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ else
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+ ;;
+
+ cygwin* | mingw* | pw32* | cegcc*)
+ # _LT_TAGVAR(hardcode_libdir_flag_spec, $1) is actually meaningless,
+ # as there is no search path for DLLs.
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir'
+ _LT_TAGVAR(allow_undefined_flag, $1)=unsupported
+ _LT_TAGVAR(always_export_symbols, $1)=no
+ _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes
+ _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[[BCDGRS]][[ ]]/s/.*[[ ]]\([[^ ]]*\)/\1 DATA/'\'' | $SED -e '\''/^[[AITW]][[ ]]/s/.*[[ ]]//'\'' | sort | uniq > $export_symbols'
+
+ if $LD --help 2>&1 | $GREP 'auto-import' > /dev/null; then
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib'
+ # If the export-symbols file already is a .def file (1st line
+ # is EXPORTS), use it as is; otherwise, prepend...
+ _LT_TAGVAR(archive_expsym_cmds, $1)='if test "x`$SED 1q $export_symbols`" = xEXPORTS; then
+ cp $export_symbols $output_objdir/$soname.def;
+ else
+ echo EXPORTS > $output_objdir/$soname.def;
+ cat $export_symbols >> $output_objdir/$soname.def;
+ fi~
+ $CC -shared $output_objdir/$soname.def $libobjs $deplibs $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib'
+ else
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+ ;;
+
+ interix[[3-9]]*)
+ _LT_TAGVAR(hardcode_direct, $1)=no
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir'
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E'
+ # Hack: On Interix 3.x, we cannot compile PIC because of a broken gcc.
+ # Instead, shared libraries are loaded at an image base (0x10000000 by
+ # default) and relocated if they conflict, which is a slow very memory
+ # consuming and fragmenting process. To avoid this, we pick a random,
+ # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link
+ # time. Moving up from 0x10000000 also allows more sbrk(2) space.
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='sed "s,^,_," $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--retain-symbols-file,$output_objdir/$soname.expsym ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib'
+ ;;
+
+ gnu* | linux* | tpf* | k*bsd*-gnu)
+ tmp_diet=no
+ if test "$host_os" = linux-dietlibc; then
+ case $cc_basename in
+ diet\ *) tmp_diet=yes;; # linux-dietlibc with static linking (!diet-dyn)
+ esac
+ fi
+ if $LD --help 2>&1 | $EGREP ': supported targets:.* elf' > /dev/null \
+ && test "$tmp_diet" = no
+ then
+ tmp_addflag=
+ tmp_sharedflag='-shared'
+ case $cc_basename,$host_cpu in
+ pgcc*) # Portland Group C compiler
+ _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; $ECHO \"$new_convenience\"` ${wl}--no-whole-archive'
+ tmp_addflag=' $pic_flag'
+ ;;
+ pgf77* | pgf90* | pgf95*) # Portland Group f77 and f90 compilers
+ _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; $ECHO \"$new_convenience\"` ${wl}--no-whole-archive'
+ tmp_addflag=' $pic_flag -Mnomain' ;;
+ ecc*,ia64* | icc*,ia64*) # Intel C compiler on ia64
+ tmp_addflag=' -i_dynamic' ;;
+ efc*,ia64* | ifort*,ia64*) # Intel Fortran compiler on ia64
+ tmp_addflag=' -i_dynamic -nofor_main' ;;
+ ifc* | ifort*) # Intel Fortran compiler
+ tmp_addflag=' -nofor_main' ;;
+ lf95*) # Lahey Fortran 8.1
+ _LT_TAGVAR(whole_archive_flag_spec, $1)=
+ tmp_sharedflag='--shared' ;;
+ xl[[cC]]*) # IBM XL C 8.0 on PPC (deal with xlf below)
+ tmp_sharedflag='-qmkshrobj'
+ tmp_addflag= ;;
+ esac
+ case `$CC -V 2>&1 | sed 5q` in
+ *Sun\ C*) # Sun C 5.9
+ _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive`new_convenience=; for conv in $convenience\"\"; do test -z \"$conv\" || new_convenience=\"$new_convenience,$conv\"; done; $ECHO \"$new_convenience\"` ${wl}--no-whole-archive'
+ _LT_TAGVAR(compiler_needs_object, $1)=yes
+ tmp_sharedflag='-G' ;;
+ *Sun\ F*) # Sun Fortran 8.3
+ tmp_sharedflag='-G' ;;
+ esac
+ _LT_TAGVAR(archive_cmds, $1)='$CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+
+ if test "x$supports_anon_versioning" = xyes; then
+ _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $output_objdir/$libname.ver~
+ cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~
+ echo "local: *; };" >> $output_objdir/$libname.ver~
+ $CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-version-script ${wl}$output_objdir/$libname.ver -o $lib'
+ fi
+
+ case $cc_basename in
+ xlf*)
+ # IBM XL Fortran 10.1 on PPC cannot create shared libs itself
+ _LT_TAGVAR(whole_archive_flag_spec, $1)='--whole-archive$convenience --no-whole-archive'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)=
+ _LT_TAGVAR(hardcode_libdir_flag_spec_ld, $1)='-rpath $libdir'
+ _LT_TAGVAR(archive_cmds, $1)='$LD -shared $libobjs $deplibs $compiler_flags -soname $soname -o $lib'
+ if test "x$supports_anon_versioning" = xyes; then
+ _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $output_objdir/$libname.ver~
+ cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~
+ echo "local: *; };" >> $output_objdir/$libname.ver~
+ $LD -shared $libobjs $deplibs $compiler_flags -soname $soname -version-script $output_objdir/$libname.ver -o $lib'
+ fi
+ ;;
+ esac
+ else
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+ ;;
+
+ netbsd*)
+ if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then
+ _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable $libobjs $deplibs $linker_flags -o $lib'
+ wlarc=
+ else
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib'
+ fi
+ ;;
+
+ solaris*)
+ if $LD -v 2>&1 | $GREP 'BFD 2\.8' > /dev/null; then
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ cat <<_LT_EOF 1>&2
+
+*** Warning: The releases 2.8.* of the GNU linker cannot reliably
+*** create shared libraries on Solaris systems. Therefore, libtool
+*** is disabling shared libraries support. We urge you to upgrade GNU
+*** binutils to release 2.9.1 or newer. Another option is to modify
+*** your PATH or compiler configuration so that the native linker is
+*** used, and then restart.
+
+_LT_EOF
+ elif $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib'
+ else
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+ ;;
+
+ sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX*)
+ case `$LD -v 2>&1` in
+ *\ [[01]].* | *\ 2.[[0-9]].* | *\ 2.1[[0-5]].*)
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ cat <<_LT_EOF 1>&2
+
+*** Warning: Releases of the GNU linker prior to 2.16.91.0.3 can not
+*** reliably create shared libraries on SCO systems. Therefore, libtool
+*** is disabling shared libraries support. We urge you to upgrade GNU
+*** binutils to release 2.16.91.0.3 or newer. Another option is to modify
+*** your PATH or compiler configuration so that the native linker is
+*** used, and then restart.
+
+_LT_EOF
+ ;;
+ *)
+ # For security reasons, it is highly recommended that you always
+ # use absolute paths for naming shared libraries, and exclude the
+ # DT_RUNPATH tag from executables and libraries. But doing so
+ # requires that you compile everything twice, which is a pain.
+ if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir'
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib'
+ else
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+ ;;
+ esac
+ ;;
+
+ sunos4*)
+ _LT_TAGVAR(archive_cmds, $1)='$LD -assert pure-text -Bshareable -o $lib $libobjs $deplibs $linker_flags'
+ wlarc=
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ ;;
+
+ *)
+ if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib'
+ else
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+ ;;
+ esac
+
+ if test "$_LT_TAGVAR(ld_shlibs, $1)" = no; then
+ runpath_var=
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)=
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)=
+ _LT_TAGVAR(whole_archive_flag_spec, $1)=
+ fi
+ else
+ # PORTME fill in a description of your system's linker (not GNU ld)
+ case $host_os in
+ aix3*)
+ _LT_TAGVAR(allow_undefined_flag, $1)=unsupported
+ _LT_TAGVAR(always_export_symbols, $1)=yes
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$LD -o $output_objdir/$soname $libobjs $deplibs $linker_flags -bE:$export_symbols -T512 -H512 -bM:SRE~$AR $AR_FLAGS $lib $output_objdir/$soname'
+ # Note: this linker hardcodes the directories in LIBPATH if there
+ # are no directories specified by -L.
+ _LT_TAGVAR(hardcode_minus_L, $1)=yes
+ if test "$GCC" = yes && test -z "$lt_prog_compiler_static"; then
+ # Neither direct hardcoding nor static linking is supported with a
+ # broken collect2.
+ _LT_TAGVAR(hardcode_direct, $1)=unsupported
+ fi
+ ;;
+
+ aix[[4-9]]*)
+ if test "$host_cpu" = ia64; then
+ # On IA64, the linker does run time linking by default, so we don't
+ # have to do anything special.
+ aix_use_runtimelinking=no
+ exp_sym_flag='-Bexport'
+ no_entry_flag=""
+ else
+ # If we're using GNU nm, then we don't want the "-C" option.
+ # -C means demangle to AIX nm, but means don't demangle with GNU nm
+ if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then
+ _LT_TAGVAR(export_symbols_cmds, $1)='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B")) && ([substr](\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols'
+ else
+ _LT_TAGVAR(export_symbols_cmds, $1)='$NM -BCpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B")) && ([substr](\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols'
+ fi
+ aix_use_runtimelinking=no
+
+ # Test if we are trying to use run time linking or normal
+ # AIX style linking. If -brtl is somewhere in LDFLAGS, we
+ # need to do runtime linking.
+ case $host_os in aix4.[[23]]|aix4.[[23]].*|aix[[5-9]]*)
+ for ld_flag in $LDFLAGS; do
+ if (test $ld_flag = "-brtl" || test $ld_flag = "-Wl,-brtl"); then
+ aix_use_runtimelinking=yes
+ break
+ fi
+ done
+ ;;
+ esac
+
+ exp_sym_flag='-bexport'
+ no_entry_flag='-bnoentry'
+ fi
+
+ # When large executables or shared objects are built, AIX ld can
+ # have problems creating the table of contents. If linking a library
+ # or program results in "error TOC overflow" add -mminimal-toc to
+ # CXXFLAGS/CFLAGS for g++/gcc. In the cases where that is not
+ # enough to fix the problem, add -Wl,-bbigtoc to LDFLAGS.
+
+ _LT_TAGVAR(archive_cmds, $1)=''
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_direct_absolute, $1)=yes
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=':'
+ _LT_TAGVAR(link_all_deplibs, $1)=yes
+ _LT_TAGVAR(file_list_spec, $1)='${wl}-f,'
+
+ if test "$GCC" = yes; then
+ case $host_os in aix4.[[012]]|aix4.[[012]].*)
+ # We only want to do this on AIX 4.2 and lower, the check
+ # below for broken collect2 doesn't work under 4.3+
+ collect2name=`${CC} -print-prog-name=collect2`
+ if test -f "$collect2name" &&
+ strings "$collect2name" | $GREP resolve_lib_name >/dev/null
+ then
+ # We have reworked collect2
+ :
+ else
+ # We have old collect2
+ _LT_TAGVAR(hardcode_direct, $1)=unsupported
+ # It fails to find uninstalled libraries when the uninstalled
+ # path is not listed in the libpath. Setting hardcode_minus_L
+ # to unsupported forces relinking
+ _LT_TAGVAR(hardcode_minus_L, $1)=yes
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=
+ fi
+ ;;
+ esac
+ shared_flag='-shared'
+ if test "$aix_use_runtimelinking" = yes; then
+ shared_flag="$shared_flag "'${wl}-G'
+ fi
+ else
+ # not using gcc
+ if test "$host_cpu" = ia64; then
+ # VisualAge C++, Version 5.5 for AIX 5L for IA-64, Beta 3 Release
+ # chokes on -Wl,-G. The following line is correct:
+ shared_flag='-G'
+ else
+ if test "$aix_use_runtimelinking" = yes; then
+ shared_flag='${wl}-G'
+ else
+ shared_flag='${wl}-bM:SRE'
+ fi
+ fi
+ fi
+
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-bexpall'
+ # It seems that -bexpall does not export symbols beginning with
+ # underscore (_), so it is better to generate a list of symbols to export.
+ _LT_TAGVAR(always_export_symbols, $1)=yes
+ if test "$aix_use_runtimelinking" = yes; then
+ # Warning - without using the other runtime loading flags (-brtl),
+ # -berok will link without error, but may produce a broken library.
+ _LT_TAGVAR(allow_undefined_flag, $1)='-berok'
+ # Determine the default libpath from the value encoded in an
+ # empty executable.
+ _LT_SYS_MODULE_PATH_AIX
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-blibpath:$libdir:'"$aix_libpath"
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags `if test "x${allow_undefined_flag}" != "x"; then $ECHO "X${wl}${allow_undefined_flag}" | $Xsed; else :; fi` '"\${wl}$exp_sym_flag:\$export_symbols $shared_flag"
+ else
+ if test "$host_cpu" = ia64; then
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-R $libdir:/usr/lib:/lib'
+ _LT_TAGVAR(allow_undefined_flag, $1)="-z nodefs"
+ _LT_TAGVAR(archive_expsym_cmds, $1)="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags ${wl}${allow_undefined_flag} '"\${wl}$exp_sym_flag:\$export_symbols"
+ else
+ # Determine the default libpath from the value encoded in an
+ # empty executable.
+ _LT_SYS_MODULE_PATH_AIX
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-blibpath:$libdir:'"$aix_libpath"
+ # Warning - without using the other run time loading flags,
+ # -berok will link without error, but may produce a broken library.
+ _LT_TAGVAR(no_undefined_flag, $1)=' ${wl}-bernotok'
+ _LT_TAGVAR(allow_undefined_flag, $1)=' ${wl}-berok'
+ # Exported symbols can be pulled into shared objects from archives
+ _LT_TAGVAR(whole_archive_flag_spec, $1)='$convenience'
+ _LT_TAGVAR(archive_cmds_need_lc, $1)=yes
+ # This is similar to how AIX traditionally builds its shared libraries.
+ _LT_TAGVAR(archive_expsym_cmds, $1)="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs ${wl}-bnoentry $compiler_flags ${wl}-bE:$export_symbols${allow_undefined_flag}~$AR $AR_FLAGS $output_objdir/$libname$release.a $output_objdir/$soname'
+ fi
+ fi
+ ;;
+
+ amigaos*)
+ case $host_cpu in
+ powerpc)
+ # see comment about AmigaOS4 .so support
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)=''
+ ;;
+ m68k)
+ _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/a2ixlibrary.data~$ECHO "#define NAME $libname" > $output_objdir/a2ixlibrary.data~$ECHO "#define LIBRARY_ID 1" >> $output_objdir/a2ixlibrary.data~$ECHO "#define VERSION $major" >> $output_objdir/a2ixlibrary.data~$ECHO "#define REVISION $revision" >> $output_objdir/a2ixlibrary.data~$AR $AR_FLAGS $lib $libobjs~$RANLIB $lib~(cd $output_objdir && a2ixlibrary -32)'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir'
+ _LT_TAGVAR(hardcode_minus_L, $1)=yes
+ ;;
+ esac
+ ;;
+
+ bsdi[[45]]*)
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)=-rdynamic
+ ;;
+
+ cygwin* | mingw* | pw32* | cegcc*)
+ # When not using gcc, we currently assume that we are using
+ # Microsoft Visual C++.
+ # hardcode_libdir_flag_spec is actually meaningless, as there is
+ # no search path for DLLs.
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)=' '
+ _LT_TAGVAR(allow_undefined_flag, $1)=unsupported
+ # Tell ltmain to make .lib files, not .a files.
+ libext=lib
+ # Tell ltmain to make .dll files, not .so files.
+ shrext_cmds=".dll"
+ # FIXME: Setting linknames here is a bad hack.
+ _LT_TAGVAR(archive_cmds, $1)='$CC -o $lib $libobjs $compiler_flags `$ECHO "X$deplibs" | $Xsed -e '\''s/ -lc$//'\''` -link -dll~linknames='
+ # The linker will automatically build a .lib file if we build a DLL.
+ _LT_TAGVAR(old_archive_from_new_cmds, $1)='true'
+ # FIXME: Should let the user specify the lib program.
+ _LT_TAGVAR(old_archive_cmds, $1)='lib -OUT:$oldlib$oldobjs$old_deplibs'
+ _LT_TAGVAR(fix_srcfile_path, $1)='`cygpath -w "$srcfile"`'
+ _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes
+ ;;
+
+ darwin* | rhapsody*)
+ _LT_DARWIN_LINKER_FEATURES($1)
+ ;;
+
+ dgux*)
+ _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir'
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ ;;
+
+ freebsd1*)
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+
+ # FreeBSD 2.2.[012] allows us to include c++rt0.o to get C++ constructor
+ # support. Future versions do this automatically, but an explicit c++rt0.o
+ # does not break anything, and helps significantly (at the cost of a little
+ # extra space).
+ freebsd2.2*)
+ _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags /usr/lib/c++rt0.o'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir'
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ ;;
+
+ # Unfortunately, older versions of FreeBSD 2 do not have this feature.
+ freebsd2*)
+ _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags'
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_minus_L, $1)=yes
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ ;;
+
+ # FreeBSD 3 and greater uses gcc -shared to do shared libraries.
+ freebsd* | dragonfly*)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -o $lib $libobjs $deplibs $compiler_flags'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir'
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ ;;
+
+ hpux9*)
+ if test "$GCC" = yes; then
+ _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/$soname~$CC -shared -fPIC ${wl}+b ${wl}$install_libdir -o $output_objdir/$soname $libobjs $deplibs $compiler_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib'
+ else
+ _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/$soname~$LD -b +b $install_libdir -o $output_objdir/$soname $libobjs $deplibs $linker_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib'
+ fi
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}+b ${wl}$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+
+ # hardcode_minus_L: Not really in the search PATH,
+ # but as the default location of the library.
+ _LT_TAGVAR(hardcode_minus_L, $1)=yes
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E'
+ ;;
+
+ hpux10*)
+ if test "$GCC" = yes -a "$with_gnu_ld" = no; then
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -fPIC ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $libobjs $deplibs $compiler_flags'
+ else
+ _LT_TAGVAR(archive_cmds, $1)='$LD -b +h $soname +b $install_libdir -o $lib $libobjs $deplibs $linker_flags'
+ fi
+ if test "$with_gnu_ld" = no; then
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}+b ${wl}$libdir'
+ _LT_TAGVAR(hardcode_libdir_flag_spec_ld, $1)='+b $libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_direct_absolute, $1)=yes
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E'
+ # hardcode_minus_L: Not really in the search PATH,
+ # but as the default location of the library.
+ _LT_TAGVAR(hardcode_minus_L, $1)=yes
+ fi
+ ;;
+
+ hpux11*)
+ if test "$GCC" = yes -a "$with_gnu_ld" = no; then
+ case $host_cpu in
+ hppa*64*)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared ${wl}+h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ ia64*)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -fPIC ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ *)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -fPIC ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ esac
+ else
+ case $host_cpu in
+ hppa*64*)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -b ${wl}+h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ ia64*)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -b ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ *)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -b ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ esac
+ fi
+ if test "$with_gnu_ld" = no; then
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}+b ${wl}$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+
+ case $host_cpu in
+ hppa*64*|ia64*)
+ _LT_TAGVAR(hardcode_direct, $1)=no
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ ;;
+ *)
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_direct_absolute, $1)=yes
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E'
+
+ # hardcode_minus_L: Not really in the search PATH,
+ # but as the default location of the library.
+ _LT_TAGVAR(hardcode_minus_L, $1)=yes
+ ;;
+ esac
+ fi
+ ;;
+
+ irix5* | irix6* | nonstopux*)
+ if test "$GCC" = yes; then
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && $ECHO "X${wl}-set_version ${wl}$verstring" | $Xsed` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib'
+ # Try to use the -exported_symbol ld option, if it does not
+ # work, assume that -exports_file does not work either and
+ # implicitly export all symbols.
+ save_LDFLAGS="$LDFLAGS"
+ LDFLAGS="$LDFLAGS -shared ${wl}-exported_symbol ${wl}foo ${wl}-update_registry ${wl}/dev/null"
+ AC_LINK_IFELSE(int foo(void) {},
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && $ECHO "X${wl}-set_version ${wl}$verstring" | $Xsed` ${wl}-update_registry ${wl}${output_objdir}/so_locations ${wl}-exports_file ${wl}$export_symbols -o $lib'
+ )
+ LDFLAGS="$save_LDFLAGS"
+ else
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && $ECHO "X-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && $ECHO "X-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -exports_file $export_symbols -o $lib'
+ fi
+ _LT_TAGVAR(archive_cmds_need_lc, $1)='no'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+ _LT_TAGVAR(inherit_rpath, $1)=yes
+ _LT_TAGVAR(link_all_deplibs, $1)=yes
+ ;;
+
+ netbsd*)
+ if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then
+ _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags' # a.out
+ else
+ _LT_TAGVAR(archive_cmds, $1)='$LD -shared -o $lib $libobjs $deplibs $linker_flags' # ELF
+ fi
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir'
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ ;;
+
+ newsos6)
+ _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ ;;
+
+ *nto* | *qnx*)
+ ;;
+
+ openbsd*)
+ if test -f /usr/libexec/ld.so; then
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ _LT_TAGVAR(hardcode_direct_absolute, $1)=yes
+ if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags ${wl}-retain-symbols-file,$export_symbols'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir'
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E'
+ else
+ case $host_os in
+ openbsd[[01]].* | openbsd2.[[0-7]] | openbsd2.[[0-7]].*)
+ _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir'
+ ;;
+ *)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir'
+ ;;
+ esac
+ fi
+ else
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+ ;;
+
+ os2*)
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir'
+ _LT_TAGVAR(hardcode_minus_L, $1)=yes
+ _LT_TAGVAR(allow_undefined_flag, $1)=unsupported
+ _LT_TAGVAR(archive_cmds, $1)='$ECHO "LIBRARY $libname INITINSTANCE" > $output_objdir/$libname.def~$ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~$ECHO DATA >> $output_objdir/$libname.def~$ECHO " SINGLE NONSHARED" >> $output_objdir/$libname.def~$ECHO EXPORTS >> $output_objdir/$libname.def~emxexp $libobjs >> $output_objdir/$libname.def~$CC -Zdll -Zcrtdll -o $lib $libobjs $deplibs $compiler_flags $output_objdir/$libname.def'
+ _LT_TAGVAR(old_archive_from_new_cmds, $1)='emximp -o $output_objdir/$libname.a $output_objdir/$libname.def'
+ ;;
+
+ osf3*)
+ if test "$GCC" = yes; then
+ _LT_TAGVAR(allow_undefined_flag, $1)=' ${wl}-expect_unresolved ${wl}\*'
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && $ECHO "X${wl}-set_version ${wl}$verstring" | $Xsed` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib'
+ else
+ _LT_TAGVAR(allow_undefined_flag, $1)=' -expect_unresolved \*'
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && $ECHO "X-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -o $lib'
+ fi
+ _LT_TAGVAR(archive_cmds_need_lc, $1)='no'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+ ;;
+
+ osf4* | osf5*) # as osf3* with the addition of -msym flag
+ if test "$GCC" = yes; then
+ _LT_TAGVAR(allow_undefined_flag, $1)=' ${wl}-expect_unresolved ${wl}\*'
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags ${wl}-msym ${wl}-soname ${wl}$soname `test -n "$verstring" && $ECHO "X${wl}-set_version ${wl}$verstring" | $Xsed` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir'
+ else
+ _LT_TAGVAR(allow_undefined_flag, $1)=' -expect_unresolved \*'
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags -msym -soname $soname `test -n "$verstring" && $ECHO "X-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='for i in `cat $export_symbols`; do printf "%s %s\\n" -exported_symbol "\$i" >> $lib.exp; done; printf "%s\\n" "-hidden">> $lib.exp~
+ $CC -shared${allow_undefined_flag} ${wl}-input ${wl}$lib.exp $compiler_flags $libobjs $deplibs -soname $soname `test -n "$verstring" && $ECHO "X-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -o $lib~$RM $lib.exp'
+
+ # Both c and cxx compiler support -rpath directly
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-rpath $libdir'
+ fi
+ _LT_TAGVAR(archive_cmds_need_lc, $1)='no'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+ ;;
+
+ solaris*)
+ _LT_TAGVAR(no_undefined_flag, $1)=' -z defs'
+ if test "$GCC" = yes; then
+ wlarc='${wl}'
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared ${wl}-z ${wl}text ${wl}-h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~
+ $CC -shared ${wl}-z ${wl}text ${wl}-M ${wl}$lib.exp ${wl}-h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags~$RM $lib.exp'
+ else
+ case `$CC -V 2>&1` in
+ *"Compilers 5.0"*)
+ wlarc=''
+ _LT_TAGVAR(archive_cmds, $1)='$LD -G${allow_undefined_flag} -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~
+ $LD -G${allow_undefined_flag} -M $lib.exp -h $soname -o $lib $libobjs $deplibs $linker_flags~$RM $lib.exp'
+ ;;
+ *)
+ wlarc='${wl}'
+ _LT_TAGVAR(archive_cmds, $1)='$CC -G${allow_undefined_flag} -h $soname -o $lib $libobjs $deplibs $compiler_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~
+ $CC -G${allow_undefined_flag} -M $lib.exp -h $soname -o $lib $libobjs $deplibs $compiler_flags~$RM $lib.exp'
+ ;;
+ esac
+ fi
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir'
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ case $host_os in
+ solaris2.[[0-5]] | solaris2.[[0-5]].*) ;;
+ *)
+ # The compiler driver will combine and reorder linker options,
+ # but understands `-z linker_flag'. GCC discards it without `$wl',
+ # but is careful enough not to reorder.
+ # Supported since Solaris 2.6 (maybe 2.5.1?)
+ if test "$GCC" = yes; then
+ _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}-z ${wl}allextract$convenience ${wl}-z ${wl}defaultextract'
+ else
+ _LT_TAGVAR(whole_archive_flag_spec, $1)='-z allextract$convenience -z defaultextract'
+ fi
+ ;;
+ esac
+ _LT_TAGVAR(link_all_deplibs, $1)=yes
+ ;;
+
+ sunos4*)
+ if test "x$host_vendor" = xsequent; then
+ # Use $CC to link under sequent, because it throws in some extra .o
+ # files that make .init and .fini sections work.
+ _LT_TAGVAR(archive_cmds, $1)='$CC -G ${wl}-h $soname -o $lib $libobjs $deplibs $compiler_flags'
+ else
+ _LT_TAGVAR(archive_cmds, $1)='$LD -assert pure-text -Bstatic -o $lib $libobjs $deplibs $linker_flags'
+ fi
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir'
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_minus_L, $1)=yes
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ ;;
+
+ sysv4)
+ case $host_vendor in
+ sni)
+ _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ _LT_TAGVAR(hardcode_direct, $1)=yes # is this really true???
+ ;;
+ siemens)
+ ## LD is ld it makes a PLAMLIB
+ ## CC just makes a GrossModule.
+ _LT_TAGVAR(archive_cmds, $1)='$LD -G -o $lib $libobjs $deplibs $linker_flags'
+ _LT_TAGVAR(reload_cmds, $1)='$CC -r -o $output$reload_objs'
+ _LT_TAGVAR(hardcode_direct, $1)=no
+ ;;
+ motorola)
+ _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ _LT_TAGVAR(hardcode_direct, $1)=no #Motorola manual says yes, but my tests say they lie
+ ;;
+ esac
+ runpath_var='LD_RUN_PATH'
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ ;;
+
+ sysv4.3*)
+ _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='-Bexport'
+ ;;
+
+ sysv4*MP*)
+ if test -d /usr/nec; then
+ _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ runpath_var=LD_RUN_PATH
+ hardcode_runpath_var=yes
+ _LT_TAGVAR(ld_shlibs, $1)=yes
+ fi
+ ;;
+
+ sysv4*uw2* | sysv5OpenUNIX* | sysv5UnixWare7.[[01]].[[10]]* | unixware7* | sco3.2v5.0.[[024]]*)
+ _LT_TAGVAR(no_undefined_flag, $1)='${wl}-z,text'
+ _LT_TAGVAR(archive_cmds_need_lc, $1)=no
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ runpath_var='LD_RUN_PATH'
+
+ if test "$GCC" = yes; then
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ else
+ _LT_TAGVAR(archive_cmds, $1)='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ fi
+ ;;
+
+ sysv5* | sco3.2v5* | sco5v6*)
+ # Note: We can NOT use -z defs as we might desire, because we do not
+ # link with -lc, and that would cause any symbols used from libc to
+ # always be unresolved, which means just about no library would
+ # ever link correctly. If we're not using GNU ld we use -z text
+ # though, which does catch some bad symbols but isn't as heavy-handed
+ # as -z defs.
+ _LT_TAGVAR(no_undefined_flag, $1)='${wl}-z,text'
+ _LT_TAGVAR(allow_undefined_flag, $1)='${wl}-z,nodefs'
+ _LT_TAGVAR(archive_cmds_need_lc, $1)=no
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-R,$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=':'
+ _LT_TAGVAR(link_all_deplibs, $1)=yes
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-Bexport'
+ runpath_var='LD_RUN_PATH'
+
+ if test "$GCC" = yes; then
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ else
+ _LT_TAGVAR(archive_cmds, $1)='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ fi
+ ;;
+
+ uts4*)
+ _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir'
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ ;;
+
+ *)
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ esac
+
+ if test x$host_vendor = xsni; then
+ case $host in
+ sysv4 | sysv4.2uw2* | sysv4.3* | sysv5*)
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-Blargedynsym'
+ ;;
+ esac
+ fi
+ fi
+])
+AC_MSG_RESULT([$_LT_TAGVAR(ld_shlibs, $1)])
+test "$_LT_TAGVAR(ld_shlibs, $1)" = no && can_build_shared=no
+
+_LT_TAGVAR(with_gnu_ld, $1)=$with_gnu_ld
+
+_LT_DECL([], [libext], [0], [Old archive suffix (normally "a")])dnl
+_LT_DECL([], [shrext_cmds], [1], [Shared library suffix (normally ".so")])dnl
+_LT_DECL([], [extract_expsyms_cmds], [2],
+ [The commands to extract the exported symbol list from a shared archive])
+
+#
+# Do we need to explicitly link libc?
+#
+case "x$_LT_TAGVAR(archive_cmds_need_lc, $1)" in
+x|xyes)
+ # Assume -lc should be added
+ _LT_TAGVAR(archive_cmds_need_lc, $1)=yes
+
+ if test "$enable_shared" = yes && test "$GCC" = yes; then
+ case $_LT_TAGVAR(archive_cmds, $1) in
+ *'~'*)
+ # FIXME: we may have to deal with multi-command sequences.
+ ;;
+ '$CC '*)
+ # Test whether the compiler implicitly links with -lc since on some
+ # systems, -lgcc has to come before -lc. If gcc already passes -lc
+ # to ld, don't add -lc before -lgcc.
+ AC_MSG_CHECKING([whether -lc should be explicitly linked in])
+ $RM conftest*
+ echo "$lt_simple_compile_test_code" > conftest.$ac_ext
+
+ if AC_TRY_EVAL(ac_compile) 2>conftest.err; then
+ soname=conftest
+ lib=conftest
+ libobjs=conftest.$ac_objext
+ deplibs=
+ wl=$_LT_TAGVAR(lt_prog_compiler_wl, $1)
+ pic_flag=$_LT_TAGVAR(lt_prog_compiler_pic, $1)
+ compiler_flags=-v
+ linker_flags=-v
+ verstring=
+ output_objdir=.
+ libname=conftest
+ lt_save_allow_undefined_flag=$_LT_TAGVAR(allow_undefined_flag, $1)
+ _LT_TAGVAR(allow_undefined_flag, $1)=
+ if AC_TRY_EVAL(_LT_TAGVAR(archive_cmds, $1) 2\>\&1 \| $GREP \" -lc \" \>/dev/null 2\>\&1)
+ then
+ _LT_TAGVAR(archive_cmds_need_lc, $1)=no
+ else
+ _LT_TAGVAR(archive_cmds_need_lc, $1)=yes
+ fi
+ _LT_TAGVAR(allow_undefined_flag, $1)=$lt_save_allow_undefined_flag
+ else
+ cat conftest.err 1>&5
+ fi
+ $RM conftest*
+ AC_MSG_RESULT([$_LT_TAGVAR(archive_cmds_need_lc, $1)])
+ ;;
+ esac
+ fi
+ ;;
+esac
+
+_LT_TAGDECL([build_libtool_need_lc], [archive_cmds_need_lc], [0],
+ [Whether or not to add -lc for building shared libraries])
+_LT_TAGDECL([allow_libtool_libs_with_static_runtimes],
+ [enable_shared_with_static_runtimes], [0],
+ [Whether or not to disallow shared libs when runtime libs are static])
+_LT_TAGDECL([], [export_dynamic_flag_spec], [1],
+ [Compiler flag to allow reflexive dlopens])
+_LT_TAGDECL([], [whole_archive_flag_spec], [1],
+ [Compiler flag to generate shared objects directly from archives])
+_LT_TAGDECL([], [compiler_needs_object], [1],
+ [Whether the compiler copes with passing no objects directly])
+_LT_TAGDECL([], [old_archive_from_new_cmds], [2],
+ [Create an old-style archive from a shared archive])
+_LT_TAGDECL([], [old_archive_from_expsyms_cmds], [2],
+ [Create a temporary old-style archive to link instead of a shared archive])
+_LT_TAGDECL([], [archive_cmds], [2], [Commands used to build a shared archive])
+_LT_TAGDECL([], [archive_expsym_cmds], [2])
+_LT_TAGDECL([], [module_cmds], [2],
+ [Commands used to build a loadable module if different from building
+ a shared archive.])
+_LT_TAGDECL([], [module_expsym_cmds], [2])
+_LT_TAGDECL([], [with_gnu_ld], [1],
+ [Whether we are building with GNU ld or not])
+_LT_TAGDECL([], [allow_undefined_flag], [1],
+ [Flag that allows shared libraries with undefined symbols to be built])
+_LT_TAGDECL([], [no_undefined_flag], [1],
+ [Flag that enforces no undefined symbols])
+_LT_TAGDECL([], [hardcode_libdir_flag_spec], [1],
+ [Flag to hardcode $libdir into a binary during linking.
+ This must work even if $libdir does not exist])
+_LT_TAGDECL([], [hardcode_libdir_flag_spec_ld], [1],
+ [[If ld is used when linking, flag to hardcode $libdir into a binary
+ during linking. This must work even if $libdir does not exist]])
+_LT_TAGDECL([], [hardcode_libdir_separator], [1],
+ [Whether we need a single "-rpath" flag with a separated argument])
+_LT_TAGDECL([], [hardcode_direct], [0],
+ [Set to "yes" if using DIR/libNAME${shared_ext} during linking hardcodes
+ DIR into the resulting binary])
+_LT_TAGDECL([], [hardcode_direct_absolute], [0],
+ [Set to "yes" if using DIR/libNAME${shared_ext} during linking hardcodes
+ DIR into the resulting binary and the resulting library dependency is
+ "absolute", i.e impossible to change by setting ${shlibpath_var} if the
+ library is relocated])
+_LT_TAGDECL([], [hardcode_minus_L], [0],
+ [Set to "yes" if using the -LDIR flag during linking hardcodes DIR
+ into the resulting binary])
+_LT_TAGDECL([], [hardcode_shlibpath_var], [0],
+ [Set to "yes" if using SHLIBPATH_VAR=DIR during linking hardcodes DIR
+ into the resulting binary])
+_LT_TAGDECL([], [hardcode_automatic], [0],
+ [Set to "yes" if building a shared library automatically hardcodes DIR
+ into the library and all subsequent libraries and executables linked
+ against it])
+_LT_TAGDECL([], [inherit_rpath], [0],
+ [Set to yes if linker adds runtime paths of dependent libraries
+ to runtime path list])
+_LT_TAGDECL([], [link_all_deplibs], [0],
+ [Whether libtool must link a program against all its dependency libraries])
+_LT_TAGDECL([], [fix_srcfile_path], [1],
+ [Fix the shell variable $srcfile for the compiler])
+_LT_TAGDECL([], [always_export_symbols], [0],
+ [Set to "yes" if exported symbols are required])
+_LT_TAGDECL([], [export_symbols_cmds], [2],
+ [The commands to list exported symbols])
+_LT_TAGDECL([], [exclude_expsyms], [1],
+ [Symbols that should not be listed in the preloaded symbols])
+_LT_TAGDECL([], [include_expsyms], [1],
+ [Symbols that must always be exported])
+_LT_TAGDECL([], [prelink_cmds], [2],
+ [Commands necessary for linking programs (against libraries) with templates])
+_LT_TAGDECL([], [file_list_spec], [1],
+ [Specify filename containing input files])
+dnl FIXME: Not yet implemented
+dnl _LT_TAGDECL([], [thread_safe_flag_spec], [1],
+dnl [Compiler flag to generate thread safe objects])
+])# _LT_LINKER_SHLIBS
+
+
+# _LT_LANG_C_CONFIG([TAG])
+# ------------------------
+# Ensure that the configuration variables for a C compiler are suitably
+# defined. These variables are subsequently used by _LT_CONFIG to write
+# the compiler configuration to `libtool'.
+m4_defun([_LT_LANG_C_CONFIG],
+[m4_require([_LT_DECL_EGREP])dnl
+lt_save_CC="$CC"
+AC_LANG_PUSH(C)
+
+# Source file extension for C test sources.
+ac_ext=c
+
+# Object file extension for compiled C test sources.
+objext=o
+_LT_TAGVAR(objext, $1)=$objext
+
+# Code to be used in simple compile tests
+lt_simple_compile_test_code="int some_variable = 0;"
+
+# Code to be used in simple link tests
+lt_simple_link_test_code='int main(){return(0);}'
+
+_LT_TAG_COMPILER
+# Save the default compiler, since it gets overwritten when the other
+# tags are being tested, and _LT_TAGVAR(compiler, []) is a NOP.
+compiler_DEFAULT=$CC
+
+# save warnings/boilerplate of simple test code
+_LT_COMPILER_BOILERPLATE
+_LT_LINKER_BOILERPLATE
+
+## CAVEAT EMPTOR:
+## There is no encapsulation within the following macros, do not change
+## the running order or otherwise move them around unless you know exactly
+## what you are doing...
+if test -n "$compiler"; then
+ _LT_COMPILER_NO_RTTI($1)
+ _LT_COMPILER_PIC($1)
+ _LT_COMPILER_C_O($1)
+ _LT_COMPILER_FILE_LOCKS($1)
+ _LT_LINKER_SHLIBS($1)
+ _LT_SYS_DYNAMIC_LINKER($1)
+ _LT_LINKER_HARDCODE_LIBPATH($1)
+ LT_SYS_DLOPEN_SELF
+ _LT_CMD_STRIPLIB
+
+ # Report which library types will actually be built
+ AC_MSG_CHECKING([if libtool supports shared libraries])
+ AC_MSG_RESULT([$can_build_shared])
+
+ AC_MSG_CHECKING([whether to build shared libraries])
+ test "$can_build_shared" = "no" && enable_shared=no
+
+ # On AIX, shared libraries and static libraries use the same namespace, and
+ # are all built from PIC.
+ case $host_os in
+ aix3*)
+ test "$enable_shared" = yes && enable_static=no
+ if test -n "$RANLIB"; then
+ archive_cmds="$archive_cmds~\$RANLIB \$lib"
+ postinstall_cmds='$RANLIB $lib'
+ fi
+ ;;
+
+ aix[[4-9]]*)
+ if test "$host_cpu" != ia64 && test "$aix_use_runtimelinking" = no ; then
+ test "$enable_shared" = yes && enable_static=no
+ fi
+ ;;
+ esac
+ AC_MSG_RESULT([$enable_shared])
+
+ AC_MSG_CHECKING([whether to build static libraries])
+ # Make sure either enable_shared or enable_static is yes.
+ test "$enable_shared" = yes || enable_static=yes
+ AC_MSG_RESULT([$enable_static])
+
+ _LT_CONFIG($1)
+fi
+AC_LANG_POP
+CC="$lt_save_CC"
+])# _LT_LANG_C_CONFIG
+
+
+# _LT_PROG_CXX
+# ------------
+# Since AC_PROG_CXX is broken, in that it returns g++ if there is no c++
+# compiler, we have our own version here.
+m4_defun([_LT_PROG_CXX],
+[
+pushdef([AC_MSG_ERROR], [_lt_caught_CXX_error=yes])
+AC_PROG_CXX
+if test -n "$CXX" && ( test "X$CXX" != "Xno" &&
+ ( (test "X$CXX" = "Xg++" && `g++ -v >/dev/null 2>&1` ) ||
+ (test "X$CXX" != "Xg++"))) ; then
+ AC_PROG_CXXCPP
+else
+ _lt_caught_CXX_error=yes
+fi
+popdef([AC_MSG_ERROR])
+])# _LT_PROG_CXX
+
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([_LT_PROG_CXX], [])
+
+
+# _LT_LANG_CXX_CONFIG([TAG])
+# --------------------------
+# Ensure that the configuration variables for a C++ compiler are suitably
+# defined. These variables are subsequently used by _LT_CONFIG to write
+# the compiler configuration to `libtool'.
+m4_defun([_LT_LANG_CXX_CONFIG],
+[AC_REQUIRE([_LT_PROG_CXX])dnl
+m4_require([_LT_FILEUTILS_DEFAULTS])dnl
+m4_require([_LT_DECL_EGREP])dnl
+
+AC_LANG_PUSH(C++)
+_LT_TAGVAR(archive_cmds_need_lc, $1)=no
+_LT_TAGVAR(allow_undefined_flag, $1)=
+_LT_TAGVAR(always_export_symbols, $1)=no
+_LT_TAGVAR(archive_expsym_cmds, $1)=
+_LT_TAGVAR(compiler_needs_object, $1)=no
+_LT_TAGVAR(export_dynamic_flag_spec, $1)=
+_LT_TAGVAR(hardcode_direct, $1)=no
+_LT_TAGVAR(hardcode_direct_absolute, $1)=no
+_LT_TAGVAR(hardcode_libdir_flag_spec, $1)=
+_LT_TAGVAR(hardcode_libdir_flag_spec_ld, $1)=
+_LT_TAGVAR(hardcode_libdir_separator, $1)=
+_LT_TAGVAR(hardcode_minus_L, $1)=no
+_LT_TAGVAR(hardcode_shlibpath_var, $1)=unsupported
+_LT_TAGVAR(hardcode_automatic, $1)=no
+_LT_TAGVAR(inherit_rpath, $1)=no
+_LT_TAGVAR(module_cmds, $1)=
+_LT_TAGVAR(module_expsym_cmds, $1)=
+_LT_TAGVAR(link_all_deplibs, $1)=unknown
+_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds
+_LT_TAGVAR(no_undefined_flag, $1)=
+_LT_TAGVAR(whole_archive_flag_spec, $1)=
+_LT_TAGVAR(enable_shared_with_static_runtimes, $1)=no
+
+# Source file extension for C++ test sources.
+ac_ext=cpp
+
+# Object file extension for compiled C++ test sources.
+objext=o
+_LT_TAGVAR(objext, $1)=$objext
+
+# No sense in running all these tests if we already determined that
+# the CXX compiler isn't working. Some variables (like enable_shared)
+# are currently assumed to apply to all compilers on this platform,
+# and will be corrupted by setting them based on a non-working compiler.
+if test "$_lt_caught_CXX_error" != yes; then
+ # Code to be used in simple compile tests
+ lt_simple_compile_test_code="int some_variable = 0;"
+
+ # Code to be used in simple link tests
+ lt_simple_link_test_code='int main(int, char *[[]]) { return(0); }'
+
+ # ltmain only uses $CC for tagged configurations so make sure $CC is set.
+ _LT_TAG_COMPILER
+
+ # save warnings/boilerplate of simple test code
+ _LT_COMPILER_BOILERPLATE
+ _LT_LINKER_BOILERPLATE
+
+ # Allow CC to be a program name with arguments.
+ lt_save_CC=$CC
+ lt_save_LD=$LD
+ lt_save_GCC=$GCC
+ GCC=$GXX
+ lt_save_with_gnu_ld=$with_gnu_ld
+ lt_save_path_LD=$lt_cv_path_LD
+ if test -n "${lt_cv_prog_gnu_ldcxx+set}"; then
+ lt_cv_prog_gnu_ld=$lt_cv_prog_gnu_ldcxx
+ else
+ $as_unset lt_cv_prog_gnu_ld
+ fi
+ if test -n "${lt_cv_path_LDCXX+set}"; then
+ lt_cv_path_LD=$lt_cv_path_LDCXX
+ else
+ $as_unset lt_cv_path_LD
+ fi
+ test -z "${LDCXX+set}" || LD=$LDCXX
+ CC=${CXX-"c++"}
+ compiler=$CC
+ _LT_TAGVAR(compiler, $1)=$CC
+ _LT_CC_BASENAME([$compiler])
+
+ if test -n "$compiler"; then
+ # We don't want -fno-exception when compiling C++ code, so set the
+ # no_builtin_flag separately
+ if test "$GXX" = yes; then
+ _LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)=' -fno-builtin'
+ else
+ _LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)=
+ fi
+
+ if test "$GXX" = yes; then
+ # Set up default GNU C++ configuration
+
+ LT_PATH_LD
+
+ # Check if GNU C++ uses GNU ld as the underlying linker, since the
+ # archiving commands below assume that GNU ld is being used.
+ if test "$with_gnu_ld" = yes; then
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib'
+
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir'
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-dynamic'
+
+ # If archive_cmds runs LD, not CC, wlarc should be empty
+ # XXX I think wlarc can be eliminated in ltcf-cxx, but I need to
+ # investigate it a little bit more. (MM)
+ wlarc='${wl}'
+
+ # ancient GNU ld didn't support --whole-archive et. al.
+ if eval "`$CC -print-prog-name=ld` --help 2>&1" |
+ $GREP 'no-whole-archive' > /dev/null; then
+ _LT_TAGVAR(whole_archive_flag_spec, $1)="$wlarc"'--whole-archive$convenience '"$wlarc"'--no-whole-archive'
+ else
+ _LT_TAGVAR(whole_archive_flag_spec, $1)=
+ fi
+ else
+ with_gnu_ld=no
+ wlarc=
+
+ # A generic and very simple default shared library creation
+ # command for GNU C++ for the case where it uses the native
+ # linker, instead of GNU ld. If possible, this setting should
+ # overridden to take advantage of the native linker features on
+ # the platform it is being used on.
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $lib'
+ fi
+
+ # Commands to make compiler produce verbose output that lists
+ # what "hidden" libraries, object files and flags are used when
+ # linking a shared library.
+ output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP "\-L"'
+
+ else
+ GXX=no
+ with_gnu_ld=no
+ wlarc=
+ fi
+
+ # PORTME: fill in a description of your system's C++ link characteristics
+ AC_MSG_CHECKING([whether the $compiler linker ($LD) supports shared libraries])
+ _LT_TAGVAR(ld_shlibs, $1)=yes
+ case $host_os in
+ aix3*)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ aix[[4-9]]*)
+ if test "$host_cpu" = ia64; then
+ # On IA64, the linker does run time linking by default, so we don't
+ # have to do anything special.
+ aix_use_runtimelinking=no
+ exp_sym_flag='-Bexport'
+ no_entry_flag=""
+ else
+ aix_use_runtimelinking=no
+
+ # Test if we are trying to use run time linking or normal
+ # AIX style linking. If -brtl is somewhere in LDFLAGS, we
+ # need to do runtime linking.
+ case $host_os in aix4.[[23]]|aix4.[[23]].*|aix[[5-9]]*)
+ for ld_flag in $LDFLAGS; do
+ case $ld_flag in
+ *-brtl*)
+ aix_use_runtimelinking=yes
+ break
+ ;;
+ esac
+ done
+ ;;
+ esac
+
+ exp_sym_flag='-bexport'
+ no_entry_flag='-bnoentry'
+ fi
+
+ # When large executables or shared objects are built, AIX ld can
+ # have problems creating the table of contents. If linking a library
+ # or program results in "error TOC overflow" add -mminimal-toc to
+ # CXXFLAGS/CFLAGS for g++/gcc. In the cases where that is not
+ # enough to fix the problem, add -Wl,-bbigtoc to LDFLAGS.
+
+ _LT_TAGVAR(archive_cmds, $1)=''
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_direct_absolute, $1)=yes
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=':'
+ _LT_TAGVAR(link_all_deplibs, $1)=yes
+ _LT_TAGVAR(file_list_spec, $1)='${wl}-f,'
+
+ if test "$GXX" = yes; then
+ case $host_os in aix4.[[012]]|aix4.[[012]].*)
+ # We only want to do this on AIX 4.2 and lower, the check
+ # below for broken collect2 doesn't work under 4.3+
+ collect2name=`${CC} -print-prog-name=collect2`
+ if test -f "$collect2name" &&
+ strings "$collect2name" | $GREP resolve_lib_name >/dev/null
+ then
+ # We have reworked collect2
+ :
+ else
+ # We have old collect2
+ _LT_TAGVAR(hardcode_direct, $1)=unsupported
+ # It fails to find uninstalled libraries when the uninstalled
+ # path is not listed in the libpath. Setting hardcode_minus_L
+ # to unsupported forces relinking
+ _LT_TAGVAR(hardcode_minus_L, $1)=yes
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=
+ fi
+ esac
+ shared_flag='-shared'
+ if test "$aix_use_runtimelinking" = yes; then
+ shared_flag="$shared_flag "'${wl}-G'
+ fi
+ else
+ # not using gcc
+ if test "$host_cpu" = ia64; then
+ # VisualAge C++, Version 5.5 for AIX 5L for IA-64, Beta 3 Release
+ # chokes on -Wl,-G. The following line is correct:
+ shared_flag='-G'
+ else
+ if test "$aix_use_runtimelinking" = yes; then
+ shared_flag='${wl}-G'
+ else
+ shared_flag='${wl}-bM:SRE'
+ fi
+ fi
+ fi
+
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-bexpall'
+ # It seems that -bexpall does not export symbols beginning with
+ # underscore (_), so it is better to generate a list of symbols to
+ # export.
+ _LT_TAGVAR(always_export_symbols, $1)=yes
+ if test "$aix_use_runtimelinking" = yes; then
+ # Warning - without using the other runtime loading flags (-brtl),
+ # -berok will link without error, but may produce a broken library.
+ _LT_TAGVAR(allow_undefined_flag, $1)='-berok'
+ # Determine the default libpath from the value encoded in an empty
+ # executable.
+ _LT_SYS_MODULE_PATH_AIX
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-blibpath:$libdir:'"$aix_libpath"
+
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags `if test "x${allow_undefined_flag}" != "x"; then $ECHO "X${wl}${allow_undefined_flag}" | $Xsed; else :; fi` '"\${wl}$exp_sym_flag:\$export_symbols $shared_flag"
+ else
+ if test "$host_cpu" = ia64; then
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-R $libdir:/usr/lib:/lib'
+ _LT_TAGVAR(allow_undefined_flag, $1)="-z nodefs"
+ _LT_TAGVAR(archive_expsym_cmds, $1)="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags ${wl}${allow_undefined_flag} '"\${wl}$exp_sym_flag:\$export_symbols"
+ else
+ # Determine the default libpath from the value encoded in an
+ # empty executable.
+ _LT_SYS_MODULE_PATH_AIX
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-blibpath:$libdir:'"$aix_libpath"
+ # Warning - without using the other run time loading flags,
+ # -berok will link without error, but may produce a broken library.
+ _LT_TAGVAR(no_undefined_flag, $1)=' ${wl}-bernotok'
+ _LT_TAGVAR(allow_undefined_flag, $1)=' ${wl}-berok'
+ # Exported symbols can be pulled into shared objects from archives
+ _LT_TAGVAR(whole_archive_flag_spec, $1)='$convenience'
+ _LT_TAGVAR(archive_cmds_need_lc, $1)=yes
+ # This is similar to how AIX traditionally builds its shared
+ # libraries.
+ _LT_TAGVAR(archive_expsym_cmds, $1)="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs ${wl}-bnoentry $compiler_flags ${wl}-bE:$export_symbols${allow_undefined_flag}~$AR $AR_FLAGS $output_objdir/$libname$release.a $output_objdir/$soname'
+ fi
+ fi
+ ;;
+
+ beos*)
+ if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then
+ _LT_TAGVAR(allow_undefined_flag, $1)=unsupported
+ # Joseph Beckenbach <jrb3 at best.com> says some releases of gcc
+ # support --undefined. This deserves some investigation. FIXME
+ _LT_TAGVAR(archive_cmds, $1)='$CC -nostart $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ else
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+ ;;
+
+ chorus*)
+ case $cc_basename in
+ *)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ esac
+ ;;
+
+ cygwin* | mingw* | pw32* | cegcc*)
+ # _LT_TAGVAR(hardcode_libdir_flag_spec, $1) is actually meaningless,
+ # as there is no search path for DLLs.
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir'
+ _LT_TAGVAR(allow_undefined_flag, $1)=unsupported
+ _LT_TAGVAR(always_export_symbols, $1)=no
+ _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes
+
+ if $LD --help 2>&1 | $GREP 'auto-import' > /dev/null; then
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib'
+ # If the export-symbols file already is a .def file (1st line
+ # is EXPORTS), use it as is; otherwise, prepend...
+ _LT_TAGVAR(archive_expsym_cmds, $1)='if test "x`$SED 1q $export_symbols`" = xEXPORTS; then
+ cp $export_symbols $output_objdir/$soname.def;
+ else
+ echo EXPORTS > $output_objdir/$soname.def;
+ cat $export_symbols >> $output_objdir/$soname.def;
+ fi~
+ $CC -shared -nostdlib $output_objdir/$soname.def $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib'
+ else
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+ ;;
+ darwin* | rhapsody*)
+ _LT_DARWIN_LINKER_FEATURES($1)
+ ;;
+
+ dgux*)
+ case $cc_basename in
+ ec++*)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ ghcx*)
+ # Green Hills C++ Compiler
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ *)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ esac
+ ;;
+
+ freebsd[[12]]*)
+ # C++ shared libraries reported to be fairly broken before
+ # switch to ELF
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+
+ freebsd-elf*)
+ _LT_TAGVAR(archive_cmds_need_lc, $1)=no
+ ;;
+
+ freebsd* | dragonfly*)
+ # FreeBSD 3 and later use GNU C++ and GNU ld with standard ELF
+ # conventions
+ _LT_TAGVAR(ld_shlibs, $1)=yes
+ ;;
+
+ gnu*)
+ ;;
+
+ hpux9*)
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}+b ${wl}$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E'
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_minus_L, $1)=yes # Not in the search PATH,
+ # but as the default
+ # location of the library.
+
+ case $cc_basename in
+ CC*)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ aCC*)
+ _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/$soname~$CC -b ${wl}+b ${wl}$install_libdir -o $output_objdir/$soname $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib'
+ # Commands to make compiler produce verbose output that lists
+ # what "hidden" libraries, object files and flags are used when
+ # linking a shared library.
+ #
+ # There doesn't appear to be a way to prevent this compiler from
+ # explicitly linking system object files so we need to strip them
+ # from the output so that they don't get included in the library
+ # dependencies.
+ output_verbose_link_cmd='templist=`($CC -b $CFLAGS -v conftest.$objext 2>&1) | $EGREP "\-L"`; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; $ECHO "X$list" | $Xsed'
+ ;;
+ *)
+ if test "$GXX" = yes; then
+ _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/$soname~$CC -shared -nostdlib -fPIC ${wl}+b ${wl}$install_libdir -o $output_objdir/$soname $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib'
+ else
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+ ;;
+ esac
+ ;;
+
+ hpux10*|hpux11*)
+ if test $with_gnu_ld = no; then
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}+b ${wl}$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+
+ case $host_cpu in
+ hppa*64*|ia64*)
+ ;;
+ *)
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E'
+ ;;
+ esac
+ fi
+ case $host_cpu in
+ hppa*64*|ia64*)
+ _LT_TAGVAR(hardcode_direct, $1)=no
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ ;;
+ *)
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_direct_absolute, $1)=yes
+ _LT_TAGVAR(hardcode_minus_L, $1)=yes # Not in the search PATH,
+ # but as the default
+ # location of the library.
+ ;;
+ esac
+
+ case $cc_basename in
+ CC*)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ aCC*)
+ case $host_cpu in
+ hppa*64*)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -b ${wl}+h ${wl}$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags'
+ ;;
+ ia64*)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -b ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags'
+ ;;
+ *)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -b ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags'
+ ;;
+ esac
+ # Commands to make compiler produce verbose output that lists
+ # what "hidden" libraries, object files and flags are used when
+ # linking a shared library.
+ #
+ # There doesn't appear to be a way to prevent this compiler from
+ # explicitly linking system object files so we need to strip them
+ # from the output so that they don't get included in the library
+ # dependencies.
+ output_verbose_link_cmd='templist=`($CC -b $CFLAGS -v conftest.$objext 2>&1) | $GREP "\-L"`; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; $ECHO "X$list" | $Xsed'
+ ;;
+ *)
+ if test "$GXX" = yes; then
+ if test $with_gnu_ld = no; then
+ case $host_cpu in
+ hppa*64*)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib -fPIC ${wl}+h ${wl}$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags'
+ ;;
+ ia64*)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib -fPIC ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags'
+ ;;
+ *)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib -fPIC ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags'
+ ;;
+ esac
+ fi
+ else
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+ ;;
+ esac
+ ;;
+
+ interix[[3-9]]*)
+ _LT_TAGVAR(hardcode_direct, $1)=no
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir'
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E'
+ # Hack: On Interix 3.x, we cannot compile PIC because of a broken gcc.
+ # Instead, shared libraries are loaded at an image base (0x10000000 by
+ # default) and relocated if they conflict, which is a slow very memory
+ # consuming and fragmenting process. To avoid this, we pick a random,
+ # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link
+ # time. Moving up from 0x10000000 also allows more sbrk(2) space.
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='sed "s,^,_," $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--retain-symbols-file,$output_objdir/$soname.expsym ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib'
+ ;;
+ irix5* | irix6*)
+ case $cc_basename in
+ CC*)
+ # SGI C++
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -all -multigot $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -soname $soname `test -n "$verstring" && $ECHO "X-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -o $lib'
+
+ # Archives containing C++ object files must be created using
+ # "CC -ar", where "CC" is the IRIX C++ compiler. This is
+ # necessary to make sure instantiated templates are included
+ # in the archive.
+ _LT_TAGVAR(old_archive_cmds, $1)='$CC -ar -WR,-u -o $oldlib $oldobjs'
+ ;;
+ *)
+ if test "$GXX" = yes; then
+ if test "$with_gnu_ld" = no; then
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && $ECHO "X${wl}-set_version ${wl}$verstring" | $Xsed` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib'
+ else
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && $ECHO "X${wl}-set_version ${wl}$verstring" | $Xsed` -o $lib'
+ fi
+ fi
+ _LT_TAGVAR(link_all_deplibs, $1)=yes
+ ;;
+ esac
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+ _LT_TAGVAR(inherit_rpath, $1)=yes
+ ;;
+
+ linux* | k*bsd*-gnu)
+ case $cc_basename in
+ KCC*)
+ # Kuck and Associates, Inc. (KAI) C++ Compiler
+
+ # KCC will only create a shared library if the output file
+ # ends with ".so" (or ".sl" for HP-UX), so rename the library
+ # to its proper name (with version) after linking.
+ _LT_TAGVAR(archive_cmds, $1)='tempext=`echo $shared_ext | $SED -e '\''s/\([[^()0-9A-Za-z{}]]\)/\\\\\1/g'\''`; templib=`echo $lib | $SED -e "s/\${tempext}\..*/.so/"`; $CC $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags --soname $soname -o \$templib; mv \$templib $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='tempext=`echo $shared_ext | $SED -e '\''s/\([[^()0-9A-Za-z{}]]\)/\\\\\1/g'\''`; templib=`echo $lib | $SED -e "s/\${tempext}\..*/.so/"`; $CC $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags --soname $soname -o \$templib ${wl}-retain-symbols-file,$export_symbols; mv \$templib $lib'
+ # Commands to make compiler produce verbose output that lists
+ # what "hidden" libraries, object files and flags are used when
+ # linking a shared library.
+ #
+ # There doesn't appear to be a way to prevent this compiler from
+ # explicitly linking system object files so we need to strip them
+ # from the output so that they don't get included in the library
+ # dependencies.
+ output_verbose_link_cmd='templist=`$CC $CFLAGS -v conftest.$objext -o libconftest$shared_ext 2>&1 | $GREP "ld"`; rm -f libconftest$shared_ext; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; $ECHO "X$list" | $Xsed'
+
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir'
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-dynamic'
+
+ # Archives containing C++ object files must be created using
+ # "CC -Bstatic", where "CC" is the KAI C++ compiler.
+ _LT_TAGVAR(old_archive_cmds, $1)='$CC -Bstatic -o $oldlib $oldobjs'
+ ;;
+ icpc* | ecpc* )
+ # Intel C++
+ with_gnu_ld=yes
+ # version 8.0 and above of icpc choke on multiply defined symbols
+ # if we add $predep_objects and $postdep_objects, however 7.1 and
+ # earlier do not add the objects themselves.
+ case `$CC -V 2>&1` in
+ *"Version 7."*)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib'
+ ;;
+ *) # Version 8.0 or newer
+ tmp_idyn=
+ case $host_cpu in
+ ia64*) tmp_idyn=' -i_dynamic';;
+ esac
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared'"$tmp_idyn"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared'"$tmp_idyn"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib'
+ ;;
+ esac
+ _LT_TAGVAR(archive_cmds_need_lc, $1)=no
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir'
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-dynamic'
+ _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive$convenience ${wl}--no-whole-archive'
+ ;;
+ pgCC* | pgcpp*)
+ # Portland Group C++ compiler
+ case `$CC -V` in
+ *pgCC\ [[1-5]]* | *pgcpp\ [[1-5]]*)
+ _LT_TAGVAR(prelink_cmds, $1)='tpldir=Template.dir~
+ rm -rf $tpldir~
+ $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~
+ compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"'
+ _LT_TAGVAR(old_archive_cmds, $1)='tpldir=Template.dir~
+ rm -rf $tpldir~
+ $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~
+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~
+ $RANLIB $oldlib'
+ _LT_TAGVAR(archive_cmds, $1)='tpldir=Template.dir~
+ rm -rf $tpldir~
+ $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~
+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='tpldir=Template.dir~
+ rm -rf $tpldir~
+ $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~
+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib'
+ ;;
+ *) # Version 6 will use weak symbols
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib'
+ ;;
+ esac
+
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}--rpath ${wl}$libdir'
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-dynamic'
+ _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; $ECHO \"$new_convenience\"` ${wl}--no-whole-archive'
+ ;;
+ cxx*)
+ # Compaq C++
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $wl$soname -o $lib ${wl}-retain-symbols-file $wl$export_symbols'
+
+ runpath_var=LD_RUN_PATH
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-rpath $libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+
+ # Commands to make compiler produce verbose output that lists
+ # what "hidden" libraries, object files and flags are used when
+ # linking a shared library.
+ #
+ # There doesn't appear to be a way to prevent this compiler from
+ # explicitly linking system object files so we need to strip them
+ # from the output so that they don't get included in the library
+ # dependencies.
+ output_verbose_link_cmd='templist=`$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP "ld"`; templist=`$ECHO "X$templist" | $Xsed -e "s/\(^.*ld.*\)\( .*ld .*$\)/\1/"`; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; $ECHO "X$list" | $Xsed'
+ ;;
+ xl*)
+ # IBM XL 8.0 on PPC, with GNU ld
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir'
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}--export-dynamic'
+ _LT_TAGVAR(archive_cmds, $1)='$CC -qmkshrobj $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ if test "x$supports_anon_versioning" = xyes; then
+ _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $output_objdir/$libname.ver~
+ cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~
+ echo "local: *; };" >> $output_objdir/$libname.ver~
+ $CC -qmkshrobj $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-version-script ${wl}$output_objdir/$libname.ver -o $lib'
+ fi
+ ;;
+ *)
+ case `$CC -V 2>&1 | sed 5q` in
+ *Sun\ C*)
+ # Sun C++ 5.9
+ _LT_TAGVAR(no_undefined_flag, $1)=' -zdefs'
+ _LT_TAGVAR(archive_cmds, $1)='$CC -G${allow_undefined_flag} -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G${allow_undefined_flag} -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-retain-symbols-file ${wl}$export_symbols'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir'
+ _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}--whole-archive`new_convenience=; for conv in $convenience\"\"; do test -z \"$conv\" || new_convenience=\"$new_convenience,$conv\"; done; $ECHO \"$new_convenience\"` ${wl}--no-whole-archive'
+ _LT_TAGVAR(compiler_needs_object, $1)=yes
+
+ # Not sure whether something based on
+ # $CC $CFLAGS -v conftest.$objext -o libconftest$shared_ext 2>&1
+ # would be better.
+ output_verbose_link_cmd='echo'
+
+ # Archives containing C++ object files must be created using
+ # "CC -xar", where "CC" is the Sun C++ compiler. This is
+ # necessary to make sure instantiated templates are included
+ # in the archive.
+ _LT_TAGVAR(old_archive_cmds, $1)='$CC -xar -o $oldlib $oldobjs'
+ ;;
+ esac
+ ;;
+ esac
+ ;;
+
+ lynxos*)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+
+ m88k*)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+
+ mvs*)
+ case $cc_basename in
+ cxx*)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ *)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ esac
+ ;;
+
+ netbsd*)
+ if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then
+ _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $predep_objects $libobjs $deplibs $postdep_objects $linker_flags'
+ wlarc=
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir'
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ fi
+ # Workaround some broken pre-1.5 toolchains
+ output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP conftest.$objext | $SED -e "s:-lgcc -lc -lgcc::"'
+ ;;
+
+ *nto* | *qnx*)
+ _LT_TAGVAR(ld_shlibs, $1)=yes
+ ;;
+
+ openbsd2*)
+ # C++ shared libraries are fairly broken
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+
+ openbsd*)
+ if test -f /usr/libexec/ld.so; then
+ _LT_TAGVAR(hardcode_direct, $1)=yes
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ _LT_TAGVAR(hardcode_direct_absolute, $1)=yes
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $lib'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir'
+ if test -z "`echo __ELF__ | $CC -E - | grep __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-retain-symbols-file,$export_symbols -o $lib'
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-E'
+ _LT_TAGVAR(whole_archive_flag_spec, $1)="$wlarc"'--whole-archive$convenience '"$wlarc"'--no-whole-archive'
+ fi
+ output_verbose_link_cmd=echo
+ else
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+ ;;
+
+ osf3* | osf4* | osf5*)
+ case $cc_basename in
+ KCC*)
+ # Kuck and Associates, Inc. (KAI) C++ Compiler
+
+ # KCC will only create a shared library if the output file
+ # ends with ".so" (or ".sl" for HP-UX), so rename the library
+ # to its proper name (with version) after linking.
+ _LT_TAGVAR(archive_cmds, $1)='tempext=`echo $shared_ext | $SED -e '\''s/\([[^()0-9A-Za-z{}]]\)/\\\\\1/g'\''`; templib=`echo "$lib" | $SED -e "s/\${tempext}\..*/.so/"`; $CC $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags --soname $soname -o \$templib; mv \$templib $lib'
+
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath,$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+
+ # Archives containing C++ object files must be created using
+ # the KAI C++ compiler.
+ case $host in
+ osf3*) _LT_TAGVAR(old_archive_cmds, $1)='$CC -Bstatic -o $oldlib $oldobjs' ;;
+ *) _LT_TAGVAR(old_archive_cmds, $1)='$CC -o $oldlib $oldobjs' ;;
+ esac
+ ;;
+ RCC*)
+ # Rational C++ 2.4.1
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ cxx*)
+ case $host in
+ osf3*)
+ _LT_TAGVAR(allow_undefined_flag, $1)=' ${wl}-expect_unresolved ${wl}\*'
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared${allow_undefined_flag} $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname $soname `test -n "$verstring" && $ECHO "X${wl}-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -o $lib'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir'
+ ;;
+ *)
+ _LT_TAGVAR(allow_undefined_flag, $1)=' -expect_unresolved \*'
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared${allow_undefined_flag} $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -msym -soname $soname `test -n "$verstring" && $ECHO "X-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='for i in `cat $export_symbols`; do printf "%s %s\\n" -exported_symbol "\$i" >> $lib.exp; done~
+ echo "-hidden">> $lib.exp~
+ $CC -shared$allow_undefined_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -msym -soname $soname ${wl}-input ${wl}$lib.exp `test -n "$verstring" && $ECHO "X-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -o $lib~
+ $RM $lib.exp'
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-rpath $libdir'
+ ;;
+ esac
+
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+
+ # Commands to make compiler produce verbose output that lists
+ # what "hidden" libraries, object files and flags are used when
+ # linking a shared library.
+ #
+ # There doesn't appear to be a way to prevent this compiler from
+ # explicitly linking system object files so we need to strip them
+ # from the output so that they don't get included in the library
+ # dependencies.
+ output_verbose_link_cmd='templist=`$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP "ld" | $GREP -v "ld:"`; templist=`$ECHO "X$templist" | $Xsed -e "s/\(^.*ld.*\)\( .*ld.*$\)/\1/"`; list=""; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; $ECHO "X$list" | $Xsed'
+ ;;
+ *)
+ if test "$GXX" = yes && test "$with_gnu_ld" = no; then
+ _LT_TAGVAR(allow_undefined_flag, $1)=' ${wl}-expect_unresolved ${wl}\*'
+ case $host in
+ osf3*)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib ${allow_undefined_flag} $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && $ECHO "X${wl}-set_version ${wl}$verstring" | $Xsed` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib'
+ ;;
+ *)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib ${allow_undefined_flag} $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-msym ${wl}-soname ${wl}$soname `test -n "$verstring" && $ECHO "${wl}-set_version ${wl}$verstring" | $Xsed` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib'
+ ;;
+ esac
+
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-rpath ${wl}$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=:
+
+ # Commands to make compiler produce verbose output that lists
+ # what "hidden" libraries, object files and flags are used when
+ # linking a shared library.
+ output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP "\-L"'
+
+ else
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ fi
+ ;;
+ esac
+ ;;
+
+ psos*)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+
+ sunos4*)
+ case $cc_basename in
+ CC*)
+ # Sun C++ 4.x
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ lcc*)
+ # Lucid
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ *)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ esac
+ ;;
+
+ solaris*)
+ case $cc_basename in
+ CC*)
+ # Sun C++ 4.2, 5.x and Centerline C++
+ _LT_TAGVAR(archive_cmds_need_lc,$1)=yes
+ _LT_TAGVAR(no_undefined_flag, $1)=' -zdefs'
+ _LT_TAGVAR(archive_cmds, $1)='$CC -G${allow_undefined_flag} -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~
+ $CC -G${allow_undefined_flag} ${wl}-M ${wl}$lib.exp -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~$RM $lib.exp'
+
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir'
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ case $host_os in
+ solaris2.[[0-5]] | solaris2.[[0-5]].*) ;;
+ *)
+ # The compiler driver will combine and reorder linker options,
+ # but understands `-z linker_flag'.
+ # Supported since Solaris 2.6 (maybe 2.5.1?)
+ _LT_TAGVAR(whole_archive_flag_spec, $1)='-z allextract$convenience -z defaultextract'
+ ;;
+ esac
+ _LT_TAGVAR(link_all_deplibs, $1)=yes
+
+ output_verbose_link_cmd='echo'
+
+ # Archives containing C++ object files must be created using
+ # "CC -xar", where "CC" is the Sun C++ compiler. This is
+ # necessary to make sure instantiated templates are included
+ # in the archive.
+ _LT_TAGVAR(old_archive_cmds, $1)='$CC -xar -o $oldlib $oldobjs'
+ ;;
+ gcx*)
+ # Green Hills C++ Compiler
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-h $wl$soname -o $lib'
+
+ # The C++ compiler must be used to create the archive.
+ _LT_TAGVAR(old_archive_cmds, $1)='$CC $LDFLAGS -archive -o $oldlib $oldobjs'
+ ;;
+ *)
+ # GNU C++ compiler with Solaris linker
+ if test "$GXX" = yes && test "$with_gnu_ld" = no; then
+ _LT_TAGVAR(no_undefined_flag, $1)=' ${wl}-z ${wl}defs'
+ if $CC --version | $GREP -v '^2\.7' > /dev/null; then
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $LDFLAGS $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-h $wl$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~
+ $CC -shared -nostdlib ${wl}-M $wl$lib.exp -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~$RM $lib.exp'
+
+ # Commands to make compiler produce verbose output that lists
+ # what "hidden" libraries, object files and flags are used when
+ # linking a shared library.
+ output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP "\-L"'
+ else
+ # g++ 2.7 appears to require `-G' NOT `-shared' on this
+ # platform.
+ _LT_TAGVAR(archive_cmds, $1)='$CC -G -nostdlib $LDFLAGS $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-h $wl$soname -o $lib'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~
+ $CC -G -nostdlib ${wl}-M $wl$lib.exp -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~$RM $lib.exp'
+
+ # Commands to make compiler produce verbose output that lists
+ # what "hidden" libraries, object files and flags are used when
+ # linking a shared library.
+ output_verbose_link_cmd='$CC -G $CFLAGS -v conftest.$objext 2>&1 | $GREP "\-L"'
+ fi
+
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-R $wl$libdir'
+ case $host_os in
+ solaris2.[[0-5]] | solaris2.[[0-5]].*) ;;
+ *)
+ _LT_TAGVAR(whole_archive_flag_spec, $1)='${wl}-z ${wl}allextract$convenience ${wl}-z ${wl}defaultextract'
+ ;;
+ esac
+ fi
+ ;;
+ esac
+ ;;
+
+ sysv4*uw2* | sysv5OpenUNIX* | sysv5UnixWare7.[[01]].[[10]]* | unixware7* | sco3.2v5.0.[[024]]*)
+ _LT_TAGVAR(no_undefined_flag, $1)='${wl}-z,text'
+ _LT_TAGVAR(archive_cmds_need_lc, $1)=no
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ runpath_var='LD_RUN_PATH'
+
+ case $cc_basename in
+ CC*)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ *)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ esac
+ ;;
+
+ sysv5* | sco3.2v5* | sco5v6*)
+ # Note: We can NOT use -z defs as we might desire, because we do not
+ # link with -lc, and that would cause any symbols used from libc to
+ # always be unresolved, which means just about no library would
+ # ever link correctly. If we're not using GNU ld we use -z text
+ # though, which does catch some bad symbols but isn't as heavy-handed
+ # as -z defs.
+ _LT_TAGVAR(no_undefined_flag, $1)='${wl}-z,text'
+ _LT_TAGVAR(allow_undefined_flag, $1)='${wl}-z,nodefs'
+ _LT_TAGVAR(archive_cmds_need_lc, $1)=no
+ _LT_TAGVAR(hardcode_shlibpath_var, $1)=no
+ _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='${wl}-R,$libdir'
+ _LT_TAGVAR(hardcode_libdir_separator, $1)=':'
+ _LT_TAGVAR(link_all_deplibs, $1)=yes
+ _LT_TAGVAR(export_dynamic_flag_spec, $1)='${wl}-Bexport'
+ runpath_var='LD_RUN_PATH'
+
+ case $cc_basename in
+ CC*)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ *)
+ _LT_TAGVAR(archive_cmds, $1)='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ esac
+ ;;
+
+ tandem*)
+ case $cc_basename in
+ NCC*)
+ # NonStop-UX NCC 3.20
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ *)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ esac
+ ;;
+
+ vxworks*)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+
+ *)
+ # FIXME: insert proper C++ library support
+ _LT_TAGVAR(ld_shlibs, $1)=no
+ ;;
+ esac
+
+ AC_MSG_RESULT([$_LT_TAGVAR(ld_shlibs, $1)])
+ test "$_LT_TAGVAR(ld_shlibs, $1)" = no && can_build_shared=no
+
+ _LT_TAGVAR(GCC, $1)="$GXX"
+ _LT_TAGVAR(LD, $1)="$LD"
+
+ ## CAVEAT EMPTOR:
+ ## There is no encapsulation within the following macros, do not change
+ ## the running order or otherwise move them around unless you know exactly
+ ## what you are doing...
+ _LT_SYS_HIDDEN_LIBDEPS($1)
+ _LT_COMPILER_PIC($1)
+ _LT_COMPILER_C_O($1)
+ _LT_COMPILER_FILE_LOCKS($1)
+ _LT_LINKER_SHLIBS($1)
+ _LT_SYS_DYNAMIC_LINKER($1)
+ _LT_LINKER_HARDCODE_LIBPATH($1)
+
+ _LT_CONFIG($1)
+ fi # test -n "$compiler"
+
+ CC=$lt_save_CC
+ LDCXX=$LD
+ LD=$lt_save_LD
+ GCC=$lt_save_GCC
+ with_gnu_ld=$lt_save_with_gnu_ld
+ lt_cv_path_LDCXX=$lt_cv_path_LD
+ lt_cv_path_LD=$lt_save_path_LD
+ lt_cv_prog_gnu_ldcxx=$lt_cv_prog_gnu_ld
+ lt_cv_prog_gnu_ld=$lt_save_with_gnu_ld
+fi # test "$_lt_caught_CXX_error" != yes
+
+AC_LANG_POP
+])# _LT_LANG_CXX_CONFIG
+
+
+# _LT_SYS_HIDDEN_LIBDEPS([TAGNAME])
+# ---------------------------------
+# Figure out "hidden" library dependencies from verbose
+# compiler output when linking a shared library.
+# Parse the compiler output and extract the necessary
+# objects, libraries and library flags.
+m4_defun([_LT_SYS_HIDDEN_LIBDEPS],
+[m4_require([_LT_FILEUTILS_DEFAULTS])dnl
+# Dependencies to place before and after the object being linked:
+_LT_TAGVAR(predep_objects, $1)=
+_LT_TAGVAR(postdep_objects, $1)=
+_LT_TAGVAR(predeps, $1)=
+_LT_TAGVAR(postdeps, $1)=
+_LT_TAGVAR(compiler_lib_search_path, $1)=
+
+dnl we can't use the lt_simple_compile_test_code here,
+dnl because it contains code intended for an executable,
+dnl not a library. It's possible we should let each
+dnl tag define a new lt_????_link_test_code variable,
+dnl but it's only used here...
+m4_if([$1], [], [cat > conftest.$ac_ext <<_LT_EOF
+int a;
+void foo (void) { a = 0; }
+_LT_EOF
+], [$1], [CXX], [cat > conftest.$ac_ext <<_LT_EOF
+class Foo
+{
+public:
+ Foo (void) { a = 0; }
+private:
+ int a;
+};
+_LT_EOF
+], [$1], [F77], [cat > conftest.$ac_ext <<_LT_EOF
+ subroutine foo
+ implicit none
+ integer*4 a
+ a=0
+ return
+ end
+_LT_EOF
+], [$1], [FC], [cat > conftest.$ac_ext <<_LT_EOF
+ subroutine foo
+ implicit none
+ integer a
+ a=0
+ return
+ end
+_LT_EOF
+], [$1], [GCJ], [cat > conftest.$ac_ext <<_LT_EOF
+public class foo {
+ private int a;
+ public void bar (void) {
+ a = 0;
+ }
+};
+_LT_EOF
+])
+dnl Parse the compiler output and extract the necessary
+dnl objects, libraries and library flags.
+if AC_TRY_EVAL(ac_compile); then
+ # Parse the compiler output and extract the necessary
+ # objects, libraries and library flags.
+
+ # Sentinel used to keep track of whether or not we are before
+ # the conftest object file.
+ pre_test_object_deps_done=no
+
+ for p in `eval "$output_verbose_link_cmd"`; do
+ case $p in
+
+ -L* | -R* | -l*)
+ # Some compilers place space between "-{L,R}" and the path.
+ # Remove the space.
+ if test $p = "-L" ||
+ test $p = "-R"; then
+ prev=$p
+ continue
+ else
+ prev=
+ fi
+
+ if test "$pre_test_object_deps_done" = no; then
+ case $p in
+ -L* | -R*)
+ # Internal compiler library paths should come after those
+ # provided the user. The postdeps already come after the
+ # user supplied libs so there is no need to process them.
+ if test -z "$_LT_TAGVAR(compiler_lib_search_path, $1)"; then
+ _LT_TAGVAR(compiler_lib_search_path, $1)="${prev}${p}"
+ else
+ _LT_TAGVAR(compiler_lib_search_path, $1)="${_LT_TAGVAR(compiler_lib_search_path, $1)} ${prev}${p}"
+ fi
+ ;;
+ # The "-l" case would never come before the object being
+ # linked, so don't bother handling this case.
+ esac
+ else
+ if test -z "$_LT_TAGVAR(postdeps, $1)"; then
+ _LT_TAGVAR(postdeps, $1)="${prev}${p}"
+ else
+ _LT_TAGVAR(postdeps, $1)="${_LT_TAGVAR(postdeps, $1)} ${prev}${p}"
+ fi
+ fi
+ ;;
+
+ *.$objext)
+ # This assumes that the test object file only shows up
+ # once in the compiler output.
+ if test "$p" = "conftest.$objext"; then
+ pre_test_object_deps_done=yes
+ continue
+ fi
+
+ if test "$pre_test_object_deps_done" = no; then
+ if test -z "$_LT_TAGVAR(predep_objects, $1)"; then
+ _LT_TAGVAR(predep_objects, $1)="$p"
+ else
+ _LT_TAGVAR(predep_objects, $1)="$_LT_TAGVAR(predep_objects, $1) $p"
+ fi
+ else
+ if test -z "$_LT_TAGVAR(postdep_objects, $1)"; then
+ _LT_TAGVAR(postdep_objects, $1)="$p"
+ else
+ _LT_TAGVAR(postdep_objects, $1)="$_LT_TAGVAR(postdep_objects, $1) $p"
+ fi
+ fi
+ ;;
+
+ *) ;; # Ignore the rest.
+
+ esac
+ done
+
+ # Clean up.
+ rm -f a.out a.exe
+else
+ echo "libtool.m4: error: problem compiling $1 test program"
+fi
+
+$RM -f confest.$objext
+
+# PORTME: override above test on systems where it is broken
+m4_if([$1], [CXX],
+[case $host_os in
+interix[[3-9]]*)
+ # Interix 3.5 installs completely hosed .la files for C++, so rather than
+ # hack all around it, let's just trust "g++" to DTRT.
+ _LT_TAGVAR(predep_objects,$1)=
+ _LT_TAGVAR(postdep_objects,$1)=
+ _LT_TAGVAR(postdeps,$1)=
+ ;;
+
+linux*)
+ case `$CC -V 2>&1 | sed 5q` in
+ *Sun\ C*)
+ # Sun C++ 5.9
+
+ # The more standards-conforming stlport4 library is
+ # incompatible with the Cstd library. Avoid specifying
+ # it if it's in CXXFLAGS. Ignore libCrun as
+ # -library=stlport4 depends on it.
+ case " $CXX $CXXFLAGS " in
+ *" -library=stlport4 "*)
+ solaris_use_stlport4=yes
+ ;;
+ esac
+
+ if test "$solaris_use_stlport4" != yes; then
+ _LT_TAGVAR(postdeps,$1)='-library=Cstd -library=Crun'
+ fi
+ ;;
+ esac
+ ;;
+
+solaris*)
+ case $cc_basename in
+ CC*)
+ # The more standards-conforming stlport4 library is
+ # incompatible with the Cstd library. Avoid specifying
+ # it if it's in CXXFLAGS. Ignore libCrun as
+ # -library=stlport4 depends on it.
+ case " $CXX $CXXFLAGS " in
+ *" -library=stlport4 "*)
+ solaris_use_stlport4=yes
+ ;;
+ esac
+
+ # Adding this requires a known-good setup of shared libraries for
+ # Sun compiler versions before 5.6, else PIC objects from an old
+ # archive will be linked into the output, leading to subtle bugs.
+ if test "$solaris_use_stlport4" != yes; then
+ _LT_TAGVAR(postdeps,$1)='-library=Cstd -library=Crun'
+ fi
+ ;;
+ esac
+ ;;
+esac
+])
+
+case " $_LT_TAGVAR(postdeps, $1) " in
+*" -lc "*) _LT_TAGVAR(archive_cmds_need_lc, $1)=no ;;
+esac
+ _LT_TAGVAR(compiler_lib_search_dirs, $1)=
+if test -n "${_LT_TAGVAR(compiler_lib_search_path, $1)}"; then
+ _LT_TAGVAR(compiler_lib_search_dirs, $1)=`echo " ${_LT_TAGVAR(compiler_lib_search_path, $1)}" | ${SED} -e 's! -L! !g' -e 's!^ !!'`
+fi
+_LT_TAGDECL([], [compiler_lib_search_dirs], [1],
+ [The directories searched by this compiler when creating a shared library])
+_LT_TAGDECL([], [predep_objects], [1],
+ [Dependencies to place before and after the objects being linked to
+ create a shared library])
+_LT_TAGDECL([], [postdep_objects], [1])
+_LT_TAGDECL([], [predeps], [1])
+_LT_TAGDECL([], [postdeps], [1])
+_LT_TAGDECL([], [compiler_lib_search_path], [1],
+ [The library search path used internally by the compiler when linking
+ a shared library])
+])# _LT_SYS_HIDDEN_LIBDEPS
+
+
+# _LT_PROG_F77
+# ------------
+# Since AC_PROG_F77 is broken, in that it returns the empty string
+# if there is no fortran compiler, we have our own version here.
+m4_defun([_LT_PROG_F77],
+[
+pushdef([AC_MSG_ERROR], [_lt_disable_F77=yes])
+AC_PROG_F77
+if test -z "$F77" || test "X$F77" = "Xno"; then
+ _lt_disable_F77=yes
+fi
+popdef([AC_MSG_ERROR])
+])# _LT_PROG_F77
+
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([_LT_PROG_F77], [])
+
+
+# _LT_LANG_F77_CONFIG([TAG])
+# --------------------------
+# Ensure that the configuration variables for a Fortran 77 compiler are
+# suitably defined. These variables are subsequently used by _LT_CONFIG
+# to write the compiler configuration to `libtool'.
+m4_defun([_LT_LANG_F77_CONFIG],
+[AC_REQUIRE([_LT_PROG_F77])dnl
+AC_LANG_PUSH(Fortran 77)
+
+_LT_TAGVAR(archive_cmds_need_lc, $1)=no
+_LT_TAGVAR(allow_undefined_flag, $1)=
+_LT_TAGVAR(always_export_symbols, $1)=no
+_LT_TAGVAR(archive_expsym_cmds, $1)=
+_LT_TAGVAR(export_dynamic_flag_spec, $1)=
+_LT_TAGVAR(hardcode_direct, $1)=no
+_LT_TAGVAR(hardcode_direct_absolute, $1)=no
+_LT_TAGVAR(hardcode_libdir_flag_spec, $1)=
+_LT_TAGVAR(hardcode_libdir_flag_spec_ld, $1)=
+_LT_TAGVAR(hardcode_libdir_separator, $1)=
+_LT_TAGVAR(hardcode_minus_L, $1)=no
+_LT_TAGVAR(hardcode_automatic, $1)=no
+_LT_TAGVAR(inherit_rpath, $1)=no
+_LT_TAGVAR(module_cmds, $1)=
+_LT_TAGVAR(module_expsym_cmds, $1)=
+_LT_TAGVAR(link_all_deplibs, $1)=unknown
+_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds
+_LT_TAGVAR(no_undefined_flag, $1)=
+_LT_TAGVAR(whole_archive_flag_spec, $1)=
+_LT_TAGVAR(enable_shared_with_static_runtimes, $1)=no
+
+# Source file extension for f77 test sources.
+ac_ext=f
+
+# Object file extension for compiled f77 test sources.
+objext=o
+_LT_TAGVAR(objext, $1)=$objext
+
+# No sense in running all these tests if we already determined that
+# the F77 compiler isn't working. Some variables (like enable_shared)
+# are currently assumed to apply to all compilers on this platform,
+# and will be corrupted by setting them based on a non-working compiler.
+if test "$_lt_disable_F77" != yes; then
+ # Code to be used in simple compile tests
+ lt_simple_compile_test_code="\
+ subroutine t
+ return
+ end
+"
+
+ # Code to be used in simple link tests
+ lt_simple_link_test_code="\
+ program t
+ end
+"
+
+ # ltmain only uses $CC for tagged configurations so make sure $CC is set.
+ _LT_TAG_COMPILER
+
+ # save warnings/boilerplate of simple test code
+ _LT_COMPILER_BOILERPLATE
+ _LT_LINKER_BOILERPLATE
+
+ # Allow CC to be a program name with arguments.
+ lt_save_CC="$CC"
+ lt_save_GCC=$GCC
+ CC=${F77-"f77"}
+ compiler=$CC
+ _LT_TAGVAR(compiler, $1)=$CC
+ _LT_CC_BASENAME([$compiler])
+ GCC=$G77
+ if test -n "$compiler"; then
+ AC_MSG_CHECKING([if libtool supports shared libraries])
+ AC_MSG_RESULT([$can_build_shared])
+
+ AC_MSG_CHECKING([whether to build shared libraries])
+ test "$can_build_shared" = "no" && enable_shared=no
+
+ # On AIX, shared libraries and static libraries use the same namespace, and
+ # are all built from PIC.
+ case $host_os in
+ aix3*)
+ test "$enable_shared" = yes && enable_static=no
+ if test -n "$RANLIB"; then
+ archive_cmds="$archive_cmds~\$RANLIB \$lib"
+ postinstall_cmds='$RANLIB $lib'
+ fi
+ ;;
+ aix[[4-9]]*)
+ if test "$host_cpu" != ia64 && test "$aix_use_runtimelinking" = no ; then
+ test "$enable_shared" = yes && enable_static=no
+ fi
+ ;;
+ esac
+ AC_MSG_RESULT([$enable_shared])
+
+ AC_MSG_CHECKING([whether to build static libraries])
+ # Make sure either enable_shared or enable_static is yes.
+ test "$enable_shared" = yes || enable_static=yes
+ AC_MSG_RESULT([$enable_static])
+
+ _LT_TAGVAR(GCC, $1)="$G77"
+ _LT_TAGVAR(LD, $1)="$LD"
+
+ ## CAVEAT EMPTOR:
+ ## There is no encapsulation within the following macros, do not change
+ ## the running order or otherwise move them around unless you know exactly
+ ## what you are doing...
+ _LT_COMPILER_PIC($1)
+ _LT_COMPILER_C_O($1)
+ _LT_COMPILER_FILE_LOCKS($1)
+ _LT_LINKER_SHLIBS($1)
+ _LT_SYS_DYNAMIC_LINKER($1)
+ _LT_LINKER_HARDCODE_LIBPATH($1)
+
+ _LT_CONFIG($1)
+ fi # test -n "$compiler"
+
+ GCC=$lt_save_GCC
+ CC="$lt_save_CC"
+fi # test "$_lt_disable_F77" != yes
+
+AC_LANG_POP
+])# _LT_LANG_F77_CONFIG
+
+
+# _LT_PROG_FC
+# -----------
+# Since AC_PROG_FC is broken, in that it returns the empty string
+# if there is no fortran compiler, we have our own version here.
+m4_defun([_LT_PROG_FC],
+[
+pushdef([AC_MSG_ERROR], [_lt_disable_FC=yes])
+AC_PROG_FC
+if test -z "$FC" || test "X$FC" = "Xno"; then
+ _lt_disable_FC=yes
+fi
+popdef([AC_MSG_ERROR])
+])# _LT_PROG_FC
+
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([_LT_PROG_FC], [])
+
+
+# _LT_LANG_FC_CONFIG([TAG])
+# -------------------------
+# Ensure that the configuration variables for a Fortran compiler are
+# suitably defined. These variables are subsequently used by _LT_CONFIG
+# to write the compiler configuration to `libtool'.
+m4_defun([_LT_LANG_FC_CONFIG],
+[AC_REQUIRE([_LT_PROG_FC])dnl
+AC_LANG_PUSH(Fortran)
+
+_LT_TAGVAR(archive_cmds_need_lc, $1)=no
+_LT_TAGVAR(allow_undefined_flag, $1)=
+_LT_TAGVAR(always_export_symbols, $1)=no
+_LT_TAGVAR(archive_expsym_cmds, $1)=
+_LT_TAGVAR(export_dynamic_flag_spec, $1)=
+_LT_TAGVAR(hardcode_direct, $1)=no
+_LT_TAGVAR(hardcode_direct_absolute, $1)=no
+_LT_TAGVAR(hardcode_libdir_flag_spec, $1)=
+_LT_TAGVAR(hardcode_libdir_flag_spec_ld, $1)=
+_LT_TAGVAR(hardcode_libdir_separator, $1)=
+_LT_TAGVAR(hardcode_minus_L, $1)=no
+_LT_TAGVAR(hardcode_automatic, $1)=no
+_LT_TAGVAR(inherit_rpath, $1)=no
+_LT_TAGVAR(module_cmds, $1)=
+_LT_TAGVAR(module_expsym_cmds, $1)=
+_LT_TAGVAR(link_all_deplibs, $1)=unknown
+_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds
+_LT_TAGVAR(no_undefined_flag, $1)=
+_LT_TAGVAR(whole_archive_flag_spec, $1)=
+_LT_TAGVAR(enable_shared_with_static_runtimes, $1)=no
+
+# Source file extension for fc test sources.
+ac_ext=${ac_fc_srcext-f}
+
+# Object file extension for compiled fc test sources.
+objext=o
+_LT_TAGVAR(objext, $1)=$objext
+
+# No sense in running all these tests if we already determined that
+# the FC compiler isn't working. Some variables (like enable_shared)
+# are currently assumed to apply to all compilers on this platform,
+# and will be corrupted by setting them based on a non-working compiler.
+if test "$_lt_disable_FC" != yes; then
+ # Code to be used in simple compile tests
+ lt_simple_compile_test_code="\
+ subroutine t
+ return
+ end
+"
+
+ # Code to be used in simple link tests
+ lt_simple_link_test_code="\
+ program t
+ end
+"
+
+ # ltmain only uses $CC for tagged configurations so make sure $CC is set.
+ _LT_TAG_COMPILER
+
+ # save warnings/boilerplate of simple test code
+ _LT_COMPILER_BOILERPLATE
+ _LT_LINKER_BOILERPLATE
+
+ # Allow CC to be a program name with arguments.
+ lt_save_CC="$CC"
+ lt_save_GCC=$GCC
+ CC=${FC-"f95"}
+ compiler=$CC
+ GCC=$ac_cv_fc_compiler_gnu
+
+ _LT_TAGVAR(compiler, $1)=$CC
+ _LT_CC_BASENAME([$compiler])
+
+ if test -n "$compiler"; then
+ AC_MSG_CHECKING([if libtool supports shared libraries])
+ AC_MSG_RESULT([$can_build_shared])
+
+ AC_MSG_CHECKING([whether to build shared libraries])
+ test "$can_build_shared" = "no" && enable_shared=no
+
+ # On AIX, shared libraries and static libraries use the same namespace, and
+ # are all built from PIC.
+ case $host_os in
+ aix3*)
+ test "$enable_shared" = yes && enable_static=no
+ if test -n "$RANLIB"; then
+ archive_cmds="$archive_cmds~\$RANLIB \$lib"
+ postinstall_cmds='$RANLIB $lib'
+ fi
+ ;;
+ aix[[4-9]]*)
+ if test "$host_cpu" != ia64 && test "$aix_use_runtimelinking" = no ; then
+ test "$enable_shared" = yes && enable_static=no
+ fi
+ ;;
+ esac
+ AC_MSG_RESULT([$enable_shared])
+
+ AC_MSG_CHECKING([whether to build static libraries])
+ # Make sure either enable_shared or enable_static is yes.
+ test "$enable_shared" = yes || enable_static=yes
+ AC_MSG_RESULT([$enable_static])
+
+ _LT_TAGVAR(GCC, $1)="$ac_cv_fc_compiler_gnu"
+ _LT_TAGVAR(LD, $1)="$LD"
+
+ ## CAVEAT EMPTOR:
+ ## There is no encapsulation within the following macros, do not change
+ ## the running order or otherwise move them around unless you know exactly
+ ## what you are doing...
+ _LT_SYS_HIDDEN_LIBDEPS($1)
+ _LT_COMPILER_PIC($1)
+ _LT_COMPILER_C_O($1)
+ _LT_COMPILER_FILE_LOCKS($1)
+ _LT_LINKER_SHLIBS($1)
+ _LT_SYS_DYNAMIC_LINKER($1)
+ _LT_LINKER_HARDCODE_LIBPATH($1)
+
+ _LT_CONFIG($1)
+ fi # test -n "$compiler"
+
+ GCC=$lt_save_GCC
+ CC="$lt_save_CC"
+fi # test "$_lt_disable_FC" != yes
+
+AC_LANG_POP
+])# _LT_LANG_FC_CONFIG
+
+
+# _LT_LANG_GCJ_CONFIG([TAG])
+# --------------------------
+# Ensure that the configuration variables for the GNU Java Compiler compiler
+# are suitably defined. These variables are subsequently used by _LT_CONFIG
+# to write the compiler configuration to `libtool'.
+m4_defun([_LT_LANG_GCJ_CONFIG],
+[AC_REQUIRE([LT_PROG_GCJ])dnl
+AC_LANG_SAVE
+
+# Source file extension for Java test sources.
+ac_ext=java
+
+# Object file extension for compiled Java test sources.
+objext=o
+_LT_TAGVAR(objext, $1)=$objext
+
+# Code to be used in simple compile tests
+lt_simple_compile_test_code="class foo {}"
+
+# Code to be used in simple link tests
+lt_simple_link_test_code='public class conftest { public static void main(String[[]] argv) {}; }'
+
+# ltmain only uses $CC for tagged configurations so make sure $CC is set.
+_LT_TAG_COMPILER
+
+# save warnings/boilerplate of simple test code
+_LT_COMPILER_BOILERPLATE
+_LT_LINKER_BOILERPLATE
+
+# Allow CC to be a program name with arguments.
+lt_save_CC="$CC"
+lt_save_GCC=$GCC
+GCC=yes
+CC=${GCJ-"gcj"}
+compiler=$CC
+_LT_TAGVAR(compiler, $1)=$CC
+_LT_TAGVAR(LD, $1)="$LD"
+_LT_CC_BASENAME([$compiler])
+
+# GCJ did not exist at the time GCC didn't implicitly link libc in.
+_LT_TAGVAR(archive_cmds_need_lc, $1)=no
+
+_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds
+
+## CAVEAT EMPTOR:
+## There is no encapsulation within the following macros, do not change
+## the running order or otherwise move them around unless you know exactly
+## what you are doing...
+if test -n "$compiler"; then
+ _LT_COMPILER_NO_RTTI($1)
+ _LT_COMPILER_PIC($1)
+ _LT_COMPILER_C_O($1)
+ _LT_COMPILER_FILE_LOCKS($1)
+ _LT_LINKER_SHLIBS($1)
+ _LT_LINKER_HARDCODE_LIBPATH($1)
+
+ _LT_CONFIG($1)
+fi
+
+AC_LANG_RESTORE
+
+GCC=$lt_save_GCC
+CC="$lt_save_CC"
+])# _LT_LANG_GCJ_CONFIG
+
+
+# _LT_LANG_RC_CONFIG([TAG])
+# -------------------------
+# Ensure that the configuration variables for the Windows resource compiler
+# are suitably defined. These variables are subsequently used by _LT_CONFIG
+# to write the compiler configuration to `libtool'.
+m4_defun([_LT_LANG_RC_CONFIG],
+[AC_REQUIRE([LT_PROG_RC])dnl
+AC_LANG_SAVE
+
+# Source file extension for RC test sources.
+ac_ext=rc
+
+# Object file extension for compiled RC test sources.
+objext=o
+_LT_TAGVAR(objext, $1)=$objext
+
+# Code to be used in simple compile tests
+lt_simple_compile_test_code='sample MENU { MENUITEM "&Soup", 100, CHECKED }'
+
+# Code to be used in simple link tests
+lt_simple_link_test_code="$lt_simple_compile_test_code"
+
+# ltmain only uses $CC for tagged configurations so make sure $CC is set.
+_LT_TAG_COMPILER
+
+# save warnings/boilerplate of simple test code
+_LT_COMPILER_BOILERPLATE
+_LT_LINKER_BOILERPLATE
+
+# Allow CC to be a program name with arguments.
+lt_save_CC="$CC"
+lt_save_GCC=$GCC
+GCC=
+CC=${RC-"windres"}
+compiler=$CC
+_LT_TAGVAR(compiler, $1)=$CC
+_LT_CC_BASENAME([$compiler])
+_LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)=yes
+
+if test -n "$compiler"; then
+ :
+ _LT_CONFIG($1)
+fi
+
+GCC=$lt_save_GCC
+AC_LANG_RESTORE
+CC="$lt_save_CC"
+])# _LT_LANG_RC_CONFIG
+
+
+# LT_PROG_GCJ
+# -----------
+AC_DEFUN([LT_PROG_GCJ],
+[m4_ifdef([AC_PROG_GCJ], [AC_PROG_GCJ],
+ [m4_ifdef([A][M_PROG_GCJ], [A][M_PROG_GCJ],
+ [AC_CHECK_TOOL(GCJ, gcj,)
+ test "x${GCJFLAGS+set}" = xset || GCJFLAGS="-g -O2"
+ AC_SUBST(GCJFLAGS)])])[]dnl
+])
+
+# Old name:
+AU_ALIAS([LT_AC_PROG_GCJ], [LT_PROG_GCJ])
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([LT_AC_PROG_GCJ], [])
+
+
+# LT_PROG_RC
+# ----------
+AC_DEFUN([LT_PROG_RC],
+[AC_CHECK_TOOL(RC, windres,)
+])
+
+# Old name:
+AU_ALIAS([LT_AC_PROG_RC], [LT_PROG_RC])
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([LT_AC_PROG_RC], [])
+
+
+# _LT_DECL_EGREP
+# --------------
+# If we don't have a new enough Autoconf to choose the best grep
+# available, choose the one first in the user's PATH.
+m4_defun([_LT_DECL_EGREP],
+[AC_REQUIRE([AC_PROG_EGREP])dnl
+AC_REQUIRE([AC_PROG_FGREP])dnl
+test -z "$GREP" && GREP=grep
+_LT_DECL([], [GREP], [1], [A grep program that handles long lines])
+_LT_DECL([], [EGREP], [1], [An ERE matcher])
+_LT_DECL([], [FGREP], [1], [A literal string matcher])
+dnl Non-bleeding-edge autoconf doesn't subst GREP, so do it here too
+AC_SUBST([GREP])
+])
+
+
+# _LT_DECL_OBJDUMP
+# --------------
+# If we don't have a new enough Autoconf to choose the best objdump
+# available, choose the one first in the user's PATH.
+m4_defun([_LT_DECL_OBJDUMP],
+[AC_CHECK_TOOL(OBJDUMP, objdump, false)
+test -z "$OBJDUMP" && OBJDUMP=objdump
+_LT_DECL([], [OBJDUMP], [1], [An object symbol dumper])
+AC_SUBST([OBJDUMP])
+])
+
+
+# _LT_DECL_SED
+# ------------
+# Check for a fully-functional sed program, that truncates
+# as few characters as possible. Prefer GNU sed if found.
+m4_defun([_LT_DECL_SED],
+[AC_PROG_SED
+test -z "$SED" && SED=sed
+Xsed="$SED -e 1s/^X//"
+_LT_DECL([], [SED], [1], [A sed program that does not truncate output])
+_LT_DECL([], [Xsed], ["\$SED -e 1s/^X//"],
+ [Sed that helps us avoid accidentally triggering echo(1) options like -n])
+])# _LT_DECL_SED
+
+m4_ifndef([AC_PROG_SED], [
+############################################################
+# NOTE: This macro has been submitted for inclusion into #
+# GNU Autoconf as AC_PROG_SED. When it is available in #
+# a released version of Autoconf we should remove this #
+# macro and use it instead. #
+############################################################
+
+m4_defun([AC_PROG_SED],
+[AC_MSG_CHECKING([for a sed that does not truncate output])
+AC_CACHE_VAL(lt_cv_path_SED,
+[# Loop through the user's path and test for sed and gsed.
+# Then use that list of sed's as ones to test for truncation.
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for lt_ac_prog in sed gsed; do
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if $as_executable_p "$as_dir/$lt_ac_prog$ac_exec_ext"; then
+ lt_ac_sed_list="$lt_ac_sed_list $as_dir/$lt_ac_prog$ac_exec_ext"
+ fi
+ done
+ done
+done
+IFS=$as_save_IFS
+lt_ac_max=0
+lt_ac_count=0
+# Add /usr/xpg4/bin/sed as it is typically found on Solaris
+# along with /bin/sed that truncates output.
+for lt_ac_sed in $lt_ac_sed_list /usr/xpg4/bin/sed; do
+ test ! -f $lt_ac_sed && continue
+ cat /dev/null > conftest.in
+ lt_ac_count=0
+ echo $ECHO_N "0123456789$ECHO_C" >conftest.in
+ # Check for GNU sed and select it if it is found.
+ if "$lt_ac_sed" --version 2>&1 < /dev/null | grep 'GNU' > /dev/null; then
+ lt_cv_path_SED=$lt_ac_sed
+ break
+ fi
+ while true; do
+ cat conftest.in conftest.in >conftest.tmp
+ mv conftest.tmp conftest.in
+ cp conftest.in conftest.nl
+ echo >>conftest.nl
+ $lt_ac_sed -e 's/a$//' < conftest.nl >conftest.out || break
+ cmp -s conftest.out conftest.nl || break
+ # 10000 chars as input seems more than enough
+ test $lt_ac_count -gt 10 && break
+ lt_ac_count=`expr $lt_ac_count + 1`
+ if test $lt_ac_count -gt $lt_ac_max; then
+ lt_ac_max=$lt_ac_count
+ lt_cv_path_SED=$lt_ac_sed
+ fi
+ done
+done
+])
+SED=$lt_cv_path_SED
+AC_SUBST([SED])
+AC_MSG_RESULT([$SED])
+])#AC_PROG_SED
+])#m4_ifndef
+
+# Old name:
+AU_ALIAS([LT_AC_PROG_SED], [AC_PROG_SED])
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([LT_AC_PROG_SED], [])
+
+
+# _LT_CHECK_SHELL_FEATURES
+# ------------------------
+# Find out whether the shell is Bourne or XSI compatible,
+# or has some other useful features.
+m4_defun([_LT_CHECK_SHELL_FEATURES],
+[AC_MSG_CHECKING([whether the shell understands some XSI constructs])
+# Try some XSI features
+xsi_shell=no
+( _lt_dummy="a/b/c"
+ test "${_lt_dummy##*/},${_lt_dummy%/*},"${_lt_dummy%"$_lt_dummy"}, \
+ = c,a/b,, \
+ && eval 'test $(( 1 + 1 )) -eq 2 \
+ && test "${#_lt_dummy}" -eq 5' ) >/dev/null 2>&1 \
+ && xsi_shell=yes
+AC_MSG_RESULT([$xsi_shell])
+_LT_CONFIG_LIBTOOL_INIT([xsi_shell='$xsi_shell'])
+
+AC_MSG_CHECKING([whether the shell understands "+="])
+lt_shell_append=no
+( foo=bar; set foo baz; eval "$[1]+=\$[2]" && test "$foo" = barbaz ) \
+ >/dev/null 2>&1 \
+ && lt_shell_append=yes
+AC_MSG_RESULT([$lt_shell_append])
+_LT_CONFIG_LIBTOOL_INIT([lt_shell_append='$lt_shell_append'])
+
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+ lt_unset=unset
+else
+ lt_unset=false
+fi
+_LT_DECL([], [lt_unset], [0], [whether the shell understands "unset"])dnl
+
+# test EBCDIC or ASCII
+case `echo X|tr X '\101'` in
+ A) # ASCII based system
+ # \n is not interpreted correctly by Solaris 8 /usr/ucb/tr
+ lt_SP2NL='tr \040 \012'
+ lt_NL2SP='tr \015\012 \040\040'
+ ;;
+ *) # EBCDIC based system
+ lt_SP2NL='tr \100 \n'
+ lt_NL2SP='tr \r\n \100\100'
+ ;;
+esac
+_LT_DECL([SP2NL], [lt_SP2NL], [1], [turn spaces into newlines])dnl
+_LT_DECL([NL2SP], [lt_NL2SP], [1], [turn newlines into spaces])dnl
+])# _LT_CHECK_SHELL_FEATURES
+
+
+# _LT_PROG_XSI_SHELLFNS
+# ---------------------
+# Bourne and XSI compatible variants of some useful shell functions.
+m4_defun([_LT_PROG_XSI_SHELLFNS],
+[case $xsi_shell in
+ yes)
+ cat << \_LT_EOF >> "$cfgfile"
+
+# func_dirname file append nondir_replacement
+# Compute the dirname of FILE. If nonempty, add APPEND to the result,
+# otherwise set result to NONDIR_REPLACEMENT.
+func_dirname ()
+{
+ case ${1} in
+ */*) func_dirname_result="${1%/*}${2}" ;;
+ * ) func_dirname_result="${3}" ;;
+ esac
+}
+
+# func_basename file
+func_basename ()
+{
+ func_basename_result="${1##*/}"
+}
+
+# func_dirname_and_basename file append nondir_replacement
+# perform func_basename and func_dirname in a single function
+# call:
+# dirname: Compute the dirname of FILE. If nonempty,
+# add APPEND to the result, otherwise set result
+# to NONDIR_REPLACEMENT.
+# value returned in "$func_dirname_result"
+# basename: Compute filename of FILE.
+# value retuned in "$func_basename_result"
+# Implementation must be kept synchronized with func_dirname
+# and func_basename. For efficiency, we do not delegate to
+# those functions but instead duplicate the functionality here.
+func_dirname_and_basename ()
+{
+ case ${1} in
+ */*) func_dirname_result="${1%/*}${2}" ;;
+ * ) func_dirname_result="${3}" ;;
+ esac
+ func_basename_result="${1##*/}"
+}
+
+# func_stripname prefix suffix name
+# strip PREFIX and SUFFIX off of NAME.
+# PREFIX and SUFFIX must not contain globbing or regex special
+# characters, hashes, percent signs, but SUFFIX may contain a leading
+# dot (in which case that matches only a dot).
+func_stripname ()
+{
+ # pdksh 5.2.14 does not do ${X%$Y} correctly if both X and Y are
+ # positional parameters, so assign one to ordinary parameter first.
+ func_stripname_result=${3}
+ func_stripname_result=${func_stripname_result#"${1}"}
+ func_stripname_result=${func_stripname_result%"${2}"}
+}
+
+# func_opt_split
+func_opt_split ()
+{
+ func_opt_split_opt=${1%%=*}
+ func_opt_split_arg=${1#*=}
+}
+
+# func_lo2o object
+func_lo2o ()
+{
+ case ${1} in
+ *.lo) func_lo2o_result=${1%.lo}.${objext} ;;
+ *) func_lo2o_result=${1} ;;
+ esac
+}
+
+# func_xform libobj-or-source
+func_xform ()
+{
+ func_xform_result=${1%.*}.lo
+}
+
+# func_arith arithmetic-term...
+func_arith ()
+{
+ func_arith_result=$(( $[*] ))
+}
+
+# func_len string
+# STRING may not start with a hyphen.
+func_len ()
+{
+ func_len_result=${#1}
+}
+
+_LT_EOF
+ ;;
+ *) # Bourne compatible functions.
+ cat << \_LT_EOF >> "$cfgfile"
+
+# func_dirname file append nondir_replacement
+# Compute the dirname of FILE. If nonempty, add APPEND to the result,
+# otherwise set result to NONDIR_REPLACEMENT.
+func_dirname ()
+{
+ # Extract subdirectory from the argument.
+ func_dirname_result=`$ECHO "X${1}" | $Xsed -e "$dirname"`
+ if test "X$func_dirname_result" = "X${1}"; then
+ func_dirname_result="${3}"
+ else
+ func_dirname_result="$func_dirname_result${2}"
+ fi
+}
+
+# func_basename file
+func_basename ()
+{
+ func_basename_result=`$ECHO "X${1}" | $Xsed -e "$basename"`
+}
+
+dnl func_dirname_and_basename
+dnl A portable version of this function is already defined in general.m4sh
+dnl so there is no need for it here.
+
+# func_stripname prefix suffix name
+# strip PREFIX and SUFFIX off of NAME.
+# PREFIX and SUFFIX must not contain globbing or regex special
+# characters, hashes, percent signs, but SUFFIX may contain a leading
+# dot (in which case that matches only a dot).
+# func_strip_suffix prefix name
+func_stripname ()
+{
+ case ${2} in
+ .*) func_stripname_result=`$ECHO "X${3}" \
+ | $Xsed -e "s%^${1}%%" -e "s%\\\\${2}\$%%"`;;
+ *) func_stripname_result=`$ECHO "X${3}" \
+ | $Xsed -e "s%^${1}%%" -e "s%${2}\$%%"`;;
+ esac
+}
+
+# sed scripts:
+my_sed_long_opt='1s/^\(-[[^=]]*\)=.*/\1/;q'
+my_sed_long_arg='1s/^-[[^=]]*=//'
+
+# func_opt_split
+func_opt_split ()
+{
+ func_opt_split_opt=`$ECHO "X${1}" | $Xsed -e "$my_sed_long_opt"`
+ func_opt_split_arg=`$ECHO "X${1}" | $Xsed -e "$my_sed_long_arg"`
+}
+
+# func_lo2o object
+func_lo2o ()
+{
+ func_lo2o_result=`$ECHO "X${1}" | $Xsed -e "$lo2o"`
+}
+
+# func_xform libobj-or-source
+func_xform ()
+{
+ func_xform_result=`$ECHO "X${1}" | $Xsed -e 's/\.[[^.]]*$/.lo/'`
+}
+
+# func_arith arithmetic-term...
+func_arith ()
+{
+ func_arith_result=`expr "$[@]"`
+}
+
+# func_len string
+# STRING may not start with a hyphen.
+func_len ()
+{
+ func_len_result=`expr "$[1]" : ".*" 2>/dev/null || echo $max_cmd_len`
+}
+
+_LT_EOF
+esac
+
+case $lt_shell_append in
+ yes)
+ cat << \_LT_EOF >> "$cfgfile"
+
+# func_append var value
+# Append VALUE to the end of shell variable VAR.
+func_append ()
+{
+ eval "$[1]+=\$[2]"
+}
+_LT_EOF
+ ;;
+ *)
+ cat << \_LT_EOF >> "$cfgfile"
+
+# func_append var value
+# Append VALUE to the end of shell variable VAR.
+func_append ()
+{
+ eval "$[1]=\$$[1]\$[2]"
+}
+
+_LT_EOF
+ ;;
+ esac
+])
diff --git a/config/ltmain.sh b/config/ltmain.sh
new file mode 100755
index 0000000..a72f2fd
--- /dev/null
+++ b/config/ltmain.sh
@@ -0,0 +1,8406 @@
+# Generated from ltmain.m4sh.
+
+# ltmain.sh (GNU libtool) 2.2.6b
+# Written by Gordon Matzigkeit <gord at gnu.ai.mit.edu>, 1996
+
+# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, 2005, 2006, 2007 2008 Free Software Foundation, Inc.
+# This is free software; see the source for copying conditions. There is NO
+# warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+# GNU Libtool is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# As a special exception to the GNU General Public License,
+# if you distribute this file as part of a program or library that
+# is built using GNU Libtool, you may include this file under the
+# same distribution terms that you use for the rest of that program.
+#
+# GNU Libtool is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Libtool; see the file COPYING. If not, a copy
+# can be downloaded from http://www.gnu.org/licenses/gpl.html,
+# or obtained by writing to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+# Usage: $progname [OPTION]... [MODE-ARG]...
+#
+# Provide generalized library-building support services.
+#
+# --config show all configuration variables
+# --debug enable verbose shell tracing
+# -n, --dry-run display commands without modifying any files
+# --features display basic configuration information and exit
+# --mode=MODE use operation mode MODE
+# --preserve-dup-deps don't remove duplicate dependency libraries
+# --quiet, --silent don't print informational messages
+# --tag=TAG use configuration variables from tag TAG
+# -v, --verbose print informational messages (default)
+# --version print version information
+# -h, --help print short or long help message
+#
+# MODE must be one of the following:
+#
+# clean remove files from the build directory
+# compile compile a source file into a libtool object
+# execute automatically set library path, then run a program
+# finish complete the installation of libtool libraries
+# install install libraries or executables
+# link create a library or an executable
+# uninstall remove libraries from an installed directory
+#
+# MODE-ARGS vary depending on the MODE.
+# Try `$progname --help --mode=MODE' for a more detailed description of MODE.
+#
+# When reporting a bug, please describe a test case to reproduce it and
+# include the following information:
+#
+# host-triplet: $host
+# shell: $SHELL
+# compiler: $LTCC
+# compiler flags: $LTCFLAGS
+# linker: $LD (gnu? $with_gnu_ld)
+# $progname: (GNU libtool) 2.2.6b
+# automake: $automake_version
+# autoconf: $autoconf_version
+#
+# Report bugs to <bug-libtool at gnu.org>.
+
+PROGRAM=ltmain.sh
+PACKAGE=libtool
+VERSION=2.2.6b
+TIMESTAMP=""
+package_revision=1.3017
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+ emulate sh
+ NULLCMD=:
+ # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+ # is contrary to our usage. Disable this feature.
+ alias -g '${1+"$@"}'='"$@"'
+ setopt NO_GLOB_SUBST
+else
+ case `(set -o) 2>/dev/null` in *posix*) set -o posix;; esac
+fi
+BIN_SH=xpg4; export BIN_SH # for Tru64
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# NLS nuisances: We save the old values to restore during execute mode.
+# Only set LANG and LC_ALL to C if already set.
+# These must not be set unconditionally because not all systems understand
+# e.g. LANG=C (notably SCO).
+lt_user_locale=
+lt_safe_locale=
+for lt_var in LANG LANGUAGE LC_ALL LC_CTYPE LC_COLLATE LC_MESSAGES
+do
+ eval "if test \"\${$lt_var+set}\" = set; then
+ save_$lt_var=\$$lt_var
+ $lt_var=C
+ export $lt_var
+ lt_user_locale=\"$lt_var=\\\$save_\$lt_var; \$lt_user_locale\"
+ lt_safe_locale=\"$lt_var=C; \$lt_safe_locale\"
+ fi"
+done
+
+$lt_unset CDPATH
+
+
+
+
+
+: ${CP="cp -f"}
+: ${ECHO="echo"}
+: ${EGREP="/bin/grep -E"}
+: ${FGREP="/bin/grep -F"}
+: ${GREP="/bin/grep"}
+: ${LN_S="ln -s"}
+: ${MAKE="make"}
+: ${MKDIR="mkdir"}
+: ${MV="mv -f"}
+: ${RM="rm -f"}
+: ${SED="/bin/sed"}
+: ${SHELL="${CONFIG_SHELL-/bin/sh}"}
+: ${Xsed="$SED -e 1s/^X//"}
+
+# Global variables:
+EXIT_SUCCESS=0
+EXIT_FAILURE=1
+EXIT_MISMATCH=63 # $? = 63 is used to indicate version mismatch to missing.
+EXIT_SKIP=77 # $? = 77 is used to indicate a skipped test to automake.
+
+exit_status=$EXIT_SUCCESS
+
+# Make sure IFS has a sensible default
+lt_nl='
+'
+IFS=" $lt_nl"
+
+dirname="s,/[^/]*$,,"
+basename="s,^.*/,,"
+
+# func_dirname_and_basename file append nondir_replacement
+# perform func_basename and func_dirname in a single function
+# call:
+# dirname: Compute the dirname of FILE. If nonempty,
+# add APPEND to the result, otherwise set result
+# to NONDIR_REPLACEMENT.
+# value returned in "$func_dirname_result"
+# basename: Compute filename of FILE.
+# value retuned in "$func_basename_result"
+# Implementation must be kept synchronized with func_dirname
+# and func_basename. For efficiency, we do not delegate to
+# those functions but instead duplicate the functionality here.
+func_dirname_and_basename ()
+{
+ # Extract subdirectory from the argument.
+ func_dirname_result=`$ECHO "X${1}" | $Xsed -e "$dirname"`
+ if test "X$func_dirname_result" = "X${1}"; then
+ func_dirname_result="${3}"
+ else
+ func_dirname_result="$func_dirname_result${2}"
+ fi
+ func_basename_result=`$ECHO "X${1}" | $Xsed -e "$basename"`
+}
+
+# Generated shell functions inserted here.
+
+# Work around backward compatibility issue on IRIX 6.5. On IRIX 6.4+, sh
+# is ksh but when the shell is invoked as "sh" and the current value of
+# the _XPG environment variable is not equal to 1 (one), the special
+# positional parameter $0, within a function call, is the name of the
+# function.
+progpath="$0"
+
+# The name of this program:
+# In the unlikely event $progname began with a '-', it would play havoc with
+# func_echo (imagine progname=-n), so we prepend ./ in that case:
+func_dirname_and_basename "$progpath"
+progname=$func_basename_result
+case $progname in
+ -*) progname=./$progname ;;
+esac
+
+# Make sure we have an absolute path for reexecution:
+case $progpath in
+ [\\/]*|[A-Za-z]:\\*) ;;
+ *[\\/]*)
+ progdir=$func_dirname_result
+ progdir=`cd "$progdir" && pwd`
+ progpath="$progdir/$progname"
+ ;;
+ *)
+ save_IFS="$IFS"
+ IFS=:
+ for progdir in $PATH; do
+ IFS="$save_IFS"
+ test -x "$progdir/$progname" && break
+ done
+ IFS="$save_IFS"
+ test -n "$progdir" || progdir=`pwd`
+ progpath="$progdir/$progname"
+ ;;
+esac
+
+# Sed substitution that helps us do robust quoting. It backslashifies
+# metacharacters that are still active within double-quoted strings.
+Xsed="${SED}"' -e 1s/^X//'
+sed_quote_subst='s/\([`"$\\]\)/\\\1/g'
+
+# Same as above, but do not quote variable references.
+double_quote_subst='s/\(["`\\]\)/\\\1/g'
+
+# Re-`\' parameter expansions in output of double_quote_subst that were
+# `\'-ed in input to the same. If an odd number of `\' preceded a '$'
+# in input to double_quote_subst, that '$' was protected from expansion.
+# Since each input `\' is now two `\'s, look for any number of runs of
+# four `\'s followed by two `\'s and then a '$'. `\' that '$'.
+bs='\\'
+bs2='\\\\'
+bs4='\\\\\\\\'
+dollar='\$'
+sed_double_backslash="\
+ s/$bs4/&\\
+/g
+ s/^$bs2$dollar/$bs&/
+ s/\\([^$bs]\\)$bs2$dollar/\\1$bs2$bs$dollar/g
+ s/\n//g"
+
+# Standard options:
+opt_dry_run=false
+opt_help=false
+opt_quiet=false
+opt_verbose=false
+opt_warning=:
+
+# func_echo arg...
+# Echo program name prefixed message, along with the current mode
+# name if it has been set yet.
+func_echo ()
+{
+ $ECHO "$progname${mode+: }$mode: $*"
+}
+
+# func_verbose arg...
+# Echo program name prefixed message in verbose mode only.
+func_verbose ()
+{
+ $opt_verbose && func_echo ${1+"$@"}
+
+ # A bug in bash halts the script if the last line of a function
+ # fails when set -e is in force, so we need another command to
+ # work around that:
+ :
+}
+
+# func_error arg...
+# Echo program name prefixed message to standard error.
+func_error ()
+{
+ $ECHO "$progname${mode+: }$mode: "${1+"$@"} 1>&2
+}
+
+# func_warning arg...
+# Echo program name prefixed warning message to standard error.
+func_warning ()
+{
+ $opt_warning && $ECHO "$progname${mode+: }$mode: warning: "${1+"$@"} 1>&2
+
+ # bash bug again:
+ :
+}
+
+# func_fatal_error arg...
+# Echo program name prefixed message to standard error, and exit.
+func_fatal_error ()
+{
+ func_error ${1+"$@"}
+ exit $EXIT_FAILURE
+}
+
+# func_fatal_help arg...
+# Echo program name prefixed message to standard error, followed by
+# a help hint, and exit.
+func_fatal_help ()
+{
+ func_error ${1+"$@"}
+ func_fatal_error "$help"
+}
+help="Try \`$progname --help' for more information." ## default
+
+
+# func_grep expression filename
+# Check whether EXPRESSION matches any line of FILENAME, without output.
+func_grep ()
+{
+ $GREP "$1" "$2" >/dev/null 2>&1
+}
+
+
+# func_mkdir_p directory-path
+# Make sure the entire path to DIRECTORY-PATH is available.
+func_mkdir_p ()
+{
+ my_directory_path="$1"
+ my_dir_list=
+
+ if test -n "$my_directory_path" && test "$opt_dry_run" != ":"; then
+
+ # Protect directory names starting with `-'
+ case $my_directory_path in
+ -*) my_directory_path="./$my_directory_path" ;;
+ esac
+
+ # While some portion of DIR does not yet exist...
+ while test ! -d "$my_directory_path"; do
+ # ...make a list in topmost first order. Use a colon delimited
+ # list incase some portion of path contains whitespace.
+ my_dir_list="$my_directory_path:$my_dir_list"
+
+ # If the last portion added has no slash in it, the list is done
+ case $my_directory_path in */*) ;; *) break ;; esac
+
+ # ...otherwise throw away the child directory and loop
+ my_directory_path=`$ECHO "X$my_directory_path" | $Xsed -e "$dirname"`
+ done
+ my_dir_list=`$ECHO "X$my_dir_list" | $Xsed -e 's,:*$,,'`
+
+ save_mkdir_p_IFS="$IFS"; IFS=':'
+ for my_dir in $my_dir_list; do
+ IFS="$save_mkdir_p_IFS"
+ # mkdir can fail with a `File exist' error if two processes
+ # try to create one of the directories concurrently. Don't
+ # stop in that case!
+ $MKDIR "$my_dir" 2>/dev/null || :
+ done
+ IFS="$save_mkdir_p_IFS"
+
+ # Bail out if we (or some other process) failed to create a directory.
+ test -d "$my_directory_path" || \
+ func_fatal_error "Failed to create \`$1'"
+ fi
+}
+
+
+# func_mktempdir [string]
+# Make a temporary directory that won't clash with other running
+# libtool processes, and avoids race conditions if possible. If
+# given, STRING is the basename for that directory.
+func_mktempdir ()
+{
+ my_template="${TMPDIR-/tmp}/${1-$progname}"
+
+ if test "$opt_dry_run" = ":"; then
+ # Return a directory name, but don't create it in dry-run mode
+ my_tmpdir="${my_template}-$$"
+ else
+
+ # If mktemp works, use that first and foremost
+ my_tmpdir=`mktemp -d "${my_template}-XXXXXXXX" 2>/dev/null`
+
+ if test ! -d "$my_tmpdir"; then
+ # Failing that, at least try and use $RANDOM to avoid a race
+ my_tmpdir="${my_template}-${RANDOM-0}$$"
+
+ save_mktempdir_umask=`umask`
+ umask 0077
+ $MKDIR "$my_tmpdir"
+ umask $save_mktempdir_umask
+ fi
+
+ # If we're not in dry-run mode, bomb out on failure
+ test -d "$my_tmpdir" || \
+ func_fatal_error "cannot create temporary directory \`$my_tmpdir'"
+ fi
+
+ $ECHO "X$my_tmpdir" | $Xsed
+}
+
+
+# func_quote_for_eval arg
+# Aesthetically quote ARG to be evaled later.
+# This function returns two values: FUNC_QUOTE_FOR_EVAL_RESULT
+# is double-quoted, suitable for a subsequent eval, whereas
+# FUNC_QUOTE_FOR_EVAL_UNQUOTED_RESULT has merely all characters
+# which are still active within double quotes backslashified.
+func_quote_for_eval ()
+{
+ case $1 in
+ *[\\\`\"\$]*)
+ func_quote_for_eval_unquoted_result=`$ECHO "X$1" | $Xsed -e "$sed_quote_subst"` ;;
+ *)
+ func_quote_for_eval_unquoted_result="$1" ;;
+ esac
+
+ case $func_quote_for_eval_unquoted_result in
+ # Double-quote args containing shell metacharacters to delay
+ # word splitting, command substitution and and variable
+ # expansion for a subsequent eval.
+ # Many Bourne shells cannot handle close brackets correctly
+ # in scan sets, so we specify it separately.
+ *[\[\~\#\^\&\*\(\)\{\}\|\;\<\>\?\'\ \ ]*|*]*|"")
+ func_quote_for_eval_result="\"$func_quote_for_eval_unquoted_result\""
+ ;;
+ *)
+ func_quote_for_eval_result="$func_quote_for_eval_unquoted_result"
+ esac
+}
+
+
+# func_quote_for_expand arg
+# Aesthetically quote ARG to be evaled later; same as above,
+# but do not quote variable references.
+func_quote_for_expand ()
+{
+ case $1 in
+ *[\\\`\"]*)
+ my_arg=`$ECHO "X$1" | $Xsed \
+ -e "$double_quote_subst" -e "$sed_double_backslash"` ;;
+ *)
+ my_arg="$1" ;;
+ esac
+
+ case $my_arg in
+ # Double-quote args containing shell metacharacters to delay
+ # word splitting and command substitution for a subsequent eval.
+ # Many Bourne shells cannot handle close brackets correctly
+ # in scan sets, so we specify it separately.
+ *[\[\~\#\^\&\*\(\)\{\}\|\;\<\>\?\'\ \ ]*|*]*|"")
+ my_arg="\"$my_arg\""
+ ;;
+ esac
+
+ func_quote_for_expand_result="$my_arg"
+}
+
+
+# func_show_eval cmd [fail_exp]
+# Unless opt_silent is true, then output CMD. Then, if opt_dryrun is
+# not true, evaluate CMD. If the evaluation of CMD fails, and FAIL_EXP
+# is given, then evaluate it.
+func_show_eval ()
+{
+ my_cmd="$1"
+ my_fail_exp="${2-:}"
+
+ ${opt_silent-false} || {
+ func_quote_for_expand "$my_cmd"
+ eval "func_echo $func_quote_for_expand_result"
+ }
+
+ if ${opt_dry_run-false}; then :; else
+ eval "$my_cmd"
+ my_status=$?
+ if test "$my_status" -eq 0; then :; else
+ eval "(exit $my_status); $my_fail_exp"
+ fi
+ fi
+}
+
+
+# func_show_eval_locale cmd [fail_exp]
+# Unless opt_silent is true, then output CMD. Then, if opt_dryrun is
+# not true, evaluate CMD. If the evaluation of CMD fails, and FAIL_EXP
+# is given, then evaluate it. Use the saved locale for evaluation.
+func_show_eval_locale ()
+{
+ my_cmd="$1"
+ my_fail_exp="${2-:}"
+
+ ${opt_silent-false} || {
+ func_quote_for_expand "$my_cmd"
+ eval "func_echo $func_quote_for_expand_result"
+ }
+
+ if ${opt_dry_run-false}; then :; else
+ eval "$lt_user_locale
+ $my_cmd"
+ my_status=$?
+ eval "$lt_safe_locale"
+ if test "$my_status" -eq 0; then :; else
+ eval "(exit $my_status); $my_fail_exp"
+ fi
+ fi
+}
+
+
+
+
+
+# func_version
+# Echo version message to standard output and exit.
+func_version ()
+{
+ $SED -n '/^# '$PROGRAM' (GNU /,/# warranty; / {
+ s/^# //
+ s/^# *$//
+ s/\((C)\)[ 0-9,-]*\( [1-9][0-9]*\)/\1\2/
+ p
+ }' < "$progpath"
+ exit $?
+}
+
+# func_usage
+# Echo short help message to standard output and exit.
+func_usage ()
+{
+ $SED -n '/^# Usage:/,/# -h/ {
+ s/^# //
+ s/^# *$//
+ s/\$progname/'$progname'/
+ p
+ }' < "$progpath"
+ $ECHO
+ $ECHO "run \`$progname --help | more' for full usage"
+ exit $?
+}
+
+# func_help
+# Echo long help message to standard output and exit.
+func_help ()
+{
+ $SED -n '/^# Usage:/,/# Report bugs to/ {
+ s/^# //
+ s/^# *$//
+ s*\$progname*'$progname'*
+ s*\$host*'"$host"'*
+ s*\$SHELL*'"$SHELL"'*
+ s*\$LTCC*'"$LTCC"'*
+ s*\$LTCFLAGS*'"$LTCFLAGS"'*
+ s*\$LD*'"$LD"'*
+ s/\$with_gnu_ld/'"$with_gnu_ld"'/
+ s/\$automake_version/'"`(automake --version) 2>/dev/null |$SED 1q`"'/
+ s/\$autoconf_version/'"`(autoconf --version) 2>/dev/null |$SED 1q`"'/
+ p
+ }' < "$progpath"
+ exit $?
+}
+
+# func_missing_arg argname
+# Echo program name prefixed message to standard error and set global
+# exit_cmd.
+func_missing_arg ()
+{
+ func_error "missing argument for $1"
+ exit_cmd=exit
+}
+
+exit_cmd=:
+
+
+
+
+
+# Check that we have a working $ECHO.
+if test "X$1" = X--no-reexec; then
+ # Discard the --no-reexec flag, and continue.
+ shift
+elif test "X$1" = X--fallback-echo; then
+ # Avoid inline document here, it may be left over
+ :
+elif test "X`{ $ECHO '\t'; } 2>/dev/null`" = 'X\t'; then
+ # Yippee, $ECHO works!
+ :
+else
+ # Restart under the correct shell, and then maybe $ECHO will work.
+ exec $SHELL "$progpath" --no-reexec ${1+"$@"}
+fi
+
+if test "X$1" = X--fallback-echo; then
+ # used as fallback echo
+ shift
+ cat <<EOF
+$*
+EOF
+ exit $EXIT_SUCCESS
+fi
+
+magic="%%%MAGIC variable%%%"
+magic_exe="%%%MAGIC EXE variable%%%"
+
+# Global variables.
+# $mode is unset
+nonopt=
+execute_dlfiles=
+preserve_args=
+lo2o="s/\\.lo\$/.${objext}/"
+o2lo="s/\\.${objext}\$/.lo/"
+extracted_archives=
+extracted_serial=0
+
+opt_dry_run=false
+opt_duplicate_deps=false
+opt_silent=false
+opt_debug=:
+
+# If this variable is set in any of the actions, the command in it
+# will be execed at the end. This prevents here-documents from being
+# left over by shells.
+exec_cmd=
+
+# func_fatal_configuration arg...
+# Echo program name prefixed message to standard error, followed by
+# a configuration failure hint, and exit.
+func_fatal_configuration ()
+{
+ func_error ${1+"$@"}
+ func_error "See the $PACKAGE documentation for more information."
+ func_fatal_error "Fatal configuration error."
+}
+
+
+# func_config
+# Display the configuration for all the tags in this script.
+func_config ()
+{
+ re_begincf='^# ### BEGIN LIBTOOL'
+ re_endcf='^# ### END LIBTOOL'
+
+ # Default configuration.
+ $SED "1,/$re_begincf CONFIG/d;/$re_endcf CONFIG/,\$d" < "$progpath"
+
+ # Now print the configurations for the tags.
+ for tagname in $taglist; do
+ $SED -n "/$re_begincf TAG CONFIG: $tagname\$/,/$re_endcf TAG CONFIG: $tagname\$/p" < "$progpath"
+ done
+
+ exit $?
+}
+
+# func_features
+# Display the features supported by this script.
+func_features ()
+{
+ $ECHO "host: $host"
+ if test "$build_libtool_libs" = yes; then
+ $ECHO "enable shared libraries"
+ else
+ $ECHO "disable shared libraries"
+ fi
+ if test "$build_old_libs" = yes; then
+ $ECHO "enable static libraries"
+ else
+ $ECHO "disable static libraries"
+ fi
+
+ exit $?
+}
+
+# func_enable_tag tagname
+# Verify that TAGNAME is valid, and either flag an error and exit, or
+# enable the TAGNAME tag. We also add TAGNAME to the global $taglist
+# variable here.
+func_enable_tag ()
+{
+ # Global variable:
+ tagname="$1"
+
+ re_begincf="^# ### BEGIN LIBTOOL TAG CONFIG: $tagname\$"
+ re_endcf="^# ### END LIBTOOL TAG CONFIG: $tagname\$"
+ sed_extractcf="/$re_begincf/,/$re_endcf/p"
+
+ # Validate tagname.
+ case $tagname in
+ *[!-_A-Za-z0-9,/]*)
+ func_fatal_error "invalid tag name: $tagname"
+ ;;
+ esac
+
+ # Don't test for the "default" C tag, as we know it's
+ # there but not specially marked.
+ case $tagname in
+ CC) ;;
+ *)
+ if $GREP "$re_begincf" "$progpath" >/dev/null 2>&1; then
+ taglist="$taglist $tagname"
+
+ # Evaluate the configuration. Be careful to quote the path
+ # and the sed script, to avoid splitting on whitespace, but
+ # also don't use non-portable quotes within backquotes within
+ # quotes we have to do it in 2 steps:
+ extractedcf=`$SED -n -e "$sed_extractcf" < "$progpath"`
+ eval "$extractedcf"
+ else
+ func_error "ignoring unknown tag $tagname"
+ fi
+ ;;
+ esac
+}
+
+# Parse options once, thoroughly. This comes as soon as possible in
+# the script to make things like `libtool --version' happen quickly.
+{
+
+ # Shorthand for --mode=foo, only valid as the first argument
+ case $1 in
+ clean|clea|cle|cl)
+ shift; set dummy --mode clean ${1+"$@"}; shift
+ ;;
+ compile|compil|compi|comp|com|co|c)
+ shift; set dummy --mode compile ${1+"$@"}; shift
+ ;;
+ execute|execut|execu|exec|exe|ex|e)
+ shift; set dummy --mode execute ${1+"$@"}; shift
+ ;;
+ finish|finis|fini|fin|fi|f)
+ shift; set dummy --mode finish ${1+"$@"}; shift
+ ;;
+ install|instal|insta|inst|ins|in|i)
+ shift; set dummy --mode install ${1+"$@"}; shift
+ ;;
+ link|lin|li|l)
+ shift; set dummy --mode link ${1+"$@"}; shift
+ ;;
+ uninstall|uninstal|uninsta|uninst|unins|unin|uni|un|u)
+ shift; set dummy --mode uninstall ${1+"$@"}; shift
+ ;;
+ esac
+
+ # Parse non-mode specific arguments:
+ while test "$#" -gt 0; do
+ opt="$1"
+ shift
+
+ case $opt in
+ --config) func_config ;;
+
+ --debug) preserve_args="$preserve_args $opt"
+ func_echo "enabling shell trace mode"
+ opt_debug='set -x'
+ $opt_debug
+ ;;
+
+ -dlopen) test "$#" -eq 0 && func_missing_arg "$opt" && break
+ execute_dlfiles="$execute_dlfiles $1"
+ shift
+ ;;
+
+ --dry-run | -n) opt_dry_run=: ;;
+ --features) func_features ;;
+ --finish) mode="finish" ;;
+
+ --mode) test "$#" -eq 0 && func_missing_arg "$opt" && break
+ case $1 in
+ # Valid mode arguments:
+ clean) ;;
+ compile) ;;
+ execute) ;;
+ finish) ;;
+ install) ;;
+ link) ;;
+ relink) ;;
+ uninstall) ;;
+
+ # Catch anything else as an error
+ *) func_error "invalid argument for $opt"
+ exit_cmd=exit
+ break
+ ;;
+ esac
+
+ mode="$1"
+ shift
+ ;;
+
+ --preserve-dup-deps)
+ opt_duplicate_deps=: ;;
+
+ --quiet|--silent) preserve_args="$preserve_args $opt"
+ opt_silent=:
+ ;;
+
+ --verbose| -v) preserve_args="$preserve_args $opt"
+ opt_silent=false
+ ;;
+
+ --tag) test "$#" -eq 0 && func_missing_arg "$opt" && break
+ preserve_args="$preserve_args $opt $1"
+ func_enable_tag "$1" # tagname is set here
+ shift
+ ;;
+
+ # Separate optargs to long options:
+ -dlopen=*|--mode=*|--tag=*)
+ func_opt_split "$opt"
+ set dummy "$func_opt_split_opt" "$func_opt_split_arg" ${1+"$@"}
+ shift
+ ;;
+
+ -\?|-h) func_usage ;;
+ --help) opt_help=: ;;
+ --version) func_version ;;
+
+ -*) func_fatal_help "unrecognized option \`$opt'" ;;
+
+ *) nonopt="$opt"
+ break
+ ;;
+ esac
+ done
+
+
+ case $host in
+ *cygwin* | *mingw* | *pw32* | *cegcc*)
+ # don't eliminate duplications in $postdeps and $predeps
+ opt_duplicate_compiler_generated_deps=:
+ ;;
+ *)
+ opt_duplicate_compiler_generated_deps=$opt_duplicate_deps
+ ;;
+ esac
+
+ # Having warned about all mis-specified options, bail out if
+ # anything was wrong.
+ $exit_cmd $EXIT_FAILURE
+}
+
+# func_check_version_match
+# Ensure that we are using m4 macros, and libtool script from the same
+# release of libtool.
+func_check_version_match ()
+{
+ if test "$package_revision" != "$macro_revision"; then
+ if test "$VERSION" != "$macro_version"; then
+ if test -z "$macro_version"; then
+ cat >&2 <<_LT_EOF
+$progname: Version mismatch error. This is $PACKAGE $VERSION, but the
+$progname: definition of this LT_INIT comes from an older release.
+$progname: You should recreate aclocal.m4 with macros from $PACKAGE $VERSION
+$progname: and run autoconf again.
+_LT_EOF
+ else
+ cat >&2 <<_LT_EOF
+$progname: Version mismatch error. This is $PACKAGE $VERSION, but the
+$progname: definition of this LT_INIT comes from $PACKAGE $macro_version.
+$progname: You should recreate aclocal.m4 with macros from $PACKAGE $VERSION
+$progname: and run autoconf again.
+_LT_EOF
+ fi
+ else
+ cat >&2 <<_LT_EOF
+$progname: Version mismatch error. This is $PACKAGE $VERSION, revision $package_revision,
+$progname: but the definition of this LT_INIT comes from revision $macro_revision.
+$progname: You should recreate aclocal.m4 with macros from revision $package_revision
+$progname: of $PACKAGE $VERSION and run autoconf again.
+_LT_EOF
+ fi
+
+ exit $EXIT_MISMATCH
+ fi
+}
+
+
+## ----------- ##
+## Main. ##
+## ----------- ##
+
+$opt_help || {
+ # Sanity checks first:
+ func_check_version_match
+
+ if test "$build_libtool_libs" != yes && test "$build_old_libs" != yes; then
+ func_fatal_configuration "not configured to build any kind of library"
+ fi
+
+ test -z "$mode" && func_fatal_error "error: you must specify a MODE."
+
+
+ # Darwin sucks
+ eval std_shrext=\"$shrext_cmds\"
+
+
+ # Only execute mode is allowed to have -dlopen flags.
+ if test -n "$execute_dlfiles" && test "$mode" != execute; then
+ func_error "unrecognized option \`-dlopen'"
+ $ECHO "$help" 1>&2
+ exit $EXIT_FAILURE
+ fi
+
+ # Change the help message to a mode-specific one.
+ generic_help="$help"
+ help="Try \`$progname --help --mode=$mode' for more information."
+}
+
+
+# func_lalib_p file
+# True iff FILE is a libtool `.la' library or `.lo' object file.
+# This function is only a basic sanity check; it will hardly flush out
+# determined imposters.
+func_lalib_p ()
+{
+ test -f "$1" &&
+ $SED -e 4q "$1" 2>/dev/null \
+ | $GREP "^# Generated by .*$PACKAGE" > /dev/null 2>&1
+}
+
+# func_lalib_unsafe_p file
+# True iff FILE is a libtool `.la' library or `.lo' object file.
+# This function implements the same check as func_lalib_p without
+# resorting to external programs. To this end, it redirects stdin and
+# closes it afterwards, without saving the original file descriptor.
+# As a safety measure, use it only where a negative result would be
+# fatal anyway. Works if `file' does not exist.
+func_lalib_unsafe_p ()
+{
+ lalib_p=no
+ if test -f "$1" && test -r "$1" && exec 5<&0 <"$1"; then
+ for lalib_p_l in 1 2 3 4
+ do
+ read lalib_p_line
+ case "$lalib_p_line" in
+ \#\ Generated\ by\ *$PACKAGE* ) lalib_p=yes; break;;
+ esac
+ done
+ exec 0<&5 5<&-
+ fi
+ test "$lalib_p" = yes
+}
+
+# func_ltwrapper_script_p file
+# True iff FILE is a libtool wrapper script
+# This function is only a basic sanity check; it will hardly flush out
+# determined imposters.
+func_ltwrapper_script_p ()
+{
+ func_lalib_p "$1"
+}
+
+# func_ltwrapper_executable_p file
+# True iff FILE is a libtool wrapper executable
+# This function is only a basic sanity check; it will hardly flush out
+# determined imposters.
+func_ltwrapper_executable_p ()
+{
+ func_ltwrapper_exec_suffix=
+ case $1 in
+ *.exe) ;;
+ *) func_ltwrapper_exec_suffix=.exe ;;
+ esac
+ $GREP "$magic_exe" "$1$func_ltwrapper_exec_suffix" >/dev/null 2>&1
+}
+
+# func_ltwrapper_scriptname file
+# Assumes file is an ltwrapper_executable
+# uses $file to determine the appropriate filename for a
+# temporary ltwrapper_script.
+func_ltwrapper_scriptname ()
+{
+ func_ltwrapper_scriptname_result=""
+ if func_ltwrapper_executable_p "$1"; then
+ func_dirname_and_basename "$1" "" "."
+ func_stripname '' '.exe' "$func_basename_result"
+ func_ltwrapper_scriptname_result="$func_dirname_result/$objdir/${func_stripname_result}_ltshwrapper"
+ fi
+}
+
+# func_ltwrapper_p file
+# True iff FILE is a libtool wrapper script or wrapper executable
+# This function is only a basic sanity check; it will hardly flush out
+# determined imposters.
+func_ltwrapper_p ()
+{
+ func_ltwrapper_script_p "$1" || func_ltwrapper_executable_p "$1"
+}
+
+
+# func_execute_cmds commands fail_cmd
+# Execute tilde-delimited COMMANDS.
+# If FAIL_CMD is given, eval that upon failure.
+# FAIL_CMD may read-access the current command in variable CMD!
+func_execute_cmds ()
+{
+ $opt_debug
+ save_ifs=$IFS; IFS='~'
+ for cmd in $1; do
+ IFS=$save_ifs
+ eval cmd=\"$cmd\"
+ func_show_eval "$cmd" "${2-:}"
+ done
+ IFS=$save_ifs
+}
+
+
+# func_source file
+# Source FILE, adding directory component if necessary.
+# Note that it is not necessary on cygwin/mingw to append a dot to
+# FILE even if both FILE and FILE.exe exist: automatic-append-.exe
+# behavior happens only for exec(3), not for open(2)! Also, sourcing
+# `FILE.' does not work on cygwin managed mounts.
+func_source ()
+{
+ $opt_debug
+ case $1 in
+ */* | *\\*) . "$1" ;;
+ *) . "./$1" ;;
+ esac
+}
+
+
+# func_infer_tag arg
+# Infer tagged configuration to use if any are available and
+# if one wasn't chosen via the "--tag" command line option.
+# Only attempt this if the compiler in the base compile
+# command doesn't match the default compiler.
+# arg is usually of the form 'gcc ...'
+func_infer_tag ()
+{
+ $opt_debug
+ if test -n "$available_tags" && test -z "$tagname"; then
+ CC_quoted=
+ for arg in $CC; do
+ func_quote_for_eval "$arg"
+ CC_quoted="$CC_quoted $func_quote_for_eval_result"
+ done
+ case $@ in
+ # Blanks in the command may have been stripped by the calling shell,
+ # but not from the CC environment variable when configure was run.
+ " $CC "* | "$CC "* | " `$ECHO $CC` "* | "`$ECHO $CC` "* | " $CC_quoted"* | "$CC_quoted "* | " `$ECHO $CC_quoted` "* | "`$ECHO $CC_quoted` "*) ;;
+ # Blanks at the start of $base_compile will cause this to fail
+ # if we don't check for them as well.
+ *)
+ for z in $available_tags; do
+ if $GREP "^# ### BEGIN LIBTOOL TAG CONFIG: $z$" < "$progpath" > /dev/null; then
+ # Evaluate the configuration.
+ eval "`${SED} -n -e '/^# ### BEGIN LIBTOOL TAG CONFIG: '$z'$/,/^# ### END LIBTOOL TAG CONFIG: '$z'$/p' < $progpath`"
+ CC_quoted=
+ for arg in $CC; do
+ # Double-quote args containing other shell metacharacters.
+ func_quote_for_eval "$arg"
+ CC_quoted="$CC_quoted $func_quote_for_eval_result"
+ done
+ case "$@ " in
+ " $CC "* | "$CC "* | " `$ECHO $CC` "* | "`$ECHO $CC` "* | " $CC_quoted"* | "$CC_quoted "* | " `$ECHO $CC_quoted` "* | "`$ECHO $CC_quoted` "*)
+ # The compiler in the base compile command matches
+ # the one in the tagged configuration.
+ # Assume this is the tagged configuration we want.
+ tagname=$z
+ break
+ ;;
+ esac
+ fi
+ done
+ # If $tagname still isn't set, then no tagged configuration
+ # was found and let the user know that the "--tag" command
+ # line option must be used.
+ if test -z "$tagname"; then
+ func_echo "unable to infer tagged configuration"
+ func_fatal_error "specify a tag with \`--tag'"
+# else
+# func_verbose "using $tagname tagged configuration"
+ fi
+ ;;
+ esac
+ fi
+}
+
+
+
+# func_write_libtool_object output_name pic_name nonpic_name
+# Create a libtool object file (analogous to a ".la" file),
+# but don't create it if we're doing a dry run.
+func_write_libtool_object ()
+{
+ write_libobj=${1}
+ if test "$build_libtool_libs" = yes; then
+ write_lobj=\'${2}\'
+ else
+ write_lobj=none
+ fi
+
+ if test "$build_old_libs" = yes; then
+ write_oldobj=\'${3}\'
+ else
+ write_oldobj=none
+ fi
+
+ $opt_dry_run || {
+ cat >${write_libobj}T <<EOF
+# $write_libobj - a libtool object file
+# Generated by $PROGRAM (GNU $PACKAGE$TIMESTAMP) $VERSION
+#
+# Please DO NOT delete this file!
+# It is necessary for linking the library.
+
+# Name of the PIC object.
+pic_object=$write_lobj
+
+# Name of the non-PIC object
+non_pic_object=$write_oldobj
+
+EOF
+ $MV "${write_libobj}T" "${write_libobj}"
+ }
+}
+
+# func_mode_compile arg...
+func_mode_compile ()
+{
+ $opt_debug
+ # Get the compilation command and the source file.
+ base_compile=
+ srcfile="$nonopt" # always keep a non-empty value in "srcfile"
+ suppress_opt=yes
+ suppress_output=
+ arg_mode=normal
+ libobj=
+ later=
+ pie_flag=
+
+ for arg
+ do
+ case $arg_mode in
+ arg )
+ # do not "continue". Instead, add this to base_compile
+ lastarg="$arg"
+ arg_mode=normal
+ ;;
+
+ target )
+ libobj="$arg"
+ arg_mode=normal
+ continue
+ ;;
+
+ normal )
+ # Accept any command-line options.
+ case $arg in
+ -o)
+ test -n "$libobj" && \
+ func_fatal_error "you cannot specify \`-o' more than once"
+ arg_mode=target
+ continue
+ ;;
+
+ -pie | -fpie | -fPIE)
+ pie_flag="$pie_flag $arg"
+ continue
+ ;;
+
+ -shared | -static | -prefer-pic | -prefer-non-pic)
+ later="$later $arg"
+ continue
+ ;;
+
+ -no-suppress)
+ suppress_opt=no
+ continue
+ ;;
+
+ -Xcompiler)
+ arg_mode=arg # the next one goes into the "base_compile" arg list
+ continue # The current "srcfile" will either be retained or
+ ;; # replaced later. I would guess that would be a bug.
+
+ -Wc,*)
+ func_stripname '-Wc,' '' "$arg"
+ args=$func_stripname_result
+ lastarg=
+ save_ifs="$IFS"; IFS=','
+ for arg in $args; do
+ IFS="$save_ifs"
+ func_quote_for_eval "$arg"
+ lastarg="$lastarg $func_quote_for_eval_result"
+ done
+ IFS="$save_ifs"
+ func_stripname ' ' '' "$lastarg"
+ lastarg=$func_stripname_result
+
+ # Add the arguments to base_compile.
+ base_compile="$base_compile $lastarg"
+ continue
+ ;;
+
+ *)
+ # Accept the current argument as the source file.
+ # The previous "srcfile" becomes the current argument.
+ #
+ lastarg="$srcfile"
+ srcfile="$arg"
+ ;;
+ esac # case $arg
+ ;;
+ esac # case $arg_mode
+
+ # Aesthetically quote the previous argument.
+ func_quote_for_eval "$lastarg"
+ base_compile="$base_compile $func_quote_for_eval_result"
+ done # for arg
+
+ case $arg_mode in
+ arg)
+ func_fatal_error "you must specify an argument for -Xcompile"
+ ;;
+ target)
+ func_fatal_error "you must specify a target with \`-o'"
+ ;;
+ *)
+ # Get the name of the library object.
+ test -z "$libobj" && {
+ func_basename "$srcfile"
+ libobj="$func_basename_result"
+ }
+ ;;
+ esac
+
+ # Recognize several different file suffixes.
+ # If the user specifies -o file.o, it is replaced with file.lo
+ case $libobj in
+ *.[cCFSifmso] | \
+ *.ada | *.adb | *.ads | *.asm | \
+ *.c++ | *.cc | *.ii | *.class | *.cpp | *.cxx | \
+ *.[fF][09]? | *.for | *.java | *.obj | *.sx)
+ func_xform "$libobj"
+ libobj=$func_xform_result
+ ;;
+ esac
+
+ case $libobj in
+ *.lo) func_lo2o "$libobj"; obj=$func_lo2o_result ;;
+ *)
+ func_fatal_error "cannot determine name of library object from \`$libobj'"
+ ;;
+ esac
+
+ func_infer_tag $base_compile
+
+ for arg in $later; do
+ case $arg in
+ -shared)
+ test "$build_libtool_libs" != yes && \
+ func_fatal_configuration "can not build a shared library"
+ build_old_libs=no
+ continue
+ ;;
+
+ -static)
+ build_libtool_libs=no
+ build_old_libs=yes
+ continue
+ ;;
+
+ -prefer-pic)
+ pic_mode=yes
+ continue
+ ;;
+
+ -prefer-non-pic)
+ pic_mode=no
+ continue
+ ;;
+ esac
+ done
+
+ func_quote_for_eval "$libobj"
+ test "X$libobj" != "X$func_quote_for_eval_result" \
+ && $ECHO "X$libobj" | $GREP '[]~#^*{};<>?"'"'"' &()|`$[]' \
+ && func_warning "libobj name \`$libobj' may not contain shell special characters."
+ func_dirname_and_basename "$obj" "/" ""
+ objname="$func_basename_result"
+ xdir="$func_dirname_result"
+ lobj=${xdir}$objdir/$objname
+
+ test -z "$base_compile" && \
+ func_fatal_help "you must specify a compilation command"
+
+ # Delete any leftover library objects.
+ if test "$build_old_libs" = yes; then
+ removelist="$obj $lobj $libobj ${libobj}T"
+ else
+ removelist="$lobj $libobj ${libobj}T"
+ fi
+
+ # On Cygwin there's no "real" PIC flag so we must build both object types
+ case $host_os in
+ cygwin* | mingw* | pw32* | os2* | cegcc*)
+ pic_mode=default
+ ;;
+ esac
+ if test "$pic_mode" = no && test "$deplibs_check_method" != pass_all; then
+ # non-PIC code in shared libraries is not supported
+ pic_mode=default
+ fi
+
+ # Calculate the filename of the output object if compiler does
+ # not support -o with -c
+ if test "$compiler_c_o" = no; then
+ output_obj=`$ECHO "X$srcfile" | $Xsed -e 's%^.*/%%' -e 's%\.[^.]*$%%'`.${objext}
+ lockfile="$output_obj.lock"
+ else
+ output_obj=
+ need_locks=no
+ lockfile=
+ fi
+
+ # Lock this critical section if it is needed
+ # We use this script file to make the link, it avoids creating a new file
+ if test "$need_locks" = yes; then
+ until $opt_dry_run || ln "$progpath" "$lockfile" 2>/dev/null; do
+ func_echo "Waiting for $lockfile to be removed"
+ sleep 2
+ done
+ elif test "$need_locks" = warn; then
+ if test -f "$lockfile"; then
+ $ECHO "\
+*** ERROR, $lockfile exists and contains:
+`cat $lockfile 2>/dev/null`
+
+This indicates that another process is trying to use the same
+temporary object file, and libtool could not work around it because
+your compiler does not support \`-c' and \`-o' together. If you
+repeat this compilation, it may succeed, by chance, but you had better
+avoid parallel builds (make -j) in this platform, or get a better
+compiler."
+
+ $opt_dry_run || $RM $removelist
+ exit $EXIT_FAILURE
+ fi
+ removelist="$removelist $output_obj"
+ $ECHO "$srcfile" > "$lockfile"
+ fi
+
+ $opt_dry_run || $RM $removelist
+ removelist="$removelist $lockfile"
+ trap '$opt_dry_run || $RM $removelist; exit $EXIT_FAILURE' 1 2 15
+
+ if test -n "$fix_srcfile_path"; then
+ eval srcfile=\"$fix_srcfile_path\"
+ fi
+ func_quote_for_eval "$srcfile"
+ qsrcfile=$func_quote_for_eval_result
+
+ # Only build a PIC object if we are building libtool libraries.
+ if test "$build_libtool_libs" = yes; then
+ # Without this assignment, base_compile gets emptied.
+ fbsd_hideous_sh_bug=$base_compile
+
+ if test "$pic_mode" != no; then
+ command="$base_compile $qsrcfile $pic_flag"
+ else
+ # Don't build PIC code
+ command="$base_compile $qsrcfile"
+ fi
+
+ func_mkdir_p "$xdir$objdir"
+
+ if test -z "$output_obj"; then
+ # Place PIC objects in $objdir
+ command="$command -o $lobj"
+ fi
+
+ func_show_eval_locale "$command" \
+ 'test -n "$output_obj" && $RM $removelist; exit $EXIT_FAILURE'
+
+ if test "$need_locks" = warn &&
+ test "X`cat $lockfile 2>/dev/null`" != "X$srcfile"; then
+ $ECHO "\
+*** ERROR, $lockfile contains:
+`cat $lockfile 2>/dev/null`
+
+but it should contain:
+$srcfile
+
+This indicates that another process is trying to use the same
+temporary object file, and libtool could not work around it because
+your compiler does not support \`-c' and \`-o' together. If you
+repeat this compilation, it may succeed, by chance, but you had better
+avoid parallel builds (make -j) in this platform, or get a better
+compiler."
+
+ $opt_dry_run || $RM $removelist
+ exit $EXIT_FAILURE
+ fi
+
+ # Just move the object if needed, then go on to compile the next one
+ if test -n "$output_obj" && test "X$output_obj" != "X$lobj"; then
+ func_show_eval '$MV "$output_obj" "$lobj"' \
+ 'error=$?; $opt_dry_run || $RM $removelist; exit $error'
+ fi
+
+ # Allow error messages only from the first compilation.
+ if test "$suppress_opt" = yes; then
+ suppress_output=' >/dev/null 2>&1'
+ fi
+ fi
+
+ # Only build a position-dependent object if we build old libraries.
+ if test "$build_old_libs" = yes; then
+ if test "$pic_mode" != yes; then
+ # Don't build PIC code
+ command="$base_compile $qsrcfile$pie_flag"
+ else
+ command="$base_compile $qsrcfile $pic_flag"
+ fi
+ if test "$compiler_c_o" = yes; then
+ command="$command -o $obj"
+ fi
+
+ # Suppress compiler output if we already did a PIC compilation.
+ command="$command$suppress_output"
+ func_show_eval_locale "$command" \
+ '$opt_dry_run || $RM $removelist; exit $EXIT_FAILURE'
+
+ if test "$need_locks" = warn &&
+ test "X`cat $lockfile 2>/dev/null`" != "X$srcfile"; then
+ $ECHO "\
+*** ERROR, $lockfile contains:
+`cat $lockfile 2>/dev/null`
+
+but it should contain:
+$srcfile
+
+This indicates that another process is trying to use the same
+temporary object file, and libtool could not work around it because
+your compiler does not support \`-c' and \`-o' together. If you
+repeat this compilation, it may succeed, by chance, but you had better
+avoid parallel builds (make -j) in this platform, or get a better
+compiler."
+
+ $opt_dry_run || $RM $removelist
+ exit $EXIT_FAILURE
+ fi
+
+ # Just move the object if needed
+ if test -n "$output_obj" && test "X$output_obj" != "X$obj"; then
+ func_show_eval '$MV "$output_obj" "$obj"' \
+ 'error=$?; $opt_dry_run || $RM $removelist; exit $error'
+ fi
+ fi
+
+ $opt_dry_run || {
+ func_write_libtool_object "$libobj" "$objdir/$objname" "$objname"
+
+ # Unlock the critical section if it was locked
+ if test "$need_locks" != no; then
+ removelist=$lockfile
+ $RM "$lockfile"
+ fi
+ }
+
+ exit $EXIT_SUCCESS
+}
+
+$opt_help || {
+test "$mode" = compile && func_mode_compile ${1+"$@"}
+}
+
+func_mode_help ()
+{
+ # We need to display help for each of the modes.
+ case $mode in
+ "")
+ # Generic help is extracted from the usage comments
+ # at the start of this file.
+ func_help
+ ;;
+
+ clean)
+ $ECHO \
+"Usage: $progname [OPTION]... --mode=clean RM [RM-OPTION]... FILE...
+
+Remove files from the build directory.
+
+RM is the name of the program to use to delete files associated with each FILE
+(typically \`/bin/rm'). RM-OPTIONS are options (such as \`-f') to be passed
+to RM.
+
+If FILE is a libtool library, object or program, all the files associated
+with it are deleted. Otherwise, only FILE itself is deleted using RM."
+ ;;
+
+ compile)
+ $ECHO \
+"Usage: $progname [OPTION]... --mode=compile COMPILE-COMMAND... SOURCEFILE
+
+Compile a source file into a libtool library object.
+
+This mode accepts the following additional options:
+
+ -o OUTPUT-FILE set the output file name to OUTPUT-FILE
+ -no-suppress do not suppress compiler output for multiple passes
+ -prefer-pic try to building PIC objects only
+ -prefer-non-pic try to building non-PIC objects only
+ -shared do not build a \`.o' file suitable for static linking
+ -static only build a \`.o' file suitable for static linking
+
+COMPILE-COMMAND is a command to be used in creating a \`standard' object file
+from the given SOURCEFILE.
+
+The output file name is determined by removing the directory component from
+SOURCEFILE, then substituting the C source code suffix \`.c' with the
+library object suffix, \`.lo'."
+ ;;
+
+ execute)
+ $ECHO \
+"Usage: $progname [OPTION]... --mode=execute COMMAND [ARGS]...
+
+Automatically set library path, then run a program.
+
+This mode accepts the following additional options:
+
+ -dlopen FILE add the directory containing FILE to the library path
+
+This mode sets the library path environment variable according to \`-dlopen'
+flags.
+
+If any of the ARGS are libtool executable wrappers, then they are translated
+into their corresponding uninstalled binary, and any of their required library
+directories are added to the library path.
+
+Then, COMMAND is executed, with ARGS as arguments."
+ ;;
+
+ finish)
+ $ECHO \
+"Usage: $progname [OPTION]... --mode=finish [LIBDIR]...
+
+Complete the installation of libtool libraries.
+
+Each LIBDIR is a directory that contains libtool libraries.
+
+The commands that this mode executes may require superuser privileges. Use
+the \`--dry-run' option if you just want to see what would be executed."
+ ;;
+
+ install)
+ $ECHO \
+"Usage: $progname [OPTION]... --mode=install INSTALL-COMMAND...
+
+Install executables or libraries.
+
+INSTALL-COMMAND is the installation command. The first component should be
+either the \`install' or \`cp' program.
+
+The following components of INSTALL-COMMAND are treated specially:
+
+ -inst-prefix PREFIX-DIR Use PREFIX-DIR as a staging area for installation
+
+The rest of the components are interpreted as arguments to that command (only
+BSD-compatible install options are recognized)."
+ ;;
+
+ link)
+ $ECHO \
+"Usage: $progname [OPTION]... --mode=link LINK-COMMAND...
+
+Link object files or libraries together to form another library, or to
+create an executable program.
+
+LINK-COMMAND is a command using the C compiler that you would use to create
+a program from several object files.
+
+The following components of LINK-COMMAND are treated specially:
+
+ -all-static do not do any dynamic linking at all
+ -avoid-version do not add a version suffix if possible
+ -dlopen FILE \`-dlpreopen' FILE if it cannot be dlopened at runtime
+ -dlpreopen FILE link in FILE and add its symbols to lt_preloaded_symbols
+ -export-dynamic allow symbols from OUTPUT-FILE to be resolved with dlsym(3)
+ -export-symbols SYMFILE
+ try to export only the symbols listed in SYMFILE
+ -export-symbols-regex REGEX
+ try to export only the symbols matching REGEX
+ -LLIBDIR search LIBDIR for required installed libraries
+ -lNAME OUTPUT-FILE requires the installed library libNAME
+ -module build a library that can dlopened
+ -no-fast-install disable the fast-install mode
+ -no-install link a not-installable executable
+ -no-undefined declare that a library does not refer to external symbols
+ -o OUTPUT-FILE create OUTPUT-FILE from the specified objects
+ -objectlist FILE Use a list of object files found in FILE to specify objects
+ -precious-files-regex REGEX
+ don't remove output files matching REGEX
+ -release RELEASE specify package release information
+ -rpath LIBDIR the created library will eventually be installed in LIBDIR
+ -R[ ]LIBDIR add LIBDIR to the runtime path of programs and libraries
+ -shared only do dynamic linking of libtool libraries
+ -shrext SUFFIX override the standard shared library file extension
+ -static do not do any dynamic linking of uninstalled libtool libraries
+ -static-libtool-libs
+ do not do any dynamic linking of libtool libraries
+ -version-info CURRENT[:REVISION[:AGE]]
+ specify library version info [each variable defaults to 0]
+ -weak LIBNAME declare that the target provides the LIBNAME interface
+
+All other options (arguments beginning with \`-') are ignored.
+
+Every other argument is treated as a filename. Files ending in \`.la' are
+treated as uninstalled libtool libraries, other files are standard or library
+object files.
+
+If the OUTPUT-FILE ends in \`.la', then a libtool library is created,
+only library objects (\`.lo' files) may be specified, and \`-rpath' is
+required, except when creating a convenience library.
+
+If OUTPUT-FILE ends in \`.a' or \`.lib', then a standard library is created
+using \`ar' and \`ranlib', or on Windows using \`lib'.
+
+If OUTPUT-FILE ends in \`.lo' or \`.${objext}', then a reloadable object file
+is created, otherwise an executable program is created."
+ ;;
+
+ uninstall)
+ $ECHO \
+"Usage: $progname [OPTION]... --mode=uninstall RM [RM-OPTION]... FILE...
+
+Remove libraries from an installation directory.
+
+RM is the name of the program to use to delete files associated with each FILE
+(typically \`/bin/rm'). RM-OPTIONS are options (such as \`-f') to be passed
+to RM.
+
+If FILE is a libtool library, all the files associated with it are deleted.
+Otherwise, only FILE itself is deleted using RM."
+ ;;
+
+ *)
+ func_fatal_help "invalid operation mode \`$mode'"
+ ;;
+ esac
+
+ $ECHO
+ $ECHO "Try \`$progname --help' for more information about other modes."
+
+ exit $?
+}
+
+ # Now that we've collected a possible --mode arg, show help if necessary
+ $opt_help && func_mode_help
+
+
+# func_mode_execute arg...
+func_mode_execute ()
+{
+ $opt_debug
+ # The first argument is the command name.
+ cmd="$nonopt"
+ test -z "$cmd" && \
+ func_fatal_help "you must specify a COMMAND"
+
+ # Handle -dlopen flags immediately.
+ for file in $execute_dlfiles; do
+ test -f "$file" \
+ || func_fatal_help "\`$file' is not a file"
+
+ dir=
+ case $file in
+ *.la)
+ # Check to see that this really is a libtool archive.
+ func_lalib_unsafe_p "$file" \
+ || func_fatal_help "\`$lib' is not a valid libtool archive"
+
+ # Read the libtool library.
+ dlname=
+ library_names=
+ func_source "$file"
+
+ # Skip this library if it cannot be dlopened.
+ if test -z "$dlname"; then
+ # Warn if it was a shared library.
+ test -n "$library_names" && \
+ func_warning "\`$file' was not linked with \`-export-dynamic'"
+ continue
+ fi
+
+ func_dirname "$file" "" "."
+ dir="$func_dirname_result"
+
+ if test -f "$dir/$objdir/$dlname"; then
+ dir="$dir/$objdir"
+ else
+ if test ! -f "$dir/$dlname"; then
+ func_fatal_error "cannot find \`$dlname' in \`$dir' or \`$dir/$objdir'"
+ fi
+ fi
+ ;;
+
+ *.lo)
+ # Just add the directory containing the .lo file.
+ func_dirname "$file" "" "."
+ dir="$func_dirname_result"
+ ;;
+
+ *)
+ func_warning "\`-dlopen' is ignored for non-libtool libraries and objects"
+ continue
+ ;;
+ esac
+
+ # Get the absolute pathname.
+ absdir=`cd "$dir" && pwd`
+ test -n "$absdir" && dir="$absdir"
+
+ # Now add the directory to shlibpath_var.
+ if eval "test -z \"\$$shlibpath_var\""; then
+ eval "$shlibpath_var=\"\$dir\""
+ else
+ eval "$shlibpath_var=\"\$dir:\$$shlibpath_var\""
+ fi
+ done
+
+ # This variable tells wrapper scripts just to set shlibpath_var
+ # rather than running their programs.
+ libtool_execute_magic="$magic"
+
+ # Check if any of the arguments is a wrapper script.
+ args=
+ for file
+ do
+ case $file in
+ -*) ;;
+ *)
+ # Do a test to see if this is really a libtool program.
+ if func_ltwrapper_script_p "$file"; then
+ func_source "$file"
+ # Transform arg to wrapped name.
+ file="$progdir/$program"
+ elif func_ltwrapper_executable_p "$file"; then
+ func_ltwrapper_scriptname "$file"
+ func_source "$func_ltwrapper_scriptname_result"
+ # Transform arg to wrapped name.
+ file="$progdir/$program"
+ fi
+ ;;
+ esac
+ # Quote arguments (to preserve shell metacharacters).
+ func_quote_for_eval "$file"
+ args="$args $func_quote_for_eval_result"
+ done
+
+ if test "X$opt_dry_run" = Xfalse; then
+ if test -n "$shlibpath_var"; then
+ # Export the shlibpath_var.
+ eval "export $shlibpath_var"
+ fi
+
+ # Restore saved environment variables
+ for lt_var in LANG LANGUAGE LC_ALL LC_CTYPE LC_COLLATE LC_MESSAGES
+ do
+ eval "if test \"\${save_$lt_var+set}\" = set; then
+ $lt_var=\$save_$lt_var; export $lt_var
+ else
+ $lt_unset $lt_var
+ fi"
+ done
+
+ # Now prepare to actually exec the command.
+ exec_cmd="\$cmd$args"
+ else
+ # Display what would be done.
+ if test -n "$shlibpath_var"; then
+ eval "\$ECHO \"\$shlibpath_var=\$$shlibpath_var\""
+ $ECHO "export $shlibpath_var"
+ fi
+ $ECHO "$cmd$args"
+ exit $EXIT_SUCCESS
+ fi
+}
+
+test "$mode" = execute && func_mode_execute ${1+"$@"}
+
+
+# func_mode_finish arg...
+func_mode_finish ()
+{
+ $opt_debug
+ libdirs="$nonopt"
+ admincmds=
+
+ if test -n "$finish_cmds$finish_eval" && test -n "$libdirs"; then
+ for dir
+ do
+ libdirs="$libdirs $dir"
+ done
+
+ for libdir in $libdirs; do
+ if test -n "$finish_cmds"; then
+ # Do each command in the finish commands.
+ func_execute_cmds "$finish_cmds" 'admincmds="$admincmds
+'"$cmd"'"'
+ fi
+ if test -n "$finish_eval"; then
+ # Do the single finish_eval.
+ eval cmds=\"$finish_eval\"
+ $opt_dry_run || eval "$cmds" || admincmds="$admincmds
+ $cmds"
+ fi
+ done
+ fi
+
+ # Exit here if they wanted silent mode.
+ $opt_silent && exit $EXIT_SUCCESS
+
+ $ECHO "X----------------------------------------------------------------------" | $Xsed
+ $ECHO "Libraries have been installed in:"
+ for libdir in $libdirs; do
+ $ECHO " $libdir"
+ done
+ $ECHO
+ $ECHO "If you ever happen to want to link against installed libraries"
+ $ECHO "in a given directory, LIBDIR, you must either use libtool, and"
+ $ECHO "specify the full pathname of the library, or use the \`-LLIBDIR'"
+ $ECHO "flag during linking and do at least one of the following:"
+ if test -n "$shlibpath_var"; then
+ $ECHO " - add LIBDIR to the \`$shlibpath_var' environment variable"
+ $ECHO " during execution"
+ fi
+ if test -n "$runpath_var"; then
+ $ECHO " - add LIBDIR to the \`$runpath_var' environment variable"
+ $ECHO " during linking"
+ fi
+ if test -n "$hardcode_libdir_flag_spec"; then
+ libdir=LIBDIR
+ eval flag=\"$hardcode_libdir_flag_spec\"
+
+ $ECHO " - use the \`$flag' linker flag"
+ fi
+ if test -n "$admincmds"; then
+ $ECHO " - have your system administrator run these commands:$admincmds"
+ fi
+ if test -f /etc/ld.so.conf; then
+ $ECHO " - have your system administrator add LIBDIR to \`/etc/ld.so.conf'"
+ fi
+ $ECHO
+
+ $ECHO "See any operating system documentation about shared libraries for"
+ case $host in
+ solaris2.[6789]|solaris2.1[0-9])
+ $ECHO "more information, such as the ld(1), crle(1) and ld.so(8) manual"
+ $ECHO "pages."
+ ;;
+ *)
+ $ECHO "more information, such as the ld(1) and ld.so(8) manual pages."
+ ;;
+ esac
+ $ECHO "X----------------------------------------------------------------------" | $Xsed
+ exit $EXIT_SUCCESS
+}
+
+test "$mode" = finish && func_mode_finish ${1+"$@"}
+
+
+# func_mode_install arg...
+func_mode_install ()
+{
+ $opt_debug
+ # There may be an optional sh(1) argument at the beginning of
+ # install_prog (especially on Windows NT).
+ if test "$nonopt" = "$SHELL" || test "$nonopt" = /bin/sh ||
+ # Allow the use of GNU shtool's install command.
+ $ECHO "X$nonopt" | $GREP shtool >/dev/null; then
+ # Aesthetically quote it.
+ func_quote_for_eval "$nonopt"
+ install_prog="$func_quote_for_eval_result "
+ arg=$1
+ shift
+ else
+ install_prog=
+ arg=$nonopt
+ fi
+
+ # The real first argument should be the name of the installation program.
+ # Aesthetically quote it.
+ func_quote_for_eval "$arg"
+ install_prog="$install_prog$func_quote_for_eval_result"
+
+ # We need to accept at least all the BSD install flags.
+ dest=
+ files=
+ opts=
+ prev=
+ install_type=
+ isdir=no
+ stripme=
+ for arg
+ do
+ if test -n "$dest"; then
+ files="$files $dest"
+ dest=$arg
+ continue
+ fi
+
+ case $arg in
+ -d) isdir=yes ;;
+ -f)
+ case " $install_prog " in
+ *[\\\ /]cp\ *) ;;
+ *) prev=$arg ;;
+ esac
+ ;;
+ -g | -m | -o)
+ prev=$arg
+ ;;
+ -s)
+ stripme=" -s"
+ continue
+ ;;
+ -*)
+ ;;
+ *)
+ # If the previous option needed an argument, then skip it.
+ if test -n "$prev"; then
+ prev=
+ else
+ dest=$arg
+ continue
+ fi
+ ;;
+ esac
+
+ # Aesthetically quote the argument.
+ func_quote_for_eval "$arg"
+ install_prog="$install_prog $func_quote_for_eval_result"
+ done
+
+ test -z "$install_prog" && \
+ func_fatal_help "you must specify an install program"
+
+ test -n "$prev" && \
+ func_fatal_help "the \`$prev' option requires an argument"
+
+ if test -z "$files"; then
+ if test -z "$dest"; then
+ func_fatal_help "no file or destination specified"
+ else
+ func_fatal_help "you must specify a destination"
+ fi
+ fi
+
+ # Strip any trailing slash from the destination.
+ func_stripname '' '/' "$dest"
+ dest=$func_stripname_result
+
+ # Check to see that the destination is a directory.
+ test -d "$dest" && isdir=yes
+ if test "$isdir" = yes; then
+ destdir="$dest"
+ destname=
+ else
+ func_dirname_and_basename "$dest" "" "."
+ destdir="$func_dirname_result"
+ destname="$func_basename_result"
+
+ # Not a directory, so check to see that there is only one file specified.
+ set dummy $files; shift
+ test "$#" -gt 1 && \
+ func_fatal_help "\`$dest' is not a directory"
+ fi
+ case $destdir in
+ [\\/]* | [A-Za-z]:[\\/]*) ;;
+ *)
+ for file in $files; do
+ case $file in
+ *.lo) ;;
+ *)
+ func_fatal_help "\`$destdir' must be an absolute directory name"
+ ;;
+ esac
+ done
+ ;;
+ esac
+
+ # This variable tells wrapper scripts just to set variables rather
+ # than running their programs.
+ libtool_install_magic="$magic"
+
+ staticlibs=
+ future_libdirs=
+ current_libdirs=
+ for file in $files; do
+
+ # Do each installation.
+ case $file in
+ *.$libext)
+ # Do the static libraries later.
+ staticlibs="$staticlibs $file"
+ ;;
+
+ *.la)
+ # Check to see that this really is a libtool archive.
+ func_lalib_unsafe_p "$file" \
+ || func_fatal_help "\`$file' is not a valid libtool archive"
+
+ library_names=
+ old_library=
+ relink_command=
+ func_source "$file"
+
+ # Add the libdir to current_libdirs if it is the destination.
+ if test "X$destdir" = "X$libdir"; then
+ case "$current_libdirs " in
+ *" $libdir "*) ;;
+ *) current_libdirs="$current_libdirs $libdir" ;;
+ esac
+ else
+ # Note the libdir as a future libdir.
+ case "$future_libdirs " in
+ *" $libdir "*) ;;
+ *) future_libdirs="$future_libdirs $libdir" ;;
+ esac
+ fi
+
+ func_dirname "$file" "/" ""
+ dir="$func_dirname_result"
+ dir="$dir$objdir"
+
+ if test -n "$relink_command"; then
+ # Determine the prefix the user has applied to our future dir.
+ inst_prefix_dir=`$ECHO "X$destdir" | $Xsed -e "s%$libdir\$%%"`
+
+ # Don't allow the user to place us outside of our expected
+ # location b/c this prevents finding dependent libraries that
+ # are installed to the same prefix.
+ # At present, this check doesn't affect windows .dll's that
+ # are installed into $libdir/../bin (currently, that works fine)
+ # but it's something to keep an eye on.
+ test "$inst_prefix_dir" = "$destdir" && \
+ func_fatal_error "error: cannot install \`$file' to a directory not ending in $libdir"
+
+ if test -n "$inst_prefix_dir"; then
+ # Stick the inst_prefix_dir data into the link command.
+ relink_command=`$ECHO "X$relink_command" | $Xsed -e "s%@inst_prefix_dir@%-inst-prefix-dir $inst_prefix_dir%"`
+ else
+ relink_command=`$ECHO "X$relink_command" | $Xsed -e "s%@inst_prefix_dir@%%"`
+ fi
+
+ func_warning "relinking \`$file'"
+ func_show_eval "$relink_command" \
+ 'func_fatal_error "error: relink \`$file'\'' with the above command before installing it"'
+ fi
+
+ # See the names of the shared library.
+ set dummy $library_names; shift
+ if test -n "$1"; then
+ realname="$1"
+ shift
+
+ srcname="$realname"
+ test -n "$relink_command" && srcname="$realname"T
+
+ # Install the shared library and build the symlinks.
+ func_show_eval "$install_prog $dir/$srcname $destdir/$realname" \
+ 'exit $?'
+ tstripme="$stripme"
+ case $host_os in
+ cygwin* | mingw* | pw32* | cegcc*)
+ case $realname in
+ *.dll.a)
+ tstripme=""
+ ;;
+ esac
+ ;;
+ esac
+ if test -n "$tstripme" && test -n "$striplib"; then
+ func_show_eval "$striplib $destdir/$realname" 'exit $?'
+ fi
+
+ if test "$#" -gt 0; then
+ # Delete the old symlinks, and create new ones.
+ # Try `ln -sf' first, because the `ln' binary might depend on
+ # the symlink we replace! Solaris /bin/ln does not understand -f,
+ # so we also need to try rm && ln -s.
+ for linkname
+ do
+ test "$linkname" != "$realname" \
+ && func_show_eval "(cd $destdir && { $LN_S -f $realname $linkname || { $RM $linkname && $LN_S $realname $linkname; }; })"
+ done
+ fi
+
+ # Do each command in the postinstall commands.
+ lib="$destdir/$realname"
+ func_execute_cmds "$postinstall_cmds" 'exit $?'
+ fi
+
+ # Install the pseudo-library for information purposes.
+ func_basename "$file"
+ name="$func_basename_result"
+ instname="$dir/$name"i
+ func_show_eval "$install_prog $instname $destdir/$name" 'exit $?'
+
+ # Maybe install the static library, too.
+ test -n "$old_library" && staticlibs="$staticlibs $dir/$old_library"
+ ;;
+
+ *.lo)
+ # Install (i.e. copy) a libtool object.
+
+ # Figure out destination file name, if it wasn't already specified.
+ if test -n "$destname"; then
+ destfile="$destdir/$destname"
+ else
+ func_basename "$file"
+ destfile="$func_basename_result"
+ destfile="$destdir/$destfile"
+ fi
+
+ # Deduce the name of the destination old-style object file.
+ case $destfile in
+ *.lo)
+ func_lo2o "$destfile"
+ staticdest=$func_lo2o_result
+ ;;
+ *.$objext)
+ staticdest="$destfile"
+ destfile=
+ ;;
+ *)
+ func_fatal_help "cannot copy a libtool object to \`$destfile'"
+ ;;
+ esac
+
+ # Install the libtool object if requested.
+ test -n "$destfile" && \
+ func_show_eval "$install_prog $file $destfile" 'exit $?'
+
+ # Install the old object if enabled.
+ if test "$build_old_libs" = yes; then
+ # Deduce the name of the old-style object file.
+ func_lo2o "$file"
+ staticobj=$func_lo2o_result
+ func_show_eval "$install_prog \$staticobj \$staticdest" 'exit $?'
+ fi
+ exit $EXIT_SUCCESS
+ ;;
+
+ *)
+ # Figure out destination file name, if it wasn't already specified.
+ if test -n "$destname"; then
+ destfile="$destdir/$destname"
+ else
+ func_basename "$file"
+ destfile="$func_basename_result"
+ destfile="$destdir/$destfile"
+ fi
+
+ # If the file is missing, and there is a .exe on the end, strip it
+ # because it is most likely a libtool script we actually want to
+ # install
+ stripped_ext=""
+ case $file in
+ *.exe)
+ if test ! -f "$file"; then
+ func_stripname '' '.exe' "$file"
+ file=$func_stripname_result
+ stripped_ext=".exe"
+ fi
+ ;;
+ esac
+
+ # Do a test to see if this is really a libtool program.
+ case $host in
+ *cygwin* | *mingw*)
+ if func_ltwrapper_executable_p "$file"; then
+ func_ltwrapper_scriptname "$file"
+ wrapper=$func_ltwrapper_scriptname_result
+ else
+ func_stripname '' '.exe' "$file"
+ wrapper=$func_stripname_result
+ fi
+ ;;
+ *)
+ wrapper=$file
+ ;;
+ esac
+ if func_ltwrapper_script_p "$wrapper"; then
+ notinst_deplibs=
+ relink_command=
+
+ func_source "$wrapper"
+
+ # Check the variables that should have been set.
+ test -z "$generated_by_libtool_version" && \
+ func_fatal_error "invalid libtool wrapper script \`$wrapper'"
+
+ finalize=yes
+ for lib in $notinst_deplibs; do
+ # Check to see that each library is installed.
+ libdir=
+ if test -f "$lib"; then
+ func_source "$lib"
+ fi
+ libfile="$libdir/"`$ECHO "X$lib" | $Xsed -e 's%^.*/%%g'` ### testsuite: skip nested quoting test
+ if test -n "$libdir" && test ! -f "$libfile"; then
+ func_warning "\`$lib' has not been installed in \`$libdir'"
+ finalize=no
+ fi
+ done
+
+ relink_command=
+ func_source "$wrapper"
+
+ outputname=
+ if test "$fast_install" = no && test -n "$relink_command"; then
+ $opt_dry_run || {
+ if test "$finalize" = yes; then
+ tmpdir=`func_mktempdir`
+ func_basename "$file$stripped_ext"
+ file="$func_basename_result"
+ outputname="$tmpdir/$file"
+ # Replace the output file specification.
+ relink_command=`$ECHO "X$relink_command" | $Xsed -e 's%@OUTPUT@%'"$outputname"'%g'`
+
+ $opt_silent || {
+ func_quote_for_expand "$relink_command"
+ eval "func_echo $func_quote_for_expand_result"
+ }
+ if eval "$relink_command"; then :
+ else
+ func_error "error: relink \`$file' with the above command before installing it"
+ $opt_dry_run || ${RM}r "$tmpdir"
+ continue
+ fi
+ file="$outputname"
+ else
+ func_warning "cannot relink \`$file'"
+ fi
+ }
+ else
+ # Install the binary that we compiled earlier.
+ file=`$ECHO "X$file$stripped_ext" | $Xsed -e "s%\([^/]*\)$%$objdir/\1%"`
+ fi
+ fi
+
+ # remove .exe since cygwin /usr/bin/install will append another
+ # one anyway
+ case $install_prog,$host in
+ */usr/bin/install*,*cygwin*)
+ case $file:$destfile in
+ *.exe:*.exe)
+ # this is ok
+ ;;
+ *.exe:*)
+ destfile=$destfile.exe
+ ;;
+ *:*.exe)
+ func_stripname '' '.exe' "$destfile"
+ destfile=$func_stripname_result
+ ;;
+ esac
+ ;;
+ esac
+ func_show_eval "$install_prog\$stripme \$file \$destfile" 'exit $?'
+ $opt_dry_run || if test -n "$outputname"; then
+ ${RM}r "$tmpdir"
+ fi
+ ;;
+ esac
+ done
+
+ for file in $staticlibs; do
+ func_basename "$file"
+ name="$func_basename_result"
+
+ # Set up the ranlib parameters.
+ oldlib="$destdir/$name"
+
+ func_show_eval "$install_prog \$file \$oldlib" 'exit $?'
+
+ if test -n "$stripme" && test -n "$old_striplib"; then
+ func_show_eval "$old_striplib $oldlib" 'exit $?'
+ fi
+
+ # Do each command in the postinstall commands.
+ func_execute_cmds "$old_postinstall_cmds" 'exit $?'
+ done
+
+ test -n "$future_libdirs" && \
+ func_warning "remember to run \`$progname --finish$future_libdirs'"
+
+ if test -n "$current_libdirs"; then
+ # Maybe just do a dry run.
+ $opt_dry_run && current_libdirs=" -n$current_libdirs"
+ exec_cmd='$SHELL $progpath $preserve_args --finish$current_libdirs'
+ else
+ exit $EXIT_SUCCESS
+ fi
+}
+
+test "$mode" = install && func_mode_install ${1+"$@"}
+
+
+# func_generate_dlsyms outputname originator pic_p
+# Extract symbols from dlprefiles and create ${outputname}S.o with
+# a dlpreopen symbol table.
+func_generate_dlsyms ()
+{
+ $opt_debug
+ my_outputname="$1"
+ my_originator="$2"
+ my_pic_p="${3-no}"
+ my_prefix=`$ECHO "$my_originator" | sed 's%[^a-zA-Z0-9]%_%g'`
+ my_dlsyms=
+
+ if test -n "$dlfiles$dlprefiles" || test "$dlself" != no; then
+ if test -n "$NM" && test -n "$global_symbol_pipe"; then
+ my_dlsyms="${my_outputname}S.c"
+ else
+ func_error "not configured to extract global symbols from dlpreopened files"
+ fi
+ fi
+
+ if test -n "$my_dlsyms"; then
+ case $my_dlsyms in
+ "") ;;
+ *.c)
+ # Discover the nlist of each of the dlfiles.
+ nlist="$output_objdir/${my_outputname}.nm"
+
+ func_show_eval "$RM $nlist ${nlist}S ${nlist}T"
+
+ # Parse the name list into a source file.
+ func_verbose "creating $output_objdir/$my_dlsyms"
+
+ $opt_dry_run || $ECHO > "$output_objdir/$my_dlsyms" "\
+/* $my_dlsyms - symbol resolution table for \`$my_outputname' dlsym emulation. */
+/* Generated by $PROGRAM (GNU $PACKAGE$TIMESTAMP) $VERSION */
+
+#ifdef __cplusplus
+extern \"C\" {
+#endif
+
+/* External symbol declarations for the compiler. */\
+"
+
+ if test "$dlself" = yes; then
+ func_verbose "generating symbol list for \`$output'"
+
+ $opt_dry_run || echo ': @PROGRAM@ ' > "$nlist"
+
+ # Add our own program objects to the symbol list.
+ progfiles=`$ECHO "X$objs$old_deplibs" | $SP2NL | $Xsed -e "$lo2o" | $NL2SP`
+ for progfile in $progfiles; do
+ func_verbose "extracting global C symbols from \`$progfile'"
+ $opt_dry_run || eval "$NM $progfile | $global_symbol_pipe >> '$nlist'"
+ done
+
+ if test -n "$exclude_expsyms"; then
+ $opt_dry_run || {
+ eval '$EGREP -v " ($exclude_expsyms)$" "$nlist" > "$nlist"T'
+ eval '$MV "$nlist"T "$nlist"'
+ }
+ fi
+
+ if test -n "$export_symbols_regex"; then
+ $opt_dry_run || {
+ eval '$EGREP -e "$export_symbols_regex" "$nlist" > "$nlist"T'
+ eval '$MV "$nlist"T "$nlist"'
+ }
+ fi
+
+ # Prepare the list of exported symbols
+ if test -z "$export_symbols"; then
+ export_symbols="$output_objdir/$outputname.exp"
+ $opt_dry_run || {
+ $RM $export_symbols
+ eval "${SED} -n -e '/^: @PROGRAM@ $/d' -e 's/^.* \(.*\)$/\1/p' "'< "$nlist" > "$export_symbols"'
+ case $host in
+ *cygwin* | *mingw* | *cegcc* )
+ eval "echo EXPORTS "'> "$output_objdir/$outputname.def"'
+ eval 'cat "$export_symbols" >> "$output_objdir/$outputname.def"'
+ ;;
+ esac
+ }
+ else
+ $opt_dry_run || {
+ eval "${SED} -e 's/\([].[*^$]\)/\\\\\1/g' -e 's/^/ /' -e 's/$/$/'"' < "$export_symbols" > "$output_objdir/$outputname.exp"'
+ eval '$GREP -f "$output_objdir/$outputname.exp" < "$nlist" > "$nlist"T'
+ eval '$MV "$nlist"T "$nlist"'
+ case $host in
+ *cygwin | *mingw* | *cegcc* )
+ eval "echo EXPORTS "'> "$output_objdir/$outputname.def"'
+ eval 'cat "$nlist" >> "$output_objdir/$outputname.def"'
+ ;;
+ esac
+ }
+ fi
+ fi
+
+ for dlprefile in $dlprefiles; do
+ func_verbose "extracting global C symbols from \`$dlprefile'"
+ func_basename "$dlprefile"
+ name="$func_basename_result"
+ $opt_dry_run || {
+ eval '$ECHO ": $name " >> "$nlist"'
+ eval "$NM $dlprefile 2>/dev/null | $global_symbol_pipe >> '$nlist'"
+ }
+ done
+
+ $opt_dry_run || {
+ # Make sure we have at least an empty file.
+ test -f "$nlist" || : > "$nlist"
+
+ if test -n "$exclude_expsyms"; then
+ $EGREP -v " ($exclude_expsyms)$" "$nlist" > "$nlist"T
+ $MV "$nlist"T "$nlist"
+ fi
+
+ # Try sorting and uniquifying the output.
+ if $GREP -v "^: " < "$nlist" |
+ if sort -k 3 </dev/null >/dev/null 2>&1; then
+ sort -k 3
+ else
+ sort +2
+ fi |
+ uniq > "$nlist"S; then
+ :
+ else
+ $GREP -v "^: " < "$nlist" > "$nlist"S
+ fi
+
+ if test -f "$nlist"S; then
+ eval "$global_symbol_to_cdecl"' < "$nlist"S >> "$output_objdir/$my_dlsyms"'
+ else
+ $ECHO '/* NONE */' >> "$output_objdir/$my_dlsyms"
+ fi
+
+ $ECHO >> "$output_objdir/$my_dlsyms" "\
+
+/* The mapping between symbol names and symbols. */
+typedef struct {
+ const char *name;
+ void *address;
+} lt_dlsymlist;
+"
+ case $host in
+ *cygwin* | *mingw* | *cegcc* )
+ $ECHO >> "$output_objdir/$my_dlsyms" "\
+/* DATA imports from DLLs on WIN32 con't be const, because
+ runtime relocations are performed -- see ld's documentation
+ on pseudo-relocs. */"
+ lt_dlsym_const= ;;
+ *osf5*)
+ echo >> "$output_objdir/$my_dlsyms" "\
+/* This system does not cope well with relocations in const data */"
+ lt_dlsym_const= ;;
+ *)
+ lt_dlsym_const=const ;;
+ esac
+
+ $ECHO >> "$output_objdir/$my_dlsyms" "\
+extern $lt_dlsym_const lt_dlsymlist
+lt_${my_prefix}_LTX_preloaded_symbols[];
+$lt_dlsym_const lt_dlsymlist
+lt_${my_prefix}_LTX_preloaded_symbols[] =
+{\
+ { \"$my_originator\", (void *) 0 },"
+
+ case $need_lib_prefix in
+ no)
+ eval "$global_symbol_to_c_name_address" < "$nlist" >> "$output_objdir/$my_dlsyms"
+ ;;
+ *)
+ eval "$global_symbol_to_c_name_address_lib_prefix" < "$nlist" >> "$output_objdir/$my_dlsyms"
+ ;;
+ esac
+ $ECHO >> "$output_objdir/$my_dlsyms" "\
+ {0, (void *) 0}
+};
+
+/* This works around a problem in FreeBSD linker */
+#ifdef FREEBSD_WORKAROUND
+static const void *lt_preloaded_setup() {
+ return lt_${my_prefix}_LTX_preloaded_symbols;
+}
+#endif
+
+#ifdef __cplusplus
+}
+#endif\
+"
+ } # !$opt_dry_run
+
+ pic_flag_for_symtable=
+ case "$compile_command " in
+ *" -static "*) ;;
+ *)
+ case $host in
+ # compiling the symbol table file with pic_flag works around
+ # a FreeBSD bug that causes programs to crash when -lm is
+ # linked before any other PIC object. But we must not use
+ # pic_flag when linking with -static. The problem exists in
+ # FreeBSD 2.2.6 and is fixed in FreeBSD 3.1.
+ *-*-freebsd2*|*-*-freebsd3.0*|*-*-freebsdelf3.0*)
+ pic_flag_for_symtable=" $pic_flag -DFREEBSD_WORKAROUND" ;;
+ *-*-hpux*)
+ pic_flag_for_symtable=" $pic_flag" ;;
+ *)
+ if test "X$my_pic_p" != Xno; then
+ pic_flag_for_symtable=" $pic_flag"
+ fi
+ ;;
+ esac
+ ;;
+ esac
+ symtab_cflags=
+ for arg in $LTCFLAGS; do
+ case $arg in
+ -pie | -fpie | -fPIE) ;;
+ *) symtab_cflags="$symtab_cflags $arg" ;;
+ esac
+ done
+
+ # Now compile the dynamic symbol file.
+ func_show_eval '(cd $output_objdir && $LTCC$symtab_cflags -c$no_builtin_flag$pic_flag_for_symtable "$my_dlsyms")' 'exit $?'
+
+ # Clean up the generated files.
+ func_show_eval '$RM "$output_objdir/$my_dlsyms" "$nlist" "${nlist}S" "${nlist}T"'
+
+ # Transform the symbol file into the correct name.
+ symfileobj="$output_objdir/${my_outputname}S.$objext"
+ case $host in
+ *cygwin* | *mingw* | *cegcc* )
+ if test -f "$output_objdir/$my_outputname.def"; then
+ compile_command=`$ECHO "X$compile_command" | $Xsed -e "s%@SYMFILE@%$output_objdir/$my_outputname.def $symfileobj%"`
+ finalize_command=`$ECHO "X$finalize_command" | $Xsed -e "s%@SYMFILE@%$output_objdir/$my_outputname.def $symfileobj%"`
+ else
+ compile_command=`$ECHO "X$compile_command" | $Xsed -e "s%@SYMFILE@%$symfileobj%"`
+ finalize_command=`$ECHO "X$finalize_command" | $Xsed -e "s%@SYMFILE@%$symfileobj%"`
+ fi
+ ;;
+ *)
+ compile_command=`$ECHO "X$compile_command" | $Xsed -e "s%@SYMFILE@%$symfileobj%"`
+ finalize_command=`$ECHO "X$finalize_command" | $Xsed -e "s%@SYMFILE@%$symfileobj%"`
+ ;;
+ esac
+ ;;
+ *)
+ func_fatal_error "unknown suffix for \`$my_dlsyms'"
+ ;;
+ esac
+ else
+ # We keep going just in case the user didn't refer to
+ # lt_preloaded_symbols. The linker will fail if global_symbol_pipe
+ # really was required.
+
+ # Nullify the symbol file.
+ compile_command=`$ECHO "X$compile_command" | $Xsed -e "s% @SYMFILE@%%"`
+ finalize_command=`$ECHO "X$finalize_command" | $Xsed -e "s% @SYMFILE@%%"`
+ fi
+}
+
+# func_win32_libid arg
+# return the library type of file 'arg'
+#
+# Need a lot of goo to handle *both* DLLs and import libs
+# Has to be a shell function in order to 'eat' the argument
+# that is supplied when $file_magic_command is called.
+func_win32_libid ()
+{
+ $opt_debug
+ win32_libid_type="unknown"
+ win32_fileres=`file -L $1 2>/dev/null`
+ case $win32_fileres in
+ *ar\ archive\ import\ library*) # definitely import
+ win32_libid_type="x86 archive import"
+ ;;
+ *ar\ archive*) # could be an import, or static
+ if eval $OBJDUMP -f $1 | $SED -e '10q' 2>/dev/null |
+ $EGREP 'file format pe-i386(.*architecture: i386)?' >/dev/null ; then
+ win32_nmres=`eval $NM -f posix -A $1 |
+ $SED -n -e '
+ 1,100{
+ / I /{
+ s,.*,import,
+ p
+ q
+ }
+ }'`
+ case $win32_nmres in
+ import*) win32_libid_type="x86 archive import";;
+ *) win32_libid_type="x86 archive static";;
+ esac
+ fi
+ ;;
+ *DLL*)
+ win32_libid_type="x86 DLL"
+ ;;
+ *executable*) # but shell scripts are "executable" too...
+ case $win32_fileres in
+ *MS\ Windows\ PE\ Intel*)
+ win32_libid_type="x86 DLL"
+ ;;
+ esac
+ ;;
+ esac
+ $ECHO "$win32_libid_type"
+}
+
+
+
+# func_extract_an_archive dir oldlib
+func_extract_an_archive ()
+{
+ $opt_debug
+ f_ex_an_ar_dir="$1"; shift
+ f_ex_an_ar_oldlib="$1"
+ func_show_eval "(cd \$f_ex_an_ar_dir && $AR x \"\$f_ex_an_ar_oldlib\")" 'exit $?'
+ if ($AR t "$f_ex_an_ar_oldlib" | sort | sort -uc >/dev/null 2>&1); then
+ :
+ else
+ func_fatal_error "object name conflicts in archive: $f_ex_an_ar_dir/$f_ex_an_ar_oldlib"
+ fi
+}
+
+
+# func_extract_archives gentop oldlib ...
+func_extract_archives ()
+{
+ $opt_debug
+ my_gentop="$1"; shift
+ my_oldlibs=${1+"$@"}
+ my_oldobjs=""
+ my_xlib=""
+ my_xabs=""
+ my_xdir=""
+
+ for my_xlib in $my_oldlibs; do
+ # Extract the objects.
+ case $my_xlib in
+ [\\/]* | [A-Za-z]:[\\/]*) my_xabs="$my_xlib" ;;
+ *) my_xabs=`pwd`"/$my_xlib" ;;
+ esac
+ func_basename "$my_xlib"
+ my_xlib="$func_basename_result"
+ my_xlib_u=$my_xlib
+ while :; do
+ case " $extracted_archives " in
+ *" $my_xlib_u "*)
+ func_arith $extracted_serial + 1
+ extracted_serial=$func_arith_result
+ my_xlib_u=lt$extracted_serial-$my_xlib ;;
+ *) break ;;
+ esac
+ done
+ extracted_archives="$extracted_archives $my_xlib_u"
+ my_xdir="$my_gentop/$my_xlib_u"
+
+ func_mkdir_p "$my_xdir"
+
+ case $host in
+ *-darwin*)
+ func_verbose "Extracting $my_xabs"
+ # Do not bother doing anything if just a dry run
+ $opt_dry_run || {
+ darwin_orig_dir=`pwd`
+ cd $my_xdir || exit $?
+ darwin_archive=$my_xabs
+ darwin_curdir=`pwd`
+ darwin_base_archive=`basename "$darwin_archive"`
+ darwin_arches=`$LIPO -info "$darwin_archive" 2>/dev/null | $GREP Architectures 2>/dev/null || true`
+ if test -n "$darwin_arches"; then
+ darwin_arches=`$ECHO "$darwin_arches" | $SED -e 's/.*are://'`
+ darwin_arch=
+ func_verbose "$darwin_base_archive has multiple architectures $darwin_arches"
+ for darwin_arch in $darwin_arches ; do
+ func_mkdir_p "unfat-$$/${darwin_base_archive}-${darwin_arch}"
+ $LIPO -thin $darwin_arch -output "unfat-$$/${darwin_base_archive}-${darwin_arch}/${darwin_base_archive}" "${darwin_archive}"
+ cd "unfat-$$/${darwin_base_archive}-${darwin_arch}"
+ func_extract_an_archive "`pwd`" "${darwin_base_archive}"
+ cd "$darwin_curdir"
+ $RM "unfat-$$/${darwin_base_archive}-${darwin_arch}/${darwin_base_archive}"
+ done # $darwin_arches
+ ## Okay now we've a bunch of thin objects, gotta fatten them up :)
+ darwin_filelist=`find unfat-$$ -type f -name \*.o -print -o -name \*.lo -print | $SED -e "$basename" | sort -u`
+ darwin_file=
+ darwin_files=
+ for darwin_file in $darwin_filelist; do
+ darwin_files=`find unfat-$$ -name $darwin_file -print | $NL2SP`
+ $LIPO -create -output "$darwin_file" $darwin_files
+ done # $darwin_filelist
+ $RM -rf unfat-$$
+ cd "$darwin_orig_dir"
+ else
+ cd $darwin_orig_dir
+ func_extract_an_archive "$my_xdir" "$my_xabs"
+ fi # $darwin_arches
+ } # !$opt_dry_run
+ ;;
+ *)
+ func_extract_an_archive "$my_xdir" "$my_xabs"
+ ;;
+ esac
+ my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | $NL2SP`
+ done
+
+ func_extract_archives_result="$my_oldobjs"
+}
+
+
+
+# func_emit_wrapper_part1 [arg=no]
+#
+# Emit the first part of a libtool wrapper script on stdout.
+# For more information, see the description associated with
+# func_emit_wrapper(), below.
+func_emit_wrapper_part1 ()
+{
+ func_emit_wrapper_part1_arg1=no
+ if test -n "$1" ; then
+ func_emit_wrapper_part1_arg1=$1
+ fi
+
+ $ECHO "\
+#! $SHELL
+
+# $output - temporary wrapper script for $objdir/$outputname
+# Generated by $PROGRAM (GNU $PACKAGE$TIMESTAMP) $VERSION
+#
+# The $output program cannot be directly executed until all the libtool
+# libraries that it depends on are installed.
+#
+# This wrapper script should never be moved out of the build directory.
+# If it is, it will not operate correctly.
+
+# Sed substitution that helps us do robust quoting. It backslashifies
+# metacharacters that are still active within double-quoted strings.
+Xsed='${SED} -e 1s/^X//'
+sed_quote_subst='$sed_quote_subst'
+
+# Be Bourne compatible
+if test -n \"\${ZSH_VERSION+set}\" && (emulate sh) >/dev/null 2>&1; then
+ emulate sh
+ NULLCMD=:
+ # Zsh 3.x and 4.x performs word splitting on \${1+\"\$@\"}, which
+ # is contrary to our usage. Disable this feature.
+ alias -g '\${1+\"\$@\"}'='\"\$@\"'
+ setopt NO_GLOB_SUBST
+else
+ case \`(set -o) 2>/dev/null\` in *posix*) set -o posix;; esac
+fi
+BIN_SH=xpg4; export BIN_SH # for Tru64
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# The HP-UX ksh and POSIX shell print the target directory to stdout
+# if CDPATH is set.
+(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
+
+relink_command=\"$relink_command\"
+
+# This environment variable determines our operation mode.
+if test \"\$libtool_install_magic\" = \"$magic\"; then
+ # install mode needs the following variables:
+ generated_by_libtool_version='$macro_version'
+ notinst_deplibs='$notinst_deplibs'
+else
+ # When we are sourced in execute mode, \$file and \$ECHO are already set.
+ if test \"\$libtool_execute_magic\" != \"$magic\"; then
+ ECHO=\"$qecho\"
+ file=\"\$0\"
+ # Make sure echo works.
+ if test \"X\$1\" = X--no-reexec; then
+ # Discard the --no-reexec flag, and continue.
+ shift
+ elif test \"X\`{ \$ECHO '\t'; } 2>/dev/null\`\" = 'X\t'; then
+ # Yippee, \$ECHO works!
+ :
+ else
+ # Restart under the correct shell, and then maybe \$ECHO will work.
+ exec $SHELL \"\$0\" --no-reexec \${1+\"\$@\"}
+ fi
+ fi\
+"
+ $ECHO "\
+
+ # Find the directory that this script lives in.
+ thisdir=\`\$ECHO \"X\$file\" | \$Xsed -e 's%/[^/]*$%%'\`
+ test \"x\$thisdir\" = \"x\$file\" && thisdir=.
+
+ # Follow symbolic links until we get to the real thisdir.
+ file=\`ls -ld \"\$file\" | ${SED} -n 's/.*-> //p'\`
+ while test -n \"\$file\"; do
+ destdir=\`\$ECHO \"X\$file\" | \$Xsed -e 's%/[^/]*\$%%'\`
+
+ # If there was a directory component, then change thisdir.
+ if test \"x\$destdir\" != \"x\$file\"; then
+ case \"\$destdir\" in
+ [\\\\/]* | [A-Za-z]:[\\\\/]*) thisdir=\"\$destdir\" ;;
+ *) thisdir=\"\$thisdir/\$destdir\" ;;
+ esac
+ fi
+
+ file=\`\$ECHO \"X\$file\" | \$Xsed -e 's%^.*/%%'\`
+ file=\`ls -ld \"\$thisdir/\$file\" | ${SED} -n 's/.*-> //p'\`
+ done
+"
+}
+# end: func_emit_wrapper_part1
+
+# func_emit_wrapper_part2 [arg=no]
+#
+# Emit the second part of a libtool wrapper script on stdout.
+# For more information, see the description associated with
+# func_emit_wrapper(), below.
+func_emit_wrapper_part2 ()
+{
+ func_emit_wrapper_part2_arg1=no
+ if test -n "$1" ; then
+ func_emit_wrapper_part2_arg1=$1
+ fi
+
+ $ECHO "\
+
+ # Usually 'no', except on cygwin/mingw when embedded into
+ # the cwrapper.
+ WRAPPER_SCRIPT_BELONGS_IN_OBJDIR=$func_emit_wrapper_part2_arg1
+ if test \"\$WRAPPER_SCRIPT_BELONGS_IN_OBJDIR\" = \"yes\"; then
+ # special case for '.'
+ if test \"\$thisdir\" = \".\"; then
+ thisdir=\`pwd\`
+ fi
+ # remove .libs from thisdir
+ case \"\$thisdir\" in
+ *[\\\\/]$objdir ) thisdir=\`\$ECHO \"X\$thisdir\" | \$Xsed -e 's%[\\\\/][^\\\\/]*$%%'\` ;;
+ $objdir ) thisdir=. ;;
+ esac
+ fi
+
+ # Try to get the absolute directory name.
+ absdir=\`cd \"\$thisdir\" && pwd\`
+ test -n \"\$absdir\" && thisdir=\"\$absdir\"
+"
+
+ if test "$fast_install" = yes; then
+ $ECHO "\
+ program=lt-'$outputname'$exeext
+ progdir=\"\$thisdir/$objdir\"
+
+ if test ! -f \"\$progdir/\$program\" ||
+ { file=\`ls -1dt \"\$progdir/\$program\" \"\$progdir/../\$program\" 2>/dev/null | ${SED} 1q\`; \\
+ test \"X\$file\" != \"X\$progdir/\$program\"; }; then
+
+ file=\"\$\$-\$program\"
+
+ if test ! -d \"\$progdir\"; then
+ $MKDIR \"\$progdir\"
+ else
+ $RM \"\$progdir/\$file\"
+ fi"
+
+ $ECHO "\
+
+ # relink executable if necessary
+ if test -n \"\$relink_command\"; then
+ if relink_command_output=\`eval \$relink_command 2>&1\`; then :
+ else
+ $ECHO \"\$relink_command_output\" >&2
+ $RM \"\$progdir/\$file\"
+ exit 1
+ fi
+ fi
+
+ $MV \"\$progdir/\$file\" \"\$progdir/\$program\" 2>/dev/null ||
+ { $RM \"\$progdir/\$program\";
+ $MV \"\$progdir/\$file\" \"\$progdir/\$program\"; }
+ $RM \"\$progdir/\$file\"
+ fi"
+ else
+ $ECHO "\
+ program='$outputname'
+ progdir=\"\$thisdir/$objdir\"
+"
+ fi
+
+ $ECHO "\
+
+ if test -f \"\$progdir/\$program\"; then"
+
+ # Export our shlibpath_var if we have one.
+ if test "$shlibpath_overrides_runpath" = yes && test -n "$shlibpath_var" && test -n "$temp_rpath"; then
+ $ECHO "\
+ # Add our own library path to $shlibpath_var
+ $shlibpath_var=\"$temp_rpath\$$shlibpath_var\"
+
+ # Some systems cannot cope with colon-terminated $shlibpath_var
+ # The second colon is a workaround for a bug in BeOS R4 sed
+ $shlibpath_var=\`\$ECHO \"X\$$shlibpath_var\" | \$Xsed -e 's/::*\$//'\`
+
+ export $shlibpath_var
+"
+ fi
+
+ # fixup the dll searchpath if we need to.
+ if test -n "$dllsearchpath"; then
+ $ECHO "\
+ # Add the dll search path components to the executable PATH
+ PATH=$dllsearchpath:\$PATH
+"
+ fi
+
+ $ECHO "\
+ if test \"\$libtool_execute_magic\" != \"$magic\"; then
+ # Run the actual program with our arguments.
+"
+ case $host in
+ # Backslashes separate directories on plain windows
+ *-*-mingw | *-*-os2* | *-cegcc*)
+ $ECHO "\
+ exec \"\$progdir\\\\\$program\" \${1+\"\$@\"}
+"
+ ;;
+
+ *)
+ $ECHO "\
+ exec \"\$progdir/\$program\" \${1+\"\$@\"}
+"
+ ;;
+ esac
+ $ECHO "\
+ \$ECHO \"\$0: cannot exec \$program \$*\" 1>&2
+ exit 1
+ fi
+ else
+ # The program doesn't exist.
+ \$ECHO \"\$0: error: \\\`\$progdir/\$program' does not exist\" 1>&2
+ \$ECHO \"This script is just a wrapper for \$program.\" 1>&2
+ $ECHO \"See the $PACKAGE documentation for more information.\" 1>&2
+ exit 1
+ fi
+fi\
+"
+}
+# end: func_emit_wrapper_part2
+
+
+# func_emit_wrapper [arg=no]
+#
+# Emit a libtool wrapper script on stdout.
+# Don't directly open a file because we may want to
+# incorporate the script contents within a cygwin/mingw
+# wrapper executable. Must ONLY be called from within
+# func_mode_link because it depends on a number of variables
+# set therein.
+#
+# ARG is the value that the WRAPPER_SCRIPT_BELONGS_IN_OBJDIR
+# variable will take. If 'yes', then the emitted script
+# will assume that the directory in which it is stored is
+# the $objdir directory. This is a cygwin/mingw-specific
+# behavior.
+func_emit_wrapper ()
+{
+ func_emit_wrapper_arg1=no
+ if test -n "$1" ; then
+ func_emit_wrapper_arg1=$1
+ fi
+
+ # split this up so that func_emit_cwrapperexe_src
+ # can call each part independently.
+ func_emit_wrapper_part1 "${func_emit_wrapper_arg1}"
+ func_emit_wrapper_part2 "${func_emit_wrapper_arg1}"
+}
+
+
+# func_to_host_path arg
+#
+# Convert paths to host format when used with build tools.
+# Intended for use with "native" mingw (where libtool itself
+# is running under the msys shell), or in the following cross-
+# build environments:
+# $build $host
+# mingw (msys) mingw [e.g. native]
+# cygwin mingw
+# *nix + wine mingw
+# where wine is equipped with the `winepath' executable.
+# In the native mingw case, the (msys) shell automatically
+# converts paths for any non-msys applications it launches,
+# but that facility isn't available from inside the cwrapper.
+# Similar accommodations are necessary for $host mingw and
+# $build cygwin. Calling this function does no harm for other
+# $host/$build combinations not listed above.
+#
+# ARG is the path (on $build) that should be converted to
+# the proper representation for $host. The result is stored
+# in $func_to_host_path_result.
+func_to_host_path ()
+{
+ func_to_host_path_result="$1"
+ if test -n "$1" ; then
+ case $host in
+ *mingw* )
+ lt_sed_naive_backslashify='s|\\\\*|\\|g;s|/|\\|g;s|\\|\\\\|g'
+ case $build in
+ *mingw* ) # actually, msys
+ # awkward: cmd appends spaces to result
+ lt_sed_strip_trailing_spaces="s/[ ]*\$//"
+ func_to_host_path_tmp1=`( cmd //c echo "$1" |\
+ $SED -e "$lt_sed_strip_trailing_spaces" ) 2>/dev/null || echo ""`
+ func_to_host_path_result=`echo "$func_to_host_path_tmp1" |\
+ $SED -e "$lt_sed_naive_backslashify"`
+ ;;
+ *cygwin* )
+ func_to_host_path_tmp1=`cygpath -w "$1"`
+ func_to_host_path_result=`echo "$func_to_host_path_tmp1" |\
+ $SED -e "$lt_sed_naive_backslashify"`
+ ;;
+ * )
+ # Unfortunately, winepath does not exit with a non-zero
+ # error code, so we are forced to check the contents of
+ # stdout. On the other hand, if the command is not
+ # found, the shell will set an exit code of 127 and print
+ # *an error message* to stdout. So we must check for both
+ # error code of zero AND non-empty stdout, which explains
+ # the odd construction:
+ func_to_host_path_tmp1=`winepath -w "$1" 2>/dev/null`
+ if test "$?" -eq 0 && test -n "${func_to_host_path_tmp1}"; then
+ func_to_host_path_result=`echo "$func_to_host_path_tmp1" |\
+ $SED -e "$lt_sed_naive_backslashify"`
+ else
+ # Allow warning below.
+ func_to_host_path_result=""
+ fi
+ ;;
+ esac
+ if test -z "$func_to_host_path_result" ; then
+ func_error "Could not determine host path corresponding to"
+ func_error " '$1'"
+ func_error "Continuing, but uninstalled executables may not work."
+ # Fallback:
+ func_to_host_path_result="$1"
+ fi
+ ;;
+ esac
+ fi
+}
+# end: func_to_host_path
+
+# func_to_host_pathlist arg
+#
+# Convert pathlists to host format when used with build tools.
+# See func_to_host_path(), above. This function supports the
+# following $build/$host combinations (but does no harm for
+# combinations not listed here):
+# $build $host
+# mingw (msys) mingw [e.g. native]
+# cygwin mingw
+# *nix + wine mingw
+#
+# Path separators are also converted from $build format to
+# $host format. If ARG begins or ends with a path separator
+# character, it is preserved (but converted to $host format)
+# on output.
+#
+# ARG is a pathlist (on $build) that should be converted to
+# the proper representation on $host. The result is stored
+# in $func_to_host_pathlist_result.
+func_to_host_pathlist ()
+{
+ func_to_host_pathlist_result="$1"
+ if test -n "$1" ; then
+ case $host in
+ *mingw* )
+ lt_sed_naive_backslashify='s|\\\\*|\\|g;s|/|\\|g;s|\\|\\\\|g'
+ # Remove leading and trailing path separator characters from
+ # ARG. msys behavior is inconsistent here, cygpath turns them
+ # into '.;' and ';.', and winepath ignores them completely.
+ func_to_host_pathlist_tmp2="$1"
+ # Once set for this call, this variable should not be
+ # reassigned. It is used in tha fallback case.
+ func_to_host_pathlist_tmp1=`echo "$func_to_host_pathlist_tmp2" |\
+ $SED -e 's|^:*||' -e 's|:*$||'`
+ case $build in
+ *mingw* ) # Actually, msys.
+ # Awkward: cmd appends spaces to result.
+ lt_sed_strip_trailing_spaces="s/[ ]*\$//"
+ func_to_host_pathlist_tmp2=`( cmd //c echo "$func_to_host_pathlist_tmp1" |\
+ $SED -e "$lt_sed_strip_trailing_spaces" ) 2>/dev/null || echo ""`
+ func_to_host_pathlist_result=`echo "$func_to_host_pathlist_tmp2" |\
+ $SED -e "$lt_sed_naive_backslashify"`
+ ;;
+ *cygwin* )
+ func_to_host_pathlist_tmp2=`cygpath -w -p "$func_to_host_pathlist_tmp1"`
+ func_to_host_pathlist_result=`echo "$func_to_host_pathlist_tmp2" |\
+ $SED -e "$lt_sed_naive_backslashify"`
+ ;;
+ * )
+ # unfortunately, winepath doesn't convert pathlists
+ func_to_host_pathlist_result=""
+ func_to_host_pathlist_oldIFS=$IFS
+ IFS=:
+ for func_to_host_pathlist_f in $func_to_host_pathlist_tmp1 ; do
+ IFS=$func_to_host_pathlist_oldIFS
+ if test -n "$func_to_host_pathlist_f" ; then
+ func_to_host_path "$func_to_host_pathlist_f"
+ if test -n "$func_to_host_path_result" ; then
+ if test -z "$func_to_host_pathlist_result" ; then
+ func_to_host_pathlist_result="$func_to_host_path_result"
+ else
+ func_to_host_pathlist_result="$func_to_host_pathlist_result;$func_to_host_path_result"
+ fi
+ fi
+ fi
+ IFS=:
+ done
+ IFS=$func_to_host_pathlist_oldIFS
+ ;;
+ esac
+ if test -z "$func_to_host_pathlist_result" ; then
+ func_error "Could not determine the host path(s) corresponding to"
+ func_error " '$1'"
+ func_error "Continuing, but uninstalled executables may not work."
+ # Fallback. This may break if $1 contains DOS-style drive
+ # specifications. The fix is not to complicate the expression
+ # below, but for the user to provide a working wine installation
+ # with winepath so that path translation in the cross-to-mingw
+ # case works properly.
+ lt_replace_pathsep_nix_to_dos="s|:|;|g"
+ func_to_host_pathlist_result=`echo "$func_to_host_pathlist_tmp1" |\
+ $SED -e "$lt_replace_pathsep_nix_to_dos"`
+ fi
+ # Now, add the leading and trailing path separators back
+ case "$1" in
+ :* ) func_to_host_pathlist_result=";$func_to_host_pathlist_result"
+ ;;
+ esac
+ case "$1" in
+ *: ) func_to_host_pathlist_result="$func_to_host_pathlist_result;"
+ ;;
+ esac
+ ;;
+ esac
+ fi
+}
+# end: func_to_host_pathlist
+
+# func_emit_cwrapperexe_src
+# emit the source code for a wrapper executable on stdout
+# Must ONLY be called from within func_mode_link because
+# it depends on a number of variable set therein.
+func_emit_cwrapperexe_src ()
+{
+ cat <<EOF
+
+/* $cwrappersource - temporary wrapper executable for $objdir/$outputname
+ Generated by $PROGRAM (GNU $PACKAGE$TIMESTAMP) $VERSION
+
+ The $output program cannot be directly executed until all the libtool
+ libraries that it depends on are installed.
+
+ This wrapper executable should never be moved out of the build directory.
+ If it is, it will not operate correctly.
+
+ Currently, it simply execs the wrapper *script* "$SHELL $output",
+ but could eventually absorb all of the scripts functionality and
+ exec $objdir/$outputname directly.
+*/
+EOF
+ cat <<"EOF"
+#include <stdio.h>
+#include <stdlib.h>
+#ifdef _MSC_VER
+# include <direct.h>
+# include <process.h>
+# include <io.h>
+# define setmode _setmode
+#else
+# include <unistd.h>
+# include <stdint.h>
+# ifdef __CYGWIN__
+# include <io.h>
+# define HAVE_SETENV
+# ifdef __STRICT_ANSI__
+char *realpath (const char *, char *);
+int putenv (char *);
+int setenv (const char *, const char *, int);
+# endif
+# endif
+#endif
+#include <malloc.h>
+#include <stdarg.h>
+#include <assert.h>
+#include <string.h>
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+
+#if defined(PATH_MAX)
+# define LT_PATHMAX PATH_MAX
+#elif defined(MAXPATHLEN)
+# define LT_PATHMAX MAXPATHLEN
+#else
+# define LT_PATHMAX 1024
+#endif
+
+#ifndef S_IXOTH
+# define S_IXOTH 0
+#endif
+#ifndef S_IXGRP
+# define S_IXGRP 0
+#endif
+
+#ifdef _MSC_VER
+# define S_IXUSR _S_IEXEC
+# define stat _stat
+# ifndef _INTPTR_T_DEFINED
+# define intptr_t int
+# endif
+#endif
+
+#ifndef DIR_SEPARATOR
+# define DIR_SEPARATOR '/'
+# define PATH_SEPARATOR ':'
+#endif
+
+#if defined (_WIN32) || defined (__MSDOS__) || defined (__DJGPP__) || \
+ defined (__OS2__)
+# define HAVE_DOS_BASED_FILE_SYSTEM
+# define FOPEN_WB "wb"
+# ifndef DIR_SEPARATOR_2
+# define DIR_SEPARATOR_2 '\\'
+# endif
+# ifndef PATH_SEPARATOR_2
+# define PATH_SEPARATOR_2 ';'
+# endif
+#endif
+
+#ifndef DIR_SEPARATOR_2
+# define IS_DIR_SEPARATOR(ch) ((ch) == DIR_SEPARATOR)
+#else /* DIR_SEPARATOR_2 */
+# define IS_DIR_SEPARATOR(ch) \
+ (((ch) == DIR_SEPARATOR) || ((ch) == DIR_SEPARATOR_2))
+#endif /* DIR_SEPARATOR_2 */
+
+#ifndef PATH_SEPARATOR_2
+# define IS_PATH_SEPARATOR(ch) ((ch) == PATH_SEPARATOR)
+#else /* PATH_SEPARATOR_2 */
+# define IS_PATH_SEPARATOR(ch) ((ch) == PATH_SEPARATOR_2)
+#endif /* PATH_SEPARATOR_2 */
+
+#ifdef __CYGWIN__
+# define FOPEN_WB "wb"
+#endif
+
+#ifndef FOPEN_WB
+# define FOPEN_WB "w"
+#endif
+#ifndef _O_BINARY
+# define _O_BINARY 0
+#endif
+
+#define XMALLOC(type, num) ((type *) xmalloc ((num) * sizeof(type)))
+#define XFREE(stale) do { \
+ if (stale) { free ((void *) stale); stale = 0; } \
+} while (0)
+
+#undef LTWRAPPER_DEBUGPRINTF
+#if defined DEBUGWRAPPER
+# define LTWRAPPER_DEBUGPRINTF(args) ltwrapper_debugprintf args
+static void
+ltwrapper_debugprintf (const char *fmt, ...)
+{
+ va_list args;
+ va_start (args, fmt);
+ (void) vfprintf (stderr, fmt, args);
+ va_end (args);
+}
+#else
+# define LTWRAPPER_DEBUGPRINTF(args)
+#endif
+
+const char *program_name = NULL;
+
+void *xmalloc (size_t num);
+char *xstrdup (const char *string);
+const char *base_name (const char *name);
+char *find_executable (const char *wrapper);
+char *chase_symlinks (const char *pathspec);
+int make_executable (const char *path);
+int check_executable (const char *path);
+char *strendzap (char *str, const char *pat);
+void lt_fatal (const char *message, ...);
+void lt_setenv (const char *name, const char *value);
+char *lt_extend_str (const char *orig_value, const char *add, int to_end);
+void lt_opt_process_env_set (const char *arg);
+void lt_opt_process_env_prepend (const char *arg);
+void lt_opt_process_env_append (const char *arg);
+int lt_split_name_value (const char *arg, char** name, char** value);
+void lt_update_exe_path (const char *name, const char *value);
+void lt_update_lib_path (const char *name, const char *value);
+
+static const char *script_text_part1 =
+EOF
+
+ func_emit_wrapper_part1 yes |
+ $SED -e 's/\([\\"]\)/\\\1/g' \
+ -e 's/^/ "/' -e 's/$/\\n"/'
+ echo ";"
+ cat <<EOF
+
+static const char *script_text_part2 =
+EOF
+ func_emit_wrapper_part2 yes |
+ $SED -e 's/\([\\"]\)/\\\1/g' \
+ -e 's/^/ "/' -e 's/$/\\n"/'
+ echo ";"
+
+ cat <<EOF
+const char * MAGIC_EXE = "$magic_exe";
+const char * LIB_PATH_VARNAME = "$shlibpath_var";
+EOF
+
+ if test "$shlibpath_overrides_runpath" = yes && test -n "$shlibpath_var" && test -n "$temp_rpath"; then
+ func_to_host_pathlist "$temp_rpath"
+ cat <<EOF
+const char * LIB_PATH_VALUE = "$func_to_host_pathlist_result";
+EOF
+ else
+ cat <<"EOF"
+const char * LIB_PATH_VALUE = "";
+EOF
+ fi
+
+ if test -n "$dllsearchpath"; then
+ func_to_host_pathlist "$dllsearchpath:"
+ cat <<EOF
+const char * EXE_PATH_VARNAME = "PATH";
+const char * EXE_PATH_VALUE = "$func_to_host_pathlist_result";
+EOF
+ else
+ cat <<"EOF"
+const char * EXE_PATH_VARNAME = "";
+const char * EXE_PATH_VALUE = "";
+EOF
+ fi
+
+ if test "$fast_install" = yes; then
+ cat <<EOF
+const char * TARGET_PROGRAM_NAME = "lt-$outputname"; /* hopefully, no .exe */
+EOF
+ else
+ cat <<EOF
+const char * TARGET_PROGRAM_NAME = "$outputname"; /* hopefully, no .exe */
+EOF
+ fi
+
+
+ cat <<"EOF"
+
+#define LTWRAPPER_OPTION_PREFIX "--lt-"
+#define LTWRAPPER_OPTION_PREFIX_LENGTH 5
+
+static const size_t opt_prefix_len = LTWRAPPER_OPTION_PREFIX_LENGTH;
+static const char *ltwrapper_option_prefix = LTWRAPPER_OPTION_PREFIX;
+
+static const char *dumpscript_opt = LTWRAPPER_OPTION_PREFIX "dump-script";
+
+static const size_t env_set_opt_len = LTWRAPPER_OPTION_PREFIX_LENGTH + 7;
+static const char *env_set_opt = LTWRAPPER_OPTION_PREFIX "env-set";
+ /* argument is putenv-style "foo=bar", value of foo is set to bar */
+
+static const size_t env_prepend_opt_len = LTWRAPPER_OPTION_PREFIX_LENGTH + 11;
+static const char *env_prepend_opt = LTWRAPPER_OPTION_PREFIX "env-prepend";
+ /* argument is putenv-style "foo=bar", new value of foo is bar${foo} */
+
+static const size_t env_append_opt_len = LTWRAPPER_OPTION_PREFIX_LENGTH + 10;
+static const char *env_append_opt = LTWRAPPER_OPTION_PREFIX "env-append";
+ /* argument is putenv-style "foo=bar", new value of foo is ${foo}bar */
+
+int
+main (int argc, char *argv[])
+{
+ char **newargz;
+ int newargc;
+ char *tmp_pathspec;
+ char *actual_cwrapper_path;
+ char *actual_cwrapper_name;
+ char *target_name;
+ char *lt_argv_zero;
+ intptr_t rval = 127;
+
+ int i;
+
+ program_name = (char *) xstrdup (base_name (argv[0]));
+ LTWRAPPER_DEBUGPRINTF (("(main) argv[0] : %s\n", argv[0]));
+ LTWRAPPER_DEBUGPRINTF (("(main) program_name : %s\n", program_name));
+
+ /* very simple arg parsing; don't want to rely on getopt */
+ for (i = 1; i < argc; i++)
+ {
+ if (strcmp (argv[i], dumpscript_opt) == 0)
+ {
+EOF
+ case "$host" in
+ *mingw* | *cygwin* )
+ # make stdout use "unix" line endings
+ echo " setmode(1,_O_BINARY);"
+ ;;
+ esac
+
+ cat <<"EOF"
+ printf ("%s", script_text_part1);
+ printf ("%s", script_text_part2);
+ return 0;
+ }
+ }
+
+ newargz = XMALLOC (char *, argc + 1);
+ tmp_pathspec = find_executable (argv[0]);
+ if (tmp_pathspec == NULL)
+ lt_fatal ("Couldn't find %s", argv[0]);
+ LTWRAPPER_DEBUGPRINTF (("(main) found exe (before symlink chase) at : %s\n",
+ tmp_pathspec));
+
+ actual_cwrapper_path = chase_symlinks (tmp_pathspec);
+ LTWRAPPER_DEBUGPRINTF (("(main) found exe (after symlink chase) at : %s\n",
+ actual_cwrapper_path));
+ XFREE (tmp_pathspec);
+
+ actual_cwrapper_name = xstrdup( base_name (actual_cwrapper_path));
+ strendzap (actual_cwrapper_path, actual_cwrapper_name);
+
+ /* wrapper name transforms */
+ strendzap (actual_cwrapper_name, ".exe");
+ tmp_pathspec = lt_extend_str (actual_cwrapper_name, ".exe", 1);
+ XFREE (actual_cwrapper_name);
+ actual_cwrapper_name = tmp_pathspec;
+ tmp_pathspec = 0;
+
+ /* target_name transforms -- use actual target program name; might have lt- prefix */
+ target_name = xstrdup (base_name (TARGET_PROGRAM_NAME));
+ strendzap (target_name, ".exe");
+ tmp_pathspec = lt_extend_str (target_name, ".exe", 1);
+ XFREE (target_name);
+ target_name = tmp_pathspec;
+ tmp_pathspec = 0;
+
+ LTWRAPPER_DEBUGPRINTF (("(main) libtool target name: %s\n",
+ target_name));
+EOF
+
+ cat <<EOF
+ newargz[0] =
+ XMALLOC (char, (strlen (actual_cwrapper_path) +
+ strlen ("$objdir") + 1 + strlen (actual_cwrapper_name) + 1));
+ strcpy (newargz[0], actual_cwrapper_path);
+ strcat (newargz[0], "$objdir");
+ strcat (newargz[0], "/");
+EOF
+
+ cat <<"EOF"
+ /* stop here, and copy so we don't have to do this twice */
+ tmp_pathspec = xstrdup (newargz[0]);
+
+ /* do NOT want the lt- prefix here, so use actual_cwrapper_name */
+ strcat (newargz[0], actual_cwrapper_name);
+
+ /* DO want the lt- prefix here if it exists, so use target_name */
+ lt_argv_zero = lt_extend_str (tmp_pathspec, target_name, 1);
+ XFREE (tmp_pathspec);
+ tmp_pathspec = NULL;
+EOF
+
+ case $host_os in
+ mingw*)
+ cat <<"EOF"
+ {
+ char* p;
+ while ((p = strchr (newargz[0], '\\')) != NULL)
+ {
+ *p = '/';
+ }
+ while ((p = strchr (lt_argv_zero, '\\')) != NULL)
+ {
+ *p = '/';
+ }
+ }
+EOF
+ ;;
+ esac
+
+ cat <<"EOF"
+ XFREE (target_name);
+ XFREE (actual_cwrapper_path);
+ XFREE (actual_cwrapper_name);
+
+ lt_setenv ("BIN_SH", "xpg4"); /* for Tru64 */
+ lt_setenv ("DUALCASE", "1"); /* for MSK sh */
+ lt_update_lib_path (LIB_PATH_VARNAME, LIB_PATH_VALUE);
+ lt_update_exe_path (EXE_PATH_VARNAME, EXE_PATH_VALUE);
+
+ newargc=0;
+ for (i = 1; i < argc; i++)
+ {
+ if (strncmp (argv[i], env_set_opt, env_set_opt_len) == 0)
+ {
+ if (argv[i][env_set_opt_len] == '=')
+ {
+ const char *p = argv[i] + env_set_opt_len + 1;
+ lt_opt_process_env_set (p);
+ }
+ else if (argv[i][env_set_opt_len] == '\0' && i + 1 < argc)
+ {
+ lt_opt_process_env_set (argv[++i]); /* don't copy */
+ }
+ else
+ lt_fatal ("%s missing required argument", env_set_opt);
+ continue;
+ }
+ if (strncmp (argv[i], env_prepend_opt, env_prepend_opt_len) == 0)
+ {
+ if (argv[i][env_prepend_opt_len] == '=')
+ {
+ const char *p = argv[i] + env_prepend_opt_len + 1;
+ lt_opt_process_env_prepend (p);
+ }
+ else if (argv[i][env_prepend_opt_len] == '\0' && i + 1 < argc)
+ {
+ lt_opt_process_env_prepend (argv[++i]); /* don't copy */
+ }
+ else
+ lt_fatal ("%s missing required argument", env_prepend_opt);
+ continue;
+ }
+ if (strncmp (argv[i], env_append_opt, env_append_opt_len) == 0)
+ {
+ if (argv[i][env_append_opt_len] == '=')
+ {
+ const char *p = argv[i] + env_append_opt_len + 1;
+ lt_opt_process_env_append (p);
+ }
+ else if (argv[i][env_append_opt_len] == '\0' && i + 1 < argc)
+ {
+ lt_opt_process_env_append (argv[++i]); /* don't copy */
+ }
+ else
+ lt_fatal ("%s missing required argument", env_append_opt);
+ continue;
+ }
+ if (strncmp (argv[i], ltwrapper_option_prefix, opt_prefix_len) == 0)
+ {
+ /* however, if there is an option in the LTWRAPPER_OPTION_PREFIX
+ namespace, but it is not one of the ones we know about and
+ have already dealt with, above (inluding dump-script), then
+ report an error. Otherwise, targets might begin to believe
+ they are allowed to use options in the LTWRAPPER_OPTION_PREFIX
+ namespace. The first time any user complains about this, we'll
+ need to make LTWRAPPER_OPTION_PREFIX a configure-time option
+ or a configure.ac-settable value.
+ */
+ lt_fatal ("Unrecognized option in %s namespace: '%s'",
+ ltwrapper_option_prefix, argv[i]);
+ }
+ /* otherwise ... */
+ newargz[++newargc] = xstrdup (argv[i]);
+ }
+ newargz[++newargc] = NULL;
+
+ LTWRAPPER_DEBUGPRINTF (("(main) lt_argv_zero : %s\n", (lt_argv_zero ? lt_argv_zero : "<NULL>")));
+ for (i = 0; i < newargc; i++)
+ {
+ LTWRAPPER_DEBUGPRINTF (("(main) newargz[%d] : %s\n", i, (newargz[i] ? newargz[i] : "<NULL>")));
+ }
+
+EOF
+
+ case $host_os in
+ mingw*)
+ cat <<"EOF"
+ /* execv doesn't actually work on mingw as expected on unix */
+ rval = _spawnv (_P_WAIT, lt_argv_zero, (const char * const *) newargz);
+ if (rval == -1)
+ {
+ /* failed to start process */
+ LTWRAPPER_DEBUGPRINTF (("(main) failed to launch target \"%s\": errno = %d\n", lt_argv_zero, errno));
+ return 127;
+ }
+ return rval;
+EOF
+ ;;
+ *)
+ cat <<"EOF"
+ execv (lt_argv_zero, newargz);
+ return rval; /* =127, but avoids unused variable warning */
+EOF
+ ;;
+ esac
+
+ cat <<"EOF"
+}
+
+void *
+xmalloc (size_t num)
+{
+ void *p = (void *) malloc (num);
+ if (!p)
+ lt_fatal ("Memory exhausted");
+
+ return p;
+}
+
+char *
+xstrdup (const char *string)
+{
+ return string ? strcpy ((char *) xmalloc (strlen (string) + 1),
+ string) : NULL;
+}
+
+const char *
+base_name (const char *name)
+{
+ const char *base;
+
+#if defined (HAVE_DOS_BASED_FILE_SYSTEM)
+ /* Skip over the disk name in MSDOS pathnames. */
+ if (isalpha ((unsigned char) name[0]) && name[1] == ':')
+ name += 2;
+#endif
+
+ for (base = name; *name; name++)
+ if (IS_DIR_SEPARATOR (*name))
+ base = name + 1;
+ return base;
+}
+
+int
+check_executable (const char *path)
+{
+ struct stat st;
+
+ LTWRAPPER_DEBUGPRINTF (("(check_executable) : %s\n",
+ path ? (*path ? path : "EMPTY!") : "NULL!"));
+ if ((!path) || (!*path))
+ return 0;
+
+ if ((stat (path, &st) >= 0)
+ && (st.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH)))
+ return 1;
+ else
+ return 0;
+}
+
+int
+make_executable (const char *path)
+{
+ int rval = 0;
+ struct stat st;
+
+ LTWRAPPER_DEBUGPRINTF (("(make_executable) : %s\n",
+ path ? (*path ? path : "EMPTY!") : "NULL!"));
+ if ((!path) || (!*path))
+ return 0;
+
+ if (stat (path, &st) >= 0)
+ {
+ rval = chmod (path, st.st_mode | S_IXOTH | S_IXGRP | S_IXUSR);
+ }
+ return rval;
+}
+
+/* Searches for the full path of the wrapper. Returns
+ newly allocated full path name if found, NULL otherwise
+ Does not chase symlinks, even on platforms that support them.
+*/
+char *
+find_executable (const char *wrapper)
+{
+ int has_slash = 0;
+ const char *p;
+ const char *p_next;
+ /* static buffer for getcwd */
+ char tmp[LT_PATHMAX + 1];
+ int tmp_len;
+ char *concat_name;
+
+ LTWRAPPER_DEBUGPRINTF (("(find_executable) : %s\n",
+ wrapper ? (*wrapper ? wrapper : "EMPTY!") : "NULL!"));
+
+ if ((wrapper == NULL) || (*wrapper == '\0'))
+ return NULL;
+
+ /* Absolute path? */
+#if defined (HAVE_DOS_BASED_FILE_SYSTEM)
+ if (isalpha ((unsigned char) wrapper[0]) && wrapper[1] == ':')
+ {
+ concat_name = xstrdup (wrapper);
+ if (check_executable (concat_name))
+ return concat_name;
+ XFREE (concat_name);
+ }
+ else
+ {
+#endif
+ if (IS_DIR_SEPARATOR (wrapper[0]))
+ {
+ concat_name = xstrdup (wrapper);
+ if (check_executable (concat_name))
+ return concat_name;
+ XFREE (concat_name);
+ }
+#if defined (HAVE_DOS_BASED_FILE_SYSTEM)
+ }
+#endif
+
+ for (p = wrapper; *p; p++)
+ if (*p == '/')
+ {
+ has_slash = 1;
+ break;
+ }
+ if (!has_slash)
+ {
+ /* no slashes; search PATH */
+ const char *path = getenv ("PATH");
+ if (path != NULL)
+ {
+ for (p = path; *p; p = p_next)
+ {
+ const char *q;
+ size_t p_len;
+ for (q = p; *q; q++)
+ if (IS_PATH_SEPARATOR (*q))
+ break;
+ p_len = q - p;
+ p_next = (*q == '\0' ? q : q + 1);
+ if (p_len == 0)
+ {
+ /* empty path: current directory */
+ if (getcwd (tmp, LT_PATHMAX) == NULL)
+ lt_fatal ("getcwd failed");
+ tmp_len = strlen (tmp);
+ concat_name =
+ XMALLOC (char, tmp_len + 1 + strlen (wrapper) + 1);
+ memcpy (concat_name, tmp, tmp_len);
+ concat_name[tmp_len] = '/';
+ strcpy (concat_name + tmp_len + 1, wrapper);
+ }
+ else
+ {
+ concat_name =
+ XMALLOC (char, p_len + 1 + strlen (wrapper) + 1);
+ memcpy (concat_name, p, p_len);
+ concat_name[p_len] = '/';
+ strcpy (concat_name + p_len + 1, wrapper);
+ }
+ if (check_executable (concat_name))
+ return concat_name;
+ XFREE (concat_name);
+ }
+ }
+ /* not found in PATH; assume curdir */
+ }
+ /* Relative path | not found in path: prepend cwd */
+ if (getcwd (tmp, LT_PATHMAX) == NULL)
+ lt_fatal ("getcwd failed");
+ tmp_len = strlen (tmp);
+ concat_name = XMALLOC (char, tmp_len + 1 + strlen (wrapper) + 1);
+ memcpy (concat_name, tmp, tmp_len);
+ concat_name[tmp_len] = '/';
+ strcpy (concat_name + tmp_len + 1, wrapper);
+
+ if (check_executable (concat_name))
+ return concat_name;
+ XFREE (concat_name);
+ return NULL;
+}
+
+char *
+chase_symlinks (const char *pathspec)
+{
+#ifndef S_ISLNK
+ return xstrdup (pathspec);
+#else
+ char buf[LT_PATHMAX];
+ struct stat s;
+ char *tmp_pathspec = xstrdup (pathspec);
+ char *p;
+ int has_symlinks = 0;
+ while (strlen (tmp_pathspec) && !has_symlinks)
+ {
+ LTWRAPPER_DEBUGPRINTF (("checking path component for symlinks: %s\n",
+ tmp_pathspec));
+ if (lstat (tmp_pathspec, &s) == 0)
+ {
+ if (S_ISLNK (s.st_mode) != 0)
+ {
+ has_symlinks = 1;
+ break;
+ }
+
+ /* search backwards for last DIR_SEPARATOR */
+ p = tmp_pathspec + strlen (tmp_pathspec) - 1;
+ while ((p > tmp_pathspec) && (!IS_DIR_SEPARATOR (*p)))
+ p--;
+ if ((p == tmp_pathspec) && (!IS_DIR_SEPARATOR (*p)))
+ {
+ /* no more DIR_SEPARATORS left */
+ break;
+ }
+ *p = '\0';
+ }
+ else
+ {
+ char *errstr = strerror (errno);
+ lt_fatal ("Error accessing file %s (%s)", tmp_pathspec, errstr);
+ }
+ }
+ XFREE (tmp_pathspec);
+
+ if (!has_symlinks)
+ {
+ return xstrdup (pathspec);
+ }
+
+ tmp_pathspec = realpath (pathspec, buf);
+ if (tmp_pathspec == 0)
+ {
+ lt_fatal ("Could not follow symlinks for %s", pathspec);
+ }
+ return xstrdup (tmp_pathspec);
+#endif
+}
+
+char *
+strendzap (char *str, const char *pat)
+{
+ size_t len, patlen;
+
+ assert (str != NULL);
+ assert (pat != NULL);
+
+ len = strlen (str);
+ patlen = strlen (pat);
+
+ if (patlen <= len)
+ {
+ str += len - patlen;
+ if (strcmp (str, pat) == 0)
+ *str = '\0';
+ }
+ return str;
+}
+
+static void
+lt_error_core (int exit_status, const char *mode,
+ const char *message, va_list ap)
+{
+ fprintf (stderr, "%s: %s: ", program_name, mode);
+ vfprintf (stderr, message, ap);
+ fprintf (stderr, ".\n");
+
+ if (exit_status >= 0)
+ exit (exit_status);
+}
+
+void
+lt_fatal (const char *message, ...)
+{
+ va_list ap;
+ va_start (ap, message);
+ lt_error_core (EXIT_FAILURE, "FATAL", message, ap);
+ va_end (ap);
+}
+
+void
+lt_setenv (const char *name, const char *value)
+{
+ LTWRAPPER_DEBUGPRINTF (("(lt_setenv) setting '%s' to '%s'\n",
+ (name ? name : "<NULL>"),
+ (value ? value : "<NULL>")));
+ {
+#ifdef HAVE_SETENV
+ /* always make a copy, for consistency with !HAVE_SETENV */
+ char *str = xstrdup (value);
+ setenv (name, str, 1);
+#else
+ int len = strlen (name) + 1 + strlen (value) + 1;
+ char *str = XMALLOC (char, len);
+ sprintf (str, "%s=%s", name, value);
+ if (putenv (str) != EXIT_SUCCESS)
+ {
+ XFREE (str);
+ }
+#endif
+ }
+}
+
+char *
+lt_extend_str (const char *orig_value, const char *add, int to_end)
+{
+ char *new_value;
+ if (orig_value && *orig_value)
+ {
+ int orig_value_len = strlen (orig_value);
+ int add_len = strlen (add);
+ new_value = XMALLOC (char, add_len + orig_value_len + 1);
+ if (to_end)
+ {
+ strcpy (new_value, orig_value);
+ strcpy (new_value + orig_value_len, add);
+ }
+ else
+ {
+ strcpy (new_value, add);
+ strcpy (new_value + add_len, orig_value);
+ }
+ }
+ else
+ {
+ new_value = xstrdup (add);
+ }
+ return new_value;
+}
+
+int
+lt_split_name_value (const char *arg, char** name, char** value)
+{
+ const char *p;
+ int len;
+ if (!arg || !*arg)
+ return 1;
+
+ p = strchr (arg, (int)'=');
+
+ if (!p)
+ return 1;
+
+ *value = xstrdup (++p);
+
+ len = strlen (arg) - strlen (*value);
+ *name = XMALLOC (char, len);
+ strncpy (*name, arg, len-1);
+ (*name)[len - 1] = '\0';
+
+ return 0;
+}
+
+void
+lt_opt_process_env_set (const char *arg)
+{
+ char *name = NULL;
+ char *value = NULL;
+
+ if (lt_split_name_value (arg, &name, &value) != 0)
+ {
+ XFREE (name);
+ XFREE (value);
+ lt_fatal ("bad argument for %s: '%s'", env_set_opt, arg);
+ }
+
+ lt_setenv (name, value);
+ XFREE (name);
+ XFREE (value);
+}
+
+void
+lt_opt_process_env_prepend (const char *arg)
+{
+ char *name = NULL;
+ char *value = NULL;
+ char *new_value = NULL;
+
+ if (lt_split_name_value (arg, &name, &value) != 0)
+ {
+ XFREE (name);
+ XFREE (value);
+ lt_fatal ("bad argument for %s: '%s'", env_prepend_opt, arg);
+ }
+
+ new_value = lt_extend_str (getenv (name), value, 0);
+ lt_setenv (name, new_value);
+ XFREE (new_value);
+ XFREE (name);
+ XFREE (value);
+}
+
+void
+lt_opt_process_env_append (const char *arg)
+{
+ char *name = NULL;
+ char *value = NULL;
+ char *new_value = NULL;
+
+ if (lt_split_name_value (arg, &name, &value) != 0)
+ {
+ XFREE (name);
+ XFREE (value);
+ lt_fatal ("bad argument for %s: '%s'", env_append_opt, arg);
+ }
+
+ new_value = lt_extend_str (getenv (name), value, 1);
+ lt_setenv (name, new_value);
+ XFREE (new_value);
+ XFREE (name);
+ XFREE (value);
+}
+
+void
+lt_update_exe_path (const char *name, const char *value)
+{
+ LTWRAPPER_DEBUGPRINTF (("(lt_update_exe_path) modifying '%s' by prepending '%s'\n",
+ (name ? name : "<NULL>"),
+ (value ? value : "<NULL>")));
+
+ if (name && *name && value && *value)
+ {
+ char *new_value = lt_extend_str (getenv (name), value, 0);
+ /* some systems can't cope with a ':'-terminated path #' */
+ int len = strlen (new_value);
+ while (((len = strlen (new_value)) > 0) && IS_PATH_SEPARATOR (new_value[len-1]))
+ {
+ new_value[len-1] = '\0';
+ }
+ lt_setenv (name, new_value);
+ XFREE (new_value);
+ }
+}
+
+void
+lt_update_lib_path (const char *name, const char *value)
+{
+ LTWRAPPER_DEBUGPRINTF (("(lt_update_lib_path) modifying '%s' by prepending '%s'\n",
+ (name ? name : "<NULL>"),
+ (value ? value : "<NULL>")));
+
+ if (name && *name && value && *value)
+ {
+ char *new_value = lt_extend_str (getenv (name), value, 0);
+ lt_setenv (name, new_value);
+ XFREE (new_value);
+ }
+}
+
+
+EOF
+}
+# end: func_emit_cwrapperexe_src
+
+# func_mode_link arg...
+func_mode_link ()
+{
+ $opt_debug
+ case $host in
+ *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-cegcc*)
+ # It is impossible to link a dll without this setting, and
+ # we shouldn't force the makefile maintainer to figure out
+ # which system we are compiling for in order to pass an extra
+ # flag for every libtool invocation.
+ # allow_undefined=no
+
+ # FIXME: Unfortunately, there are problems with the above when trying
+ # to make a dll which has undefined symbols, in which case not
+ # even a static library is built. For now, we need to specify
+ # -no-undefined on the libtool link line when we can be certain
+ # that all symbols are satisfied, otherwise we get a static library.
+ allow_undefined=yes
+ ;;
+ *)
+ allow_undefined=yes
+ ;;
+ esac
+ libtool_args=$nonopt
+ base_compile="$nonopt $@"
+ compile_command=$nonopt
+ finalize_command=$nonopt
+
+ compile_rpath=
+ finalize_rpath=
+ compile_shlibpath=
+ finalize_shlibpath=
+ convenience=
+ old_convenience=
+ deplibs=
+ old_deplibs=
+ compiler_flags=
+ linker_flags=
+ dllsearchpath=
+ lib_search_path=`pwd`
+ inst_prefix_dir=
+ new_inherited_linker_flags=
+
+ avoid_version=no
+ dlfiles=
+ dlprefiles=
+ dlself=no
+ export_dynamic=no
+ export_symbols=
+ export_symbols_regex=
+ generated=
+ libobjs=
+ ltlibs=
+ module=no
+ no_install=no
+ objs=
+ non_pic_objects=
+ precious_files_regex=
+ prefer_static_libs=no
+ preload=no
+ prev=
+ prevarg=
+ release=
+ rpath=
+ xrpath=
+ perm_rpath=
+ temp_rpath=
+ thread_safe=no
+ vinfo=
+ vinfo_number=no
+ weak_libs=
+ single_module="${wl}-single_module"
+ func_infer_tag $base_compile
+
+ # We need to know -static, to get the right output filenames.
+ for arg
+ do
+ case $arg in
+ -shared)
+ test "$build_libtool_libs" != yes && \
+ func_fatal_configuration "can not build a shared library"
+ build_old_libs=no
+ break
+ ;;
+ -all-static | -static | -static-libtool-libs)
+ case $arg in
+ -all-static)
+ if test "$build_libtool_libs" = yes && test -z "$link_static_flag"; then
+ func_warning "complete static linking is impossible in this configuration"
+ fi
+ if test -n "$link_static_flag"; then
+ dlopen_self=$dlopen_self_static
+ fi
+ prefer_static_libs=yes
+ ;;
+ -static)
+ if test -z "$pic_flag" && test -n "$link_static_flag"; then
+ dlopen_self=$dlopen_self_static
+ fi
+ prefer_static_libs=built
+ ;;
+ -static-libtool-libs)
+ if test -z "$pic_flag" && test -n "$link_static_flag"; then
+ dlopen_self=$dlopen_self_static
+ fi
+ prefer_static_libs=yes
+ ;;
+ esac
+ build_libtool_libs=no
+ build_old_libs=yes
+ break
+ ;;
+ esac
+ done
+
+ # See if our shared archives depend on static archives.
+ test -n "$old_archive_from_new_cmds" && build_old_libs=yes
+
+ # Go through the arguments, transforming them on the way.
+ while test "$#" -gt 0; do
+ arg="$1"
+ shift
+ func_quote_for_eval "$arg"
+ qarg=$func_quote_for_eval_unquoted_result
+ func_append libtool_args " $func_quote_for_eval_result"
+
+ # If the previous option needs an argument, assign it.
+ if test -n "$prev"; then
+ case $prev in
+ output)
+ func_append compile_command " @OUTPUT@"
+ func_append finalize_command " @OUTPUT@"
+ ;;
+ esac
+
+ case $prev in
+ dlfiles|dlprefiles)
+ if test "$preload" = no; then
+ # Add the symbol object into the linking commands.
+ func_append compile_command " @SYMFILE@"
+ func_append finalize_command " @SYMFILE@"
+ preload=yes
+ fi
+ case $arg in
+ *.la | *.lo) ;; # We handle these cases below.
+ force)
+ if test "$dlself" = no; then
+ dlself=needless
+ export_dynamic=yes
+ fi
+ prev=
+ continue
+ ;;
+ self)
+ if test "$prev" = dlprefiles; then
+ dlself=yes
+ elif test "$prev" = dlfiles && test "$dlopen_self" != yes; then
+ dlself=yes
+ else
+ dlself=needless
+ export_dynamic=yes
+ fi
+ prev=
+ continue
+ ;;
+ *)
+ if test "$prev" = dlfiles; then
+ dlfiles="$dlfiles $arg"
+ else
+ dlprefiles="$dlprefiles $arg"
+ fi
+ prev=
+ continue
+ ;;
+ esac
+ ;;
+ expsyms)
+ export_symbols="$arg"
+ test -f "$arg" \
+ || func_fatal_error "symbol file \`$arg' does not exist"
+ prev=
+ continue
+ ;;
+ expsyms_regex)
+ export_symbols_regex="$arg"
+ prev=
+ continue
+ ;;
+ framework)
+ case $host in
+ *-*-darwin*)
+ case "$deplibs " in
+ *" $qarg.ltframework "*) ;;
+ *) deplibs="$deplibs $qarg.ltframework" # this is fixed later
+ ;;
+ esac
+ ;;
+ esac
+ prev=
+ continue
+ ;;
+ inst_prefix)
+ inst_prefix_dir="$arg"
+ prev=
+ continue
+ ;;
+ objectlist)
+ if test -f "$arg"; then
+ save_arg=$arg
+ moreargs=
+ for fil in `cat "$save_arg"`
+ do
+# moreargs="$moreargs $fil"
+ arg=$fil
+ # A libtool-controlled object.
+
+ # Check to see that this really is a libtool object.
+ if func_lalib_unsafe_p "$arg"; then
+ pic_object=
+ non_pic_object=
+
+ # Read the .lo file
+ func_source "$arg"
+
+ if test -z "$pic_object" ||
+ test -z "$non_pic_object" ||
+ test "$pic_object" = none &&
+ test "$non_pic_object" = none; then
+ func_fatal_error "cannot find name of object for \`$arg'"
+ fi
+
+ # Extract subdirectory from the argument.
+ func_dirname "$arg" "/" ""
+ xdir="$func_dirname_result"
+
+ if test "$pic_object" != none; then
+ # Prepend the subdirectory the object is found in.
+ pic_object="$xdir$pic_object"
+
+ if test "$prev" = dlfiles; then
+ if test "$build_libtool_libs" = yes && test "$dlopen_support" = yes; then
+ dlfiles="$dlfiles $pic_object"
+ prev=
+ continue
+ else
+ # If libtool objects are unsupported, then we need to preload.
+ prev=dlprefiles
+ fi
+ fi
+
+ # CHECK ME: I think I busted this. -Ossama
+ if test "$prev" = dlprefiles; then
+ # Preload the old-style object.
+ dlprefiles="$dlprefiles $pic_object"
+ prev=
+ fi
+
+ # A PIC object.
+ func_append libobjs " $pic_object"
+ arg="$pic_object"
+ fi
+
+ # Non-PIC object.
+ if test "$non_pic_object" != none; then
+ # Prepend the subdirectory the object is found in.
+ non_pic_object="$xdir$non_pic_object"
+
+ # A standard non-PIC object
+ func_append non_pic_objects " $non_pic_object"
+ if test -z "$pic_object" || test "$pic_object" = none ; then
+ arg="$non_pic_object"
+ fi
+ else
+ # If the PIC object exists, use it instead.
+ # $xdir was prepended to $pic_object above.
+ non_pic_object="$pic_object"
+ func_append non_pic_objects " $non_pic_object"
+ fi
+ else
+ # Only an error if not doing a dry-run.
+ if $opt_dry_run; then
+ # Extract subdirectory from the argument.
+ func_dirname "$arg" "/" ""
+ xdir="$func_dirname_result"
+
+ func_lo2o "$arg"
+ pic_object=$xdir$objdir/$func_lo2o_result
+ non_pic_object=$xdir$func_lo2o_result
+ func_append libobjs " $pic_object"
+ func_append non_pic_objects " $non_pic_object"
+ else
+ func_fatal_error "\`$arg' is not a valid libtool object"
+ fi
+ fi
+ done
+ else
+ func_fatal_error "link input file \`$arg' does not exist"
+ fi
+ arg=$save_arg
+ prev=
+ continue
+ ;;
+ precious_regex)
+ precious_files_regex="$arg"
+ prev=
+ continue
+ ;;
+ release)
+ release="-$arg"
+ prev=
+ continue
+ ;;
+ rpath | xrpath)
+ # We need an absolute path.
+ case $arg in
+ [\\/]* | [A-Za-z]:[\\/]*) ;;
+ *)
+ func_fatal_error "only absolute run-paths are allowed"
+ ;;
+ esac
+ if test "$prev" = rpath; then
+ case "$rpath " in
+ *" $arg "*) ;;
+ *) rpath="$rpath $arg" ;;
+ esac
+ else
+ case "$xrpath " in
+ *" $arg "*) ;;
+ *) xrpath="$xrpath $arg" ;;
+ esac
+ fi
+ prev=
+ continue
+ ;;
+ shrext)
+ shrext_cmds="$arg"
+ prev=
+ continue
+ ;;
+ weak)
+ weak_libs="$weak_libs $arg"
+ prev=
+ continue
+ ;;
+ xcclinker)
+ linker_flags="$linker_flags $qarg"
+ compiler_flags="$compiler_flags $qarg"
+ prev=
+ func_append compile_command " $qarg"
+ func_append finalize_command " $qarg"
+ continue
+ ;;
+ xcompiler)
+ compiler_flags="$compiler_flags $qarg"
+ prev=
+ func_append compile_command " $qarg"
+ func_append finalize_command " $qarg"
+ continue
+ ;;
+ xlinker)
+ linker_flags="$linker_flags $qarg"
+ compiler_flags="$compiler_flags $wl$qarg"
+ prev=
+ func_append compile_command " $wl$qarg"
+ func_append finalize_command " $wl$qarg"
+ continue
+ ;;
+ *)
+ eval "$prev=\"\$arg\""
+ prev=
+ continue
+ ;;
+ esac
+ fi # test -n "$prev"
+
+ prevarg="$arg"
+
+ case $arg in
+ -all-static)
+ if test -n "$link_static_flag"; then
+ # See comment for -static flag below, for more details.
+ func_append compile_command " $link_static_flag"
+ func_append finalize_command " $link_static_flag"
+ fi
+ continue
+ ;;
+
+ -allow-undefined)
+ # FIXME: remove this flag sometime in the future.
+ func_fatal_error "\`-allow-undefined' must not be used because it is the default"
+ ;;
+
+ -avoid-version)
+ avoid_version=yes
+ continue
+ ;;
+
+ -dlopen)
+ prev=dlfiles
+ continue
+ ;;
+
+ -dlpreopen)
+ prev=dlprefiles
+ continue
+ ;;
+
+ -export-dynamic)
+ export_dynamic=yes
+ continue
+ ;;
+
+ -export-symbols | -export-symbols-regex)
+ if test -n "$export_symbols" || test -n "$export_symbols_regex"; then
+ func_fatal_error "more than one -exported-symbols argument is not allowed"
+ fi
+ if test "X$arg" = "X-export-symbols"; then
+ prev=expsyms
+ else
+ prev=expsyms_regex
+ fi
+ continue
+ ;;
+
+ -framework)
+ prev=framework
+ continue
+ ;;
+
+ -inst-prefix-dir)
+ prev=inst_prefix
+ continue
+ ;;
+
+ # The native IRIX linker understands -LANG:*, -LIST:* and -LNO:*
+ # so, if we see these flags be careful not to treat them like -L
+ -L[A-Z][A-Z]*:*)
+ case $with_gcc/$host in
+ no/*-*-irix* | /*-*-irix*)
+ func_append compile_command " $arg"
+ func_append finalize_command " $arg"
+ ;;
+ esac
+ continue
+ ;;
+
+ -L*)
+ func_stripname '-L' '' "$arg"
+ dir=$func_stripname_result
+ if test -z "$dir"; then
+ if test "$#" -gt 0; then
+ func_fatal_error "require no space between \`-L' and \`$1'"
+ else
+ func_fatal_error "need path for \`-L' option"
+ fi
+ fi
+ # We need an absolute path.
+ case $dir in
+ [\\/]* | [A-Za-z]:[\\/]*) ;;
+ *)
+ absdir=`cd "$dir" && pwd`
+ test -z "$absdir" && \
+ func_fatal_error "cannot determine absolute directory name of \`$dir'"
+ dir="$absdir"
+ ;;
+ esac
+ case "$deplibs " in
+ *" -L$dir "*) ;;
+ *)
+ deplibs="$deplibs -L$dir"
+ lib_search_path="$lib_search_path $dir"
+ ;;
+ esac
+ case $host in
+ *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-cegcc*)
+ testbindir=`$ECHO "X$dir" | $Xsed -e 's*/lib$*/bin*'`
+ case :$dllsearchpath: in
+ *":$dir:"*) ;;
+ ::) dllsearchpath=$dir;;
+ *) dllsearchpath="$dllsearchpath:$dir";;
+ esac
+ case :$dllsearchpath: in
+ *":$testbindir:"*) ;;
+ ::) dllsearchpath=$testbindir;;
+ *) dllsearchpath="$dllsearchpath:$testbindir";;
+ esac
+ ;;
+ esac
+ continue
+ ;;
+
+ -l*)
+ if test "X$arg" = "X-lc" || test "X$arg" = "X-lm"; then
+ case $host in
+ *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-beos* | *-cegcc*)
+ # These systems don't actually have a C or math library (as such)
+ continue
+ ;;
+ *-*-os2*)
+ # These systems don't actually have a C library (as such)
+ test "X$arg" = "X-lc" && continue
+ ;;
+ *-*-openbsd* | *-*-freebsd* | *-*-dragonfly*)
+ # Do not include libc due to us having libc/libc_r.
+ test "X$arg" = "X-lc" && continue
+ ;;
+ *-*-rhapsody* | *-*-darwin1.[012])
+ # Rhapsody C and math libraries are in the System framework
+ deplibs="$deplibs System.ltframework"
+ continue
+ ;;
+ *-*-sco3.2v5* | *-*-sco5v6*)
+ # Causes problems with __ctype
+ test "X$arg" = "X-lc" && continue
+ ;;
+ *-*-sysv4.2uw2* | *-*-sysv5* | *-*-unixware* | *-*-OpenUNIX*)
+ # Compiler inserts libc in the correct place for threads to work
+ test "X$arg" = "X-lc" && continue
+ ;;
+ esac
+ elif test "X$arg" = "X-lc_r"; then
+ case $host in
+ *-*-openbsd* | *-*-freebsd* | *-*-dragonfly*)
+ # Do not include libc_r directly, use -pthread flag.
+ continue
+ ;;
+ esac
+ fi
+ deplibs="$deplibs $arg"
+ continue
+ ;;
+
+ -module)
+ module=yes
+ continue
+ ;;
+
+ # Tru64 UNIX uses -model [arg] to determine the layout of C++
+ # classes, name mangling, and exception handling.
+ # Darwin uses the -arch flag to determine output architecture.
+ -model|-arch|-isysroot)
+ compiler_flags="$compiler_flags $arg"
+ func_append compile_command " $arg"
+ func_append finalize_command " $arg"
+ prev=xcompiler
+ continue
+ ;;
+
+ -mt|-mthreads|-kthread|-Kthread|-pthread|-pthreads|--thread-safe|-threads)
+ compiler_flags="$compiler_flags $arg"
+ func_append compile_command " $arg"
+ func_append finalize_command " $arg"
+ case "$new_inherited_linker_flags " in
+ *" $arg "*) ;;
+ * ) new_inherited_linker_flags="$new_inherited_linker_flags $arg" ;;
+ esac
+ continue
+ ;;
+
+ -multi_module)
+ single_module="${wl}-multi_module"
+ continue
+ ;;
+
+ -no-fast-install)
+ fast_install=no
+ continue
+ ;;
+
+ -no-install)
+ case $host in
+ *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-*-darwin* | *-cegcc*)
+ # The PATH hackery in wrapper scripts is required on Windows
+ # and Darwin in order for the loader to find any dlls it needs.
+ func_warning "\`-no-install' is ignored for $host"
+ func_warning "assuming \`-no-fast-install' instead"
+ fast_install=no
+ ;;
+ *) no_install=yes ;;
+ esac
+ continue
+ ;;
+
+ -no-undefined)
+ allow_undefined=no
+ continue
+ ;;
+
+ -objectlist)
+ prev=objectlist
+ continue
+ ;;
+
+ -o) prev=output ;;
+
+ -precious-files-regex)
+ prev=precious_regex
+ continue
+ ;;
+
+ -release)
+ prev=release
+ continue
+ ;;
+
+ -rpath)
+ prev=rpath
+ continue
+ ;;
+
+ -R)
+ prev=xrpath
+ continue
+ ;;
+
+ -R*)
+ func_stripname '-R' '' "$arg"
+ dir=$func_stripname_result
+ # We need an absolute path.
+ case $dir in
+ [\\/]* | [A-Za-z]:[\\/]*) ;;
+ *)
+ func_fatal_error "only absolute run-paths are allowed"
+ ;;
+ esac
+ case "$xrpath " in
+ *" $dir "*) ;;
+ *) xrpath="$xrpath $dir" ;;
+ esac
+ continue
+ ;;
+
+ -shared)
+ # The effects of -shared are defined in a previous loop.
+ continue
+ ;;
+
+ -shrext)
+ prev=shrext
+ continue
+ ;;
+
+ -static | -static-libtool-libs)
+ # The effects of -static are defined in a previous loop.
+ # We used to do the same as -all-static on platforms that
+ # didn't have a PIC flag, but the assumption that the effects
+ # would be equivalent was wrong. It would break on at least
+ # Digital Unix and AIX.
+ continue
+ ;;
+
+ -thread-safe)
+ thread_safe=yes
+ continue
+ ;;
+
+ -version-info)
+ prev=vinfo
+ continue
+ ;;
+
+ -version-number)
+ prev=vinfo
+ vinfo_number=yes
+ continue
+ ;;
+
+ -weak)
+ prev=weak
+ continue
+ ;;
+
+ -Wc,*)
+ func_stripname '-Wc,' '' "$arg"
+ args=$func_stripname_result
+ arg=
+ save_ifs="$IFS"; IFS=','
+ for flag in $args; do
+ IFS="$save_ifs"
+ func_quote_for_eval "$flag"
+ arg="$arg $wl$func_quote_for_eval_result"
+ compiler_flags="$compiler_flags $func_quote_for_eval_result"
+ done
+ IFS="$save_ifs"
+ func_stripname ' ' '' "$arg"
+ arg=$func_stripname_result
+ ;;
+
+ -Wl,*)
+ func_stripname '-Wl,' '' "$arg"
+ args=$func_stripname_result
+ arg=
+ save_ifs="$IFS"; IFS=','
+ for flag in $args; do
+ IFS="$save_ifs"
+ func_quote_for_eval "$flag"
+ arg="$arg $wl$func_quote_for_eval_result"
+ compiler_flags="$compiler_flags $wl$func_quote_for_eval_result"
+ linker_flags="$linker_flags $func_quote_for_eval_result"
+ done
+ IFS="$save_ifs"
+ func_stripname ' ' '' "$arg"
+ arg=$func_stripname_result
+ ;;
+
+ -Xcompiler)
+ prev=xcompiler
+ continue
+ ;;
+
+ -Xlinker)
+ prev=xlinker
+ continue
+ ;;
+
+ -XCClinker)
+ prev=xcclinker
+ continue
+ ;;
+
+ # -msg_* for osf cc
+ -msg_*)
+ func_quote_for_eval "$arg"
+ arg="$func_quote_for_eval_result"
+ ;;
+
+ # -64, -mips[0-9] enable 64-bit mode on the SGI compiler
+ # -r[0-9][0-9]* specifies the processor on the SGI compiler
+ # -xarch=*, -xtarget=* enable 64-bit mode on the Sun compiler
+ # +DA*, +DD* enable 64-bit mode on the HP compiler
+ # -q* pass through compiler args for the IBM compiler
+ # -m*, -t[45]*, -txscale* pass through architecture-specific
+ # compiler args for GCC
+ # -F/path gives path to uninstalled frameworks, gcc on darwin
+ # -p, -pg, --coverage, -fprofile-* pass through profiling flag for GCC
+ # @file GCC response files
+ -64|-mips[0-9]|-r[0-9][0-9]*|-xarch=*|-xtarget=*|+DA*|+DD*|-q*|-m*| \
+ -t[45]*|-txscale*|-p|-pg|--coverage|-fprofile-*|-F*|@*)
+ func_quote_for_eval "$arg"
+ arg="$func_quote_for_eval_result"
+ func_append compile_command " $arg"
+ func_append finalize_command " $arg"
+ compiler_flags="$compiler_flags $arg"
+ continue
+ ;;
+
+ # Some other compiler flag.
+ -* | +*)
+ func_quote_for_eval "$arg"
+ arg="$func_quote_for_eval_result"
+ ;;
+
+ *.$objext)
+ # A standard object.
+ objs="$objs $arg"
+ ;;
+
+ *.lo)
+ # A libtool-controlled object.
+
+ # Check to see that this really is a libtool object.
+ if func_lalib_unsafe_p "$arg"; then
+ pic_object=
+ non_pic_object=
+
+ # Read the .lo file
+ func_source "$arg"
+
+ if test -z "$pic_object" ||
+ test -z "$non_pic_object" ||
+ test "$pic_object" = none &&
+ test "$non_pic_object" = none; then
+ func_fatal_error "cannot find name of object for \`$arg'"
+ fi
+
+ # Extract subdirectory from the argument.
+ func_dirname "$arg" "/" ""
+ xdir="$func_dirname_result"
+
+ if test "$pic_object" != none; then
+ # Prepend the subdirectory the object is found in.
+ pic_object="$xdir$pic_object"
+
+ if test "$prev" = dlfiles; then
+ if test "$build_libtool_libs" = yes && test "$dlopen_support" = yes; then
+ dlfiles="$dlfiles $pic_object"
+ prev=
+ continue
+ else
+ # If libtool objects are unsupported, then we need to preload.
+ prev=dlprefiles
+ fi
+ fi
+
+ # CHECK ME: I think I busted this. -Ossama
+ if test "$prev" = dlprefiles; then
+ # Preload the old-style object.
+ dlprefiles="$dlprefiles $pic_object"
+ prev=
+ fi
+
+ # A PIC object.
+ func_append libobjs " $pic_object"
+ arg="$pic_object"
+ fi
+
+ # Non-PIC object.
+ if test "$non_pic_object" != none; then
+ # Prepend the subdirectory the object is found in.
+ non_pic_object="$xdir$non_pic_object"
+
+ # A standard non-PIC object
+ func_append non_pic_objects " $non_pic_object"
+ if test -z "$pic_object" || test "$pic_object" = none ; then
+ arg="$non_pic_object"
+ fi
+ else
+ # If the PIC object exists, use it instead.
+ # $xdir was prepended to $pic_object above.
+ non_pic_object="$pic_object"
+ func_append non_pic_objects " $non_pic_object"
+ fi
+ else
+ # Only an error if not doing a dry-run.
+ if $opt_dry_run; then
+ # Extract subdirectory from the argument.
+ func_dirname "$arg" "/" ""
+ xdir="$func_dirname_result"
+
+ func_lo2o "$arg"
+ pic_object=$xdir$objdir/$func_lo2o_result
+ non_pic_object=$xdir$func_lo2o_result
+ func_append libobjs " $pic_object"
+ func_append non_pic_objects " $non_pic_object"
+ else
+ func_fatal_error "\`$arg' is not a valid libtool object"
+ fi
+ fi
+ ;;
+
+ *.$libext)
+ # An archive.
+ deplibs="$deplibs $arg"
+ old_deplibs="$old_deplibs $arg"
+ continue
+ ;;
+
+ *.la)
+ # A libtool-controlled library.
+
+ if test "$prev" = dlfiles; then
+ # This library was specified with -dlopen.
+ dlfiles="$dlfiles $arg"
+ prev=
+ elif test "$prev" = dlprefiles; then
+ # The library was specified with -dlpreopen.
+ dlprefiles="$dlprefiles $arg"
+ prev=
+ else
+ deplibs="$deplibs $arg"
+ fi
+ continue
+ ;;
+
+ # Some other compiler argument.
+ *)
+ # Unknown arguments in both finalize_command and compile_command need
+ # to be aesthetically quoted because they are evaled later.
+ func_quote_for_eval "$arg"
+ arg="$func_quote_for_eval_result"
+ ;;
+ esac # arg
+
+ # Now actually substitute the argument into the commands.
+ if test -n "$arg"; then
+ func_append compile_command " $arg"
+ func_append finalize_command " $arg"
+ fi
+ done # argument parsing loop
+
+ test -n "$prev" && \
+ func_fatal_help "the \`$prevarg' option requires an argument"
+
+ if test "$export_dynamic" = yes && test -n "$export_dynamic_flag_spec"; then
+ eval arg=\"$export_dynamic_flag_spec\"
+ func_append compile_command " $arg"
+ func_append finalize_command " $arg"
+ fi
+
+ oldlibs=
+ # calculate the name of the file, without its directory
+ func_basename "$output"
+ outputname="$func_basename_result"
+ libobjs_save="$libobjs"
+
+ if test -n "$shlibpath_var"; then
+ # get the directories listed in $shlibpath_var
+ eval shlib_search_path=\`\$ECHO \"X\${$shlibpath_var}\" \| \$Xsed -e \'s/:/ /g\'\`
+ else
+ shlib_search_path=
+ fi
+ eval sys_lib_search_path=\"$sys_lib_search_path_spec\"
+ eval sys_lib_dlsearch_path=\"$sys_lib_dlsearch_path_spec\"
+
+ func_dirname "$output" "/" ""
+ output_objdir="$func_dirname_result$objdir"
+ # Create the object directory.
+ func_mkdir_p "$output_objdir"
+
+ # Determine the type of output
+ case $output in
+ "")
+ func_fatal_help "you must specify an output file"
+ ;;
+ *.$libext) linkmode=oldlib ;;
+ *.lo | *.$objext) linkmode=obj ;;
+ *.la) linkmode=lib ;;
+ *) linkmode=prog ;; # Anything else should be a program.
+ esac
+
+ specialdeplibs=
+
+ libs=
+ # Find all interdependent deplibs by searching for libraries
+ # that are linked more than once (e.g. -la -lb -la)
+ for deplib in $deplibs; do
+ if $opt_duplicate_deps ; then
+ case "$libs " in
+ *" $deplib "*) specialdeplibs="$specialdeplibs $deplib" ;;
+ esac
+ fi
+ libs="$libs $deplib"
+ done
+
+ if test "$linkmode" = lib; then
+ libs="$predeps $libs $compiler_lib_search_path $postdeps"
+
+ # Compute libraries that are listed more than once in $predeps
+ # $postdeps and mark them as special (i.e., whose duplicates are
+ # not to be eliminated).
+ pre_post_deps=
+ if $opt_duplicate_compiler_generated_deps; then
+ for pre_post_dep in $predeps $postdeps; do
+ case "$pre_post_deps " in
+ *" $pre_post_dep "*) specialdeplibs="$specialdeplibs $pre_post_deps" ;;
+ esac
+ pre_post_deps="$pre_post_deps $pre_post_dep"
+ done
+ fi
+ pre_post_deps=
+ fi
+
+ deplibs=
+ newdependency_libs=
+ newlib_search_path=
+ need_relink=no # whether we're linking any uninstalled libtool libraries
+ notinst_deplibs= # not-installed libtool libraries
+ notinst_path= # paths that contain not-installed libtool libraries
+
+ case $linkmode in
+ lib)
+ passes="conv dlpreopen link"
+ for file in $dlfiles $dlprefiles; do
+ case $file in
+ *.la) ;;
+ *)
+ func_fatal_help "libraries can \`-dlopen' only libtool libraries: $file"
+ ;;
+ esac
+ done
+ ;;
+ prog)
+ compile_deplibs=
+ finalize_deplibs=
+ alldeplibs=no
+ newdlfiles=
+ newdlprefiles=
+ passes="conv scan dlopen dlpreopen link"
+ ;;
+ *) passes="conv"
+ ;;
+ esac
+
+ for pass in $passes; do
+ # The preopen pass in lib mode reverses $deplibs; put it back here
+ # so that -L comes before libs that need it for instance...
+ if test "$linkmode,$pass" = "lib,link"; then
+ ## FIXME: Find the place where the list is rebuilt in the wrong
+ ## order, and fix it there properly
+ tmp_deplibs=
+ for deplib in $deplibs; do
+ tmp_deplibs="$deplib $tmp_deplibs"
+ done
+ deplibs="$tmp_deplibs"
+ fi
+
+ if test "$linkmode,$pass" = "lib,link" ||
+ test "$linkmode,$pass" = "prog,scan"; then
+ libs="$deplibs"
+ deplibs=
+ fi
+ if test "$linkmode" = prog; then
+ case $pass in
+ dlopen) libs="$dlfiles" ;;
+ dlpreopen) libs="$dlprefiles" ;;
+ link) libs="$deplibs %DEPLIBS% $dependency_libs" ;;
+ esac
+ fi
+ if test "$linkmode,$pass" = "lib,dlpreopen"; then
+ # Collect and forward deplibs of preopened libtool libs
+ for lib in $dlprefiles; do
+ # Ignore non-libtool-libs
+ dependency_libs=
+ case $lib in
+ *.la) func_source "$lib" ;;
+ esac
+
+ # Collect preopened libtool deplibs, except any this library
+ # has declared as weak libs
+ for deplib in $dependency_libs; do
+ deplib_base=`$ECHO "X$deplib" | $Xsed -e "$basename"`
+ case " $weak_libs " in
+ *" $deplib_base "*) ;;
+ *) deplibs="$deplibs $deplib" ;;
+ esac
+ done
+ done
+ libs="$dlprefiles"
+ fi
+ if test "$pass" = dlopen; then
+ # Collect dlpreopened libraries
+ save_deplibs="$deplibs"
+ deplibs=
+ fi
+
+ for deplib in $libs; do
+ lib=
+ found=no
+ case $deplib in
+ -mt|-mthreads|-kthread|-Kthread|-pthread|-pthreads|--thread-safe|-threads)
+ if test "$linkmode,$pass" = "prog,link"; then
+ compile_deplibs="$deplib $compile_deplibs"
+ finalize_deplibs="$deplib $finalize_deplibs"
+ else
+ compiler_flags="$compiler_flags $deplib"
+ if test "$linkmode" = lib ; then
+ case "$new_inherited_linker_flags " in
+ *" $deplib "*) ;;
+ * ) new_inherited_linker_flags="$new_inherited_linker_flags $deplib" ;;
+ esac
+ fi
+ fi
+ continue
+ ;;
+ -l*)
+ if test "$linkmode" != lib && test "$linkmode" != prog; then
+ func_warning "\`-l' is ignored for archives/objects"
+ continue
+ fi
+ func_stripname '-l' '' "$deplib"
+ name=$func_stripname_result
+ if test "$linkmode" = lib; then
+ searchdirs="$newlib_search_path $lib_search_path $compiler_lib_search_dirs $sys_lib_search_path $shlib_search_path"
+ else
+ searchdirs="$newlib_search_path $lib_search_path $sys_lib_search_path $shlib_search_path"
+ fi
+ for searchdir in $searchdirs; do
+ for search_ext in .la $std_shrext .so .a; do
+ # Search the libtool library
+ lib="$searchdir/lib${name}${search_ext}"
+ if test -f "$lib"; then
+ if test "$search_ext" = ".la"; then
+ found=yes
+ else
+ found=no
+ fi
+ break 2
+ fi
+ done
+ done
+ if test "$found" != yes; then
+ # deplib doesn't seem to be a libtool library
+ if test "$linkmode,$pass" = "prog,link"; then
+ compile_deplibs="$deplib $compile_deplibs"
+ finalize_deplibs="$deplib $finalize_deplibs"
+ else
+ deplibs="$deplib $deplibs"
+ test "$linkmode" = lib && newdependency_libs="$deplib $newdependency_libs"
+ fi
+ continue
+ else # deplib is a libtool library
+ # If $allow_libtool_libs_with_static_runtimes && $deplib is a stdlib,
+ # We need to do some special things here, and not later.
+ if test "X$allow_libtool_libs_with_static_runtimes" = "Xyes" ; then
+ case " $predeps $postdeps " in
+ *" $deplib "*)
+ if func_lalib_p "$lib"; then
+ library_names=
+ old_library=
+ func_source "$lib"
+ for l in $old_library $library_names; do
+ ll="$l"
+ done
+ if test "X$ll" = "X$old_library" ; then # only static version available
+ found=no
+ func_dirname "$lib" "" "."
+ ladir="$func_dirname_result"
+ lib=$ladir/$old_library
+ if test "$linkmode,$pass" = "prog,link"; then
+ compile_deplibs="$deplib $compile_deplibs"
+ finalize_deplibs="$deplib $finalize_deplibs"
+ else
+ deplibs="$deplib $deplibs"
+ test "$linkmode" = lib && newdependency_libs="$deplib $newdependency_libs"
+ fi
+ continue
+ fi
+ fi
+ ;;
+ *) ;;
+ esac
+ fi
+ fi
+ ;; # -l
+ *.ltframework)
+ if test "$linkmode,$pass" = "prog,link"; then
+ compile_deplibs="$deplib $compile_deplibs"
+ finalize_deplibs="$deplib $finalize_deplibs"
+ else
+ deplibs="$deplib $deplibs"
+ if test "$linkmode" = lib ; then
+ case "$new_inherited_linker_flags " in
+ *" $deplib "*) ;;
+ * ) new_inherited_linker_flags="$new_inherited_linker_flags $deplib" ;;
+ esac
+ fi
+ fi
+ continue
+ ;;
+ -L*)
+ case $linkmode in
+ lib)
+ deplibs="$deplib $deplibs"
+ test "$pass" = conv && continue
+ newdependency_libs="$deplib $newdependency_libs"
+ func_stripname '-L' '' "$deplib"
+ newlib_search_path="$newlib_search_path $func_stripname_result"
+ ;;
+ prog)
+ if test "$pass" = conv; then
+ deplibs="$deplib $deplibs"
+ continue
+ fi
+ if test "$pass" = scan; then
+ deplibs="$deplib $deplibs"
+ else
+ compile_deplibs="$deplib $compile_deplibs"
+ finalize_deplibs="$deplib $finalize_deplibs"
+ fi
+ func_stripname '-L' '' "$deplib"
+ newlib_search_path="$newlib_search_path $func_stripname_result"
+ ;;
+ *)
+ func_warning "\`-L' is ignored for archives/objects"
+ ;;
+ esac # linkmode
+ continue
+ ;; # -L
+ -R*)
+ if test "$pass" = link; then
+ func_stripname '-R' '' "$deplib"
+ dir=$func_stripname_result
+ # Make sure the xrpath contains only unique directories.
+ case "$xrpath " in
+ *" $dir "*) ;;
+ *) xrpath="$xrpath $dir" ;;
+ esac
+ fi
+ deplibs="$deplib $deplibs"
+ continue
+ ;;
+ *.la) lib="$deplib" ;;
+ *.$libext)
+ if test "$pass" = conv; then
+ deplibs="$deplib $deplibs"
+ continue
+ fi
+ case $linkmode in
+ lib)
+ # Linking convenience modules into shared libraries is allowed,
+ # but linking other static libraries is non-portable.
+ case " $dlpreconveniencelibs " in
+ *" $deplib "*) ;;
+ *)
+ valid_a_lib=no
+ case $deplibs_check_method in
+ match_pattern*)
+ set dummy $deplibs_check_method; shift
+ match_pattern_regex=`expr "$deplibs_check_method" : "$1 \(.*\)"`
+ if eval "\$ECHO \"X$deplib\"" 2>/dev/null | $Xsed -e 10q \
+ | $EGREP "$match_pattern_regex" > /dev/null; then
+ valid_a_lib=yes
+ fi
+ ;;
+ pass_all)
+ valid_a_lib=yes
+ ;;
+ esac
+ if test "$valid_a_lib" != yes; then
+ $ECHO
+ $ECHO "*** Warning: Trying to link with static lib archive $deplib."
+ $ECHO "*** I have the capability to make that library automatically link in when"
+ $ECHO "*** you link to this library. But I can only do this if you have a"
+ $ECHO "*** shared version of the library, which you do not appear to have"
+ $ECHO "*** because the file extensions .$libext of this argument makes me believe"
+ $ECHO "*** that it is just a static archive that I should not use here."
+ else
+ $ECHO
+ $ECHO "*** Warning: Linking the shared library $output against the"
+ $ECHO "*** static library $deplib is not portable!"
+ deplibs="$deplib $deplibs"
+ fi
+ ;;
+ esac
+ continue
+ ;;
+ prog)
+ if test "$pass" != link; then
+ deplibs="$deplib $deplibs"
+ else
+ compile_deplibs="$deplib $compile_deplibs"
+ finalize_deplibs="$deplib $finalize_deplibs"
+ fi
+ continue
+ ;;
+ esac # linkmode
+ ;; # *.$libext
+ *.lo | *.$objext)
+ if test "$pass" = conv; then
+ deplibs="$deplib $deplibs"
+ elif test "$linkmode" = prog; then
+ if test "$pass" = dlpreopen || test "$dlopen_support" != yes || test "$build_libtool_libs" = no; then
+ # If there is no dlopen support or we're linking statically,
+ # we need to preload.
+ newdlprefiles="$newdlprefiles $deplib"
+ compile_deplibs="$deplib $compile_deplibs"
+ finalize_deplibs="$deplib $finalize_deplibs"
+ else
+ newdlfiles="$newdlfiles $deplib"
+ fi
+ fi
+ continue
+ ;;
+ %DEPLIBS%)
+ alldeplibs=yes
+ continue
+ ;;
+ esac # case $deplib
+
+ if test "$found" = yes || test -f "$lib"; then :
+ else
+ func_fatal_error "cannot find the library \`$lib' or unhandled argument \`$deplib'"
+ fi
+
+ # Check to see that this really is a libtool archive.
+ func_lalib_unsafe_p "$lib" \
+ || func_fatal_error "\`$lib' is not a valid libtool archive"
+
+ func_dirname "$lib" "" "."
+ ladir="$func_dirname_result"
+
+ dlname=
+ dlopen=
+ dlpreopen=
+ libdir=
+ library_names=
+ old_library=
+ inherited_linker_flags=
+ # If the library was installed with an old release of libtool,
+ # it will not redefine variables installed, or shouldnotlink
+ installed=yes
+ shouldnotlink=no
+ avoidtemprpath=
+
+
+ # Read the .la file
+ func_source "$lib"
+
+ # Convert "-framework foo" to "foo.ltframework"
+ if test -n "$inherited_linker_flags"; then
+ tmp_inherited_linker_flags=`$ECHO "X$inherited_linker_flags" | $Xsed -e 's/-framework \([^ $]*\)/\1.ltframework/g'`
+ for tmp_inherited_linker_flag in $tmp_inherited_linker_flags; do
+ case " $new_inherited_linker_flags " in
+ *" $tmp_inherited_linker_flag "*) ;;
+ *) new_inherited_linker_flags="$new_inherited_linker_flags $tmp_inherited_linker_flag";;
+ esac
+ done
+ fi
+ dependency_libs=`$ECHO "X $dependency_libs" | $Xsed -e 's% \([^ $]*\).ltframework% -framework \1%g'`
+ if test "$linkmode,$pass" = "lib,link" ||
+ test "$linkmode,$pass" = "prog,scan" ||
+ { test "$linkmode" != prog && test "$linkmode" != lib; }; then
+ test -n "$dlopen" && dlfiles="$dlfiles $dlopen"
+ test -n "$dlpreopen" && dlprefiles="$dlprefiles $dlpreopen"
+ fi
+
+ if test "$pass" = conv; then
+ # Only check for convenience libraries
+ deplibs="$lib $deplibs"
+ if test -z "$libdir"; then
+ if test -z "$old_library"; then
+ func_fatal_error "cannot find name of link library for \`$lib'"
+ fi
+ # It is a libtool convenience library, so add in its objects.
+ convenience="$convenience $ladir/$objdir/$old_library"
+ old_convenience="$old_convenience $ladir/$objdir/$old_library"
+ elif test "$linkmode" != prog && test "$linkmode" != lib; then
+ func_fatal_error "\`$lib' is not a convenience library"
+ fi
+ tmp_libs=
+ for deplib in $dependency_libs; do
+ deplibs="$deplib $deplibs"
+ if $opt_duplicate_deps ; then
+ case "$tmp_libs " in
+ *" $deplib "*) specialdeplibs="$specialdeplibs $deplib" ;;
+ esac
+ fi
+ tmp_libs="$tmp_libs $deplib"
+ done
+ continue
+ fi # $pass = conv
+
+
+ # Get the name of the library we link against.
+ linklib=
+ for l in $old_library $library_names; do
+ linklib="$l"
+ done
+ if test -z "$linklib"; then
+ func_fatal_error "cannot find name of link library for \`$lib'"
+ fi
+
+ # This library was specified with -dlopen.
+ if test "$pass" = dlopen; then
+ if test -z "$libdir"; then
+ func_fatal_error "cannot -dlopen a convenience library: \`$lib'"
+ fi
+ if test -z "$dlname" ||
+ test "$dlopen_support" != yes ||
+ test "$build_libtool_libs" = no; then
+ # If there is no dlname, no dlopen support or we're linking
+ # statically, we need to preload. We also need to preload any
+ # dependent libraries so libltdl's deplib preloader doesn't
+ # bomb out in the load deplibs phase.
+ dlprefiles="$dlprefiles $lib $dependency_libs"
+ else
+ newdlfiles="$newdlfiles $lib"
+ fi
+ continue
+ fi # $pass = dlopen
+
+ # We need an absolute path.
+ case $ladir in
+ [\\/]* | [A-Za-z]:[\\/]*) abs_ladir="$ladir" ;;
+ *)
+ abs_ladir=`cd "$ladir" && pwd`
+ if test -z "$abs_ladir"; then
+ func_warning "cannot determine absolute directory name of \`$ladir'"
+ func_warning "passing it literally to the linker, although it might fail"
+ abs_ladir="$ladir"
+ fi
+ ;;
+ esac
+ func_basename "$lib"
+ laname="$func_basename_result"
+
+ # Find the relevant object directory and library name.
+ if test "X$installed" = Xyes; then
+ if test ! -f "$libdir/$linklib" && test -f "$abs_ladir/$linklib"; then
+ func_warning "library \`$lib' was moved."
+ dir="$ladir"
+ absdir="$abs_ladir"
+ libdir="$abs_ladir"
+ else
+ dir="$libdir"
+ absdir="$libdir"
+ fi
+ test "X$hardcode_automatic" = Xyes && avoidtemprpath=yes
+ else
+ if test ! -f "$ladir/$objdir/$linklib" && test -f "$abs_ladir/$linklib"; then
+ dir="$ladir"
+ absdir="$abs_ladir"
+ # Remove this search path later
+ notinst_path="$notinst_path $abs_ladir"
+ else
+ dir="$ladir/$objdir"
+ absdir="$abs_ladir/$objdir"
+ # Remove this search path later
+ notinst_path="$notinst_path $abs_ladir"
+ fi
+ fi # $installed = yes
+ func_stripname 'lib' '.la' "$laname"
+ name=$func_stripname_result
+
+ # This library was specified with -dlpreopen.
+ if test "$pass" = dlpreopen; then
+ if test -z "$libdir" && test "$linkmode" = prog; then
+ func_fatal_error "only libraries may -dlpreopen a convenience library: \`$lib'"
+ fi
+ # Prefer using a static library (so that no silly _DYNAMIC symbols
+ # are required to link).
+ if test -n "$old_library"; then
+ newdlprefiles="$newdlprefiles $dir/$old_library"
+ # Keep a list of preopened convenience libraries to check
+ # that they are being used correctly in the link pass.
+ test -z "$libdir" && \
+ dlpreconveniencelibs="$dlpreconveniencelibs $dir/$old_library"
+ # Otherwise, use the dlname, so that lt_dlopen finds it.
+ elif test -n "$dlname"; then
+ newdlprefiles="$newdlprefiles $dir/$dlname"
+ else
+ newdlprefiles="$newdlprefiles $dir/$linklib"
+ fi
+ fi # $pass = dlpreopen
+
+ if test -z "$libdir"; then
+ # Link the convenience library
+ if test "$linkmode" = lib; then
+ deplibs="$dir/$old_library $deplibs"
+ elif test "$linkmode,$pass" = "prog,link"; then
+ compile_deplibs="$dir/$old_library $compile_deplibs"
+ finalize_deplibs="$dir/$old_library $finalize_deplibs"
+ else
+ deplibs="$lib $deplibs" # used for prog,scan pass
+ fi
+ continue
+ fi
+
+
+ if test "$linkmode" = prog && test "$pass" != link; then
+ newlib_search_path="$newlib_search_path $ladir"
+ deplibs="$lib $deplibs"
+
+ linkalldeplibs=no
+ if test "$link_all_deplibs" != no || test -z "$library_names" ||
+ test "$build_libtool_libs" = no; then
+ linkalldeplibs=yes
+ fi
+
+ tmp_libs=
+ for deplib in $dependency_libs; do
+ case $deplib in
+ -L*) func_stripname '-L' '' "$deplib"
+ newlib_search_path="$newlib_search_path $func_stripname_result"
+ ;;
+ esac
+ # Need to link against all dependency_libs?
+ if test "$linkalldeplibs" = yes; then
+ deplibs="$deplib $deplibs"
+ else
+ # Need to hardcode shared library paths
+ # or/and link against static libraries
+ newdependency_libs="$deplib $newdependency_libs"
+ fi
+ if $opt_duplicate_deps ; then
+ case "$tmp_libs " in
+ *" $deplib "*) specialdeplibs="$specialdeplibs $deplib" ;;
+ esac
+ fi
+ tmp_libs="$tmp_libs $deplib"
+ done # for deplib
+ continue
+ fi # $linkmode = prog...
+
+ if test "$linkmode,$pass" = "prog,link"; then
+ if test -n "$library_names" &&
+ { { test "$prefer_static_libs" = no ||
+ test "$prefer_static_libs,$installed" = "built,yes"; } ||
+ test -z "$old_library"; }; then
+ # We need to hardcode the library path
+ if test -n "$shlibpath_var" && test -z "$avoidtemprpath" ; then
+ # Make sure the rpath contains only unique directories.
+ case "$temp_rpath:" in
+ *"$absdir:"*) ;;
+ *) temp_rpath="$temp_rpath$absdir:" ;;
+ esac
+ fi
+
+ # Hardcode the library path.
+ # Skip directories that are in the system default run-time
+ # search path.
+ case " $sys_lib_dlsearch_path " in
+ *" $absdir "*) ;;
+ *)
+ case "$compile_rpath " in
+ *" $absdir "*) ;;
+ *) compile_rpath="$compile_rpath $absdir"
+ esac
+ ;;
+ esac
+ case " $sys_lib_dlsearch_path " in
+ *" $libdir "*) ;;
+ *)
+ case "$finalize_rpath " in
+ *" $libdir "*) ;;
+ *) finalize_rpath="$finalize_rpath $libdir"
+ esac
+ ;;
+ esac
+ fi # $linkmode,$pass = prog,link...
+
+ if test "$alldeplibs" = yes &&
+ { test "$deplibs_check_method" = pass_all ||
+ { test "$build_libtool_libs" = yes &&
+ test -n "$library_names"; }; }; then
+ # We only need to search for static libraries
+ continue
+ fi
+ fi
+
+ link_static=no # Whether the deplib will be linked statically
+ use_static_libs=$prefer_static_libs
+ if test "$use_static_libs" = built && test "$installed" = yes; then
+ use_static_libs=no
+ fi
+ if test -n "$library_names" &&
+ { test "$use_static_libs" = no || test -z "$old_library"; }; then
+ case $host in
+ *cygwin* | *mingw* | *cegcc*)
+ # No point in relinking DLLs because paths are not encoded
+ notinst_deplibs="$notinst_deplibs $lib"
+ need_relink=no
+ ;;
+ *)
+ if test "$installed" = no; then
+ notinst_deplibs="$notinst_deplibs $lib"
+ need_relink=yes
+ fi
+ ;;
+ esac
+ # This is a shared library
+
+ # Warn about portability, can't link against -module's on some
+ # systems (darwin). Don't bleat about dlopened modules though!
+ dlopenmodule=""
+ for dlpremoduletest in $dlprefiles; do
+ if test "X$dlpremoduletest" = "X$lib"; then
+ dlopenmodule="$dlpremoduletest"
+ break
+ fi
+ done
+ if test -z "$dlopenmodule" && test "$shouldnotlink" = yes && test "$pass" = link; then
+ $ECHO
+ if test "$linkmode" = prog; then
+ $ECHO "*** Warning: Linking the executable $output against the loadable module"
+ else
+ $ECHO "*** Warning: Linking the shared library $output against the loadable module"
+ fi
+ $ECHO "*** $linklib is not portable!"
+ fi
+ if test "$linkmode" = lib &&
+ test "$hardcode_into_libs" = yes; then
+ # Hardcode the library path.
+ # Skip directories that are in the system default run-time
+ # search path.
+ case " $sys_lib_dlsearch_path " in
+ *" $absdir "*) ;;
+ *)
+ case "$compile_rpath " in
+ *" $absdir "*) ;;
+ *) compile_rpath="$compile_rpath $absdir"
+ esac
+ ;;
+ esac
+ case " $sys_lib_dlsearch_path " in
+ *" $libdir "*) ;;
+ *)
+ case "$finalize_rpath " in
+ *" $libdir "*) ;;
+ *) finalize_rpath="$finalize_rpath $libdir"
+ esac
+ ;;
+ esac
+ fi
+
+ if test -n "$old_archive_from_expsyms_cmds"; then
+ # figure out the soname
+ set dummy $library_names
+ shift
+ realname="$1"
+ shift
+ libname=`eval "\\$ECHO \"$libname_spec\""`
+ # use dlname if we got it. it's perfectly good, no?
+ if test -n "$dlname"; then
+ soname="$dlname"
+ elif test -n "$soname_spec"; then
+ # bleh windows
+ case $host in
+ *cygwin* | mingw* | *cegcc*)
+ func_arith $current - $age
+ major=$func_arith_result
+ versuffix="-$major"
+ ;;
+ esac
+ eval soname=\"$soname_spec\"
+ else
+ soname="$realname"
+ fi
+
+ # Make a new name for the extract_expsyms_cmds to use
+ soroot="$soname"
+ func_basename "$soroot"
+ soname="$func_basename_result"
+ func_stripname 'lib' '.dll' "$soname"
+ newlib=libimp-$func_stripname_result.a
+
+ # If the library has no export list, then create one now
+ if test -f "$output_objdir/$soname-def"; then :
+ else
+ func_verbose "extracting exported symbol list from \`$soname'"
+ func_execute_cmds "$extract_expsyms_cmds" 'exit $?'
+ fi
+
+ # Create $newlib
+ if test -f "$output_objdir/$newlib"; then :; else
+ func_verbose "generating import library for \`$soname'"
+ func_execute_cmds "$old_archive_from_expsyms_cmds" 'exit $?'
+ fi
+ # make sure the library variables are pointing to the new library
+ dir=$output_objdir
+ linklib=$newlib
+ fi # test -n "$old_archive_from_expsyms_cmds"
+
+ if test "$linkmode" = prog || test "$mode" != relink; then
+ add_shlibpath=
+ add_dir=
+ add=
+ lib_linked=yes
+ case $hardcode_action in
+ immediate | unsupported)
+ if test "$hardcode_direct" = no; then
+ add="$dir/$linklib"
+ case $host in
+ *-*-sco3.2v5.0.[024]*) add_dir="-L$dir" ;;
+ *-*-sysv4*uw2*) add_dir="-L$dir" ;;
+ *-*-sysv5OpenUNIX* | *-*-sysv5UnixWare7.[01].[10]* | \
+ *-*-unixware7*) add_dir="-L$dir" ;;
+ *-*-darwin* )
+ # if the lib is a (non-dlopened) module then we can not
+ # link against it, someone is ignoring the earlier warnings
+ if /usr/bin/file -L $add 2> /dev/null |
+ $GREP ": [^:]* bundle" >/dev/null ; then
+ if test "X$dlopenmodule" != "X$lib"; then
+ $ECHO "*** Warning: lib $linklib is a module, not a shared library"
+ if test -z "$old_library" ; then
+ $ECHO
+ $ECHO "*** And there doesn't seem to be a static archive available"
+ $ECHO "*** The link will probably fail, sorry"
+ else
+ add="$dir/$old_library"
+ fi
+ elif test -n "$old_library"; then
+ add="$dir/$old_library"
+ fi
+ fi
+ esac
+ elif test "$hardcode_minus_L" = no; then
+ case $host in
+ *-*-sunos*) add_shlibpath="$dir" ;;
+ esac
+ add_dir="-L$dir"
+ add="-l$name"
+ elif test "$hardcode_shlibpath_var" = no; then
+ add_shlibpath="$dir"
+ add="-l$name"
+ else
+ lib_linked=no
+ fi
+ ;;
+ relink)
+ if test "$hardcode_direct" = yes &&
+ test "$hardcode_direct_absolute" = no; then
+ add="$dir/$linklib"
+ elif test "$hardcode_minus_L" = yes; then
+ add_dir="-L$dir"
+ # Try looking first in the location we're being installed to.
+ if test -n "$inst_prefix_dir"; then
+ case $libdir in
+ [\\/]*)
+ add_dir="$add_dir -L$inst_prefix_dir$libdir"
+ ;;
+ esac
+ fi
+ add="-l$name"
+ elif test "$hardcode_shlibpath_var" = yes; then
+ add_shlibpath="$dir"
+ add="-l$name"
+ else
+ lib_linked=no
+ fi
+ ;;
+ *) lib_linked=no ;;
+ esac
+
+ if test "$lib_linked" != yes; then
+ func_fatal_configuration "unsupported hardcode properties"
+ fi
+
+ if test -n "$add_shlibpath"; then
+ case :$compile_shlibpath: in
+ *":$add_shlibpath:"*) ;;
+ *) compile_shlibpath="$compile_shlibpath$add_shlibpath:" ;;
+ esac
+ fi
+ if test "$linkmode" = prog; then
+ test -n "$add_dir" && compile_deplibs="$add_dir $compile_deplibs"
+ test -n "$add" && compile_deplibs="$add $compile_deplibs"
+ else
+ test -n "$add_dir" && deplibs="$add_dir $deplibs"
+ test -n "$add" && deplibs="$add $deplibs"
+ if test "$hardcode_direct" != yes &&
+ test "$hardcode_minus_L" != yes &&
+ test "$hardcode_shlibpath_var" = yes; then
+ case :$finalize_shlibpath: in
+ *":$libdir:"*) ;;
+ *) finalize_shlibpath="$finalize_shlibpath$libdir:" ;;
+ esac
+ fi
+ fi
+ fi
+
+ if test "$linkmode" = prog || test "$mode" = relink; then
+ add_shlibpath=
+ add_dir=
+ add=
+ # Finalize command for both is simple: just hardcode it.
+ if test "$hardcode_direct" = yes &&
+ test "$hardcode_direct_absolute" = no; then
+ add="$libdir/$linklib"
+ elif test "$hardcode_minus_L" = yes; then
+ add_dir="-L$libdir"
+ add="-l$name"
+ elif test "$hardcode_shlibpath_var" = yes; then
+ case :$finalize_shlibpath: in
+ *":$libdir:"*) ;;
+ *) finalize_shlibpath="$finalize_shlibpath$libdir:" ;;
+ esac
+ add="-l$name"
+ elif test "$hardcode_automatic" = yes; then
+ if test -n "$inst_prefix_dir" &&
+ test -f "$inst_prefix_dir$libdir/$linklib" ; then
+ add="$inst_prefix_dir$libdir/$linklib"
+ else
+ add="$libdir/$linklib"
+ fi
+ else
+ # We cannot seem to hardcode it, guess we'll fake it.
+ add_dir="-L$libdir"
+ # Try looking first in the location we're being installed to.
+ if test -n "$inst_prefix_dir"; then
+ case $libdir in
+ [\\/]*)
+ add_dir="$add_dir -L$inst_prefix_dir$libdir"
+ ;;
+ esac
+ fi
+ add="-l$name"
+ fi
+
+ if test "$linkmode" = prog; then
+ test -n "$add_dir" && finalize_deplibs="$add_dir $finalize_deplibs"
+ test -n "$add" && finalize_deplibs="$add $finalize_deplibs"
+ else
+ test -n "$add_dir" && deplibs="$add_dir $deplibs"
+ test -n "$add" && deplibs="$add $deplibs"
+ fi
+ fi
+ elif test "$linkmode" = prog; then
+ # Here we assume that one of hardcode_direct or hardcode_minus_L
+ # is not unsupported. This is valid on all known static and
+ # shared platforms.
+ if test "$hardcode_direct" != unsupported; then
+ test -n "$old_library" && linklib="$old_library"
+ compile_deplibs="$dir/$linklib $compile_deplibs"
+ finalize_deplibs="$dir/$linklib $finalize_deplibs"
+ else
+ compile_deplibs="-l$name -L$dir $compile_deplibs"
+ finalize_deplibs="-l$name -L$dir $finalize_deplibs"
+ fi
+ elif test "$build_libtool_libs" = yes; then
+ # Not a shared library
+ if test "$deplibs_check_method" != pass_all; then
+ # We're trying link a shared library against a static one
+ # but the system doesn't support it.
+
+ # Just print a warning and add the library to dependency_libs so
+ # that the program can be linked against the static library.
+ $ECHO
+ $ECHO "*** Warning: This system can not link to static lib archive $lib."
+ $ECHO "*** I have the capability to make that library automatically link in when"
+ $ECHO "*** you link to this library. But I can only do this if you have a"
+ $ECHO "*** shared version of the library, which you do not appear to have."
+ if test "$module" = yes; then
+ $ECHO "*** But as you try to build a module library, libtool will still create "
+ $ECHO "*** a static module, that should work as long as the dlopening application"
+ $ECHO "*** is linked with the -dlopen flag to resolve symbols at runtime."
+ if test -z "$global_symbol_pipe"; then
+ $ECHO
+ $ECHO "*** However, this would only work if libtool was able to extract symbol"
+ $ECHO "*** lists from a program, using \`nm' or equivalent, but libtool could"
+ $ECHO "*** not find such a program. So, this module is probably useless."
+ $ECHO "*** \`nm' from GNU binutils and a full rebuild may help."
+ fi
+ if test "$build_old_libs" = no; then
+ build_libtool_libs=module
+ build_old_libs=yes
+ else
+ build_libtool_libs=no
+ fi
+ fi
+ else
+ deplibs="$dir/$old_library $deplibs"
+ link_static=yes
+ fi
+ fi # link shared/static library?
+
+ if test "$linkmode" = lib; then
+ if test -n "$dependency_libs" &&
+ { test "$hardcode_into_libs" != yes ||
+ test "$build_old_libs" = yes ||
+ test "$link_static" = yes; }; then
+ # Extract -R from dependency_libs
+ temp_deplibs=
+ for libdir in $dependency_libs; do
+ case $libdir in
+ -R*) func_stripname '-R' '' "$libdir"
+ temp_xrpath=$func_stripname_result
+ case " $xrpath " in
+ *" $temp_xrpath "*) ;;
+ *) xrpath="$xrpath $temp_xrpath";;
+ esac;;
+ *) temp_deplibs="$temp_deplibs $libdir";;
+ esac
+ done
+ dependency_libs="$temp_deplibs"
+ fi
+
+ newlib_search_path="$newlib_search_path $absdir"
+ # Link against this library
+ test "$link_static" = no && newdependency_libs="$abs_ladir/$laname $newdependency_libs"
+ # ... and its dependency_libs
+ tmp_libs=
+ for deplib in $dependency_libs; do
+ newdependency_libs="$deplib $newdependency_libs"
+ if $opt_duplicate_deps ; then
+ case "$tmp_libs " in
+ *" $deplib "*) specialdeplibs="$specialdeplibs $deplib" ;;
+ esac
+ fi
+ tmp_libs="$tmp_libs $deplib"
+ done
+
+ if test "$link_all_deplibs" != no; then
+ # Add the search paths of all dependency libraries
+ for deplib in $dependency_libs; do
+ case $deplib in
+ -L*) path="$deplib" ;;
+ *.la)
+ func_dirname "$deplib" "" "."
+ dir="$func_dirname_result"
+ # We need an absolute path.
+ case $dir in
+ [\\/]* | [A-Za-z]:[\\/]*) absdir="$dir" ;;
+ *)
+ absdir=`cd "$dir" && pwd`
+ if test -z "$absdir"; then
+ func_warning "cannot determine absolute directory name of \`$dir'"
+ absdir="$dir"
+ fi
+ ;;
+ esac
+ if $GREP "^installed=no" $deplib > /dev/null; then
+ case $host in
+ *-*-darwin*)
+ depdepl=
+ eval deplibrary_names=`${SED} -n -e 's/^library_names=\(.*\)$/\1/p' $deplib`
+ if test -n "$deplibrary_names" ; then
+ for tmp in $deplibrary_names ; do
+ depdepl=$tmp
+ done
+ if test -f "$absdir/$objdir/$depdepl" ; then
+ depdepl="$absdir/$objdir/$depdepl"
+ darwin_install_name=`${OTOOL} -L $depdepl | awk '{if (NR == 2) {print $1;exit}}'`
+ if test -z "$darwin_install_name"; then
+ darwin_install_name=`${OTOOL64} -L $depdepl | awk '{if (NR == 2) {print $1;exit}}'`
+ fi
+ compiler_flags="$compiler_flags ${wl}-dylib_file ${wl}${darwin_install_name}:${depdepl}"
+ linker_flags="$linker_flags -dylib_file ${darwin_install_name}:${depdepl}"
+ path=
+ fi
+ fi
+ ;;
+ *)
+ path="-L$absdir/$objdir"
+ ;;
+ esac
+ else
+ eval libdir=`${SED} -n -e 's/^libdir=\(.*\)$/\1/p' $deplib`
+ test -z "$libdir" && \
+ func_fatal_error "\`$deplib' is not a valid libtool archive"
+ test "$absdir" != "$libdir" && \
+ func_warning "\`$deplib' seems to be moved"
+
+ path="-L$absdir"
+ fi
+ ;;
+ esac
+ case " $deplibs " in
+ *" $path "*) ;;
+ *) deplibs="$path $deplibs" ;;
+ esac
+ done
+ fi # link_all_deplibs != no
+ fi # linkmode = lib
+ done # for deplib in $libs
+ if test "$pass" = link; then
+ if test "$linkmode" = "prog"; then
+ compile_deplibs="$new_inherited_linker_flags $compile_deplibs"
+ finalize_deplibs="$new_inherited_linker_flags $finalize_deplibs"
+ else
+ compiler_flags="$compiler_flags "`$ECHO "X $new_inherited_linker_flags" | $Xsed -e 's% \([^ $]*\).ltframework% -framework \1%g'`
+ fi
+ fi
+ dependency_libs="$newdependency_libs"
+ if test "$pass" = dlpreopen; then
+ # Link the dlpreopened libraries before other libraries
+ for deplib in $save_deplibs; do
+ deplibs="$deplib $deplibs"
+ done
+ fi
+ if test "$pass" != dlopen; then
+ if test "$pass" != conv; then
+ # Make sure lib_search_path contains only unique directories.
+ lib_search_path=
+ for dir in $newlib_search_path; do
+ case "$lib_search_path " in
+ *" $dir "*) ;;
+ *) lib_search_path="$lib_search_path $dir" ;;
+ esac
+ done
+ newlib_search_path=
+ fi
+
+ if test "$linkmode,$pass" != "prog,link"; then
+ vars="deplibs"
+ else
+ vars="compile_deplibs finalize_deplibs"
+ fi
+ for var in $vars dependency_libs; do
+ # Add libraries to $var in reverse order
+ eval tmp_libs=\"\$$var\"
+ new_libs=
+ for deplib in $tmp_libs; do
+ # FIXME: Pedantically, this is the right thing to do, so
+ # that some nasty dependency loop isn't accidentally
+ # broken:
+ #new_libs="$deplib $new_libs"
+ # Pragmatically, this seems to cause very few problems in
+ # practice:
+ case $deplib in
+ -L*) new_libs="$deplib $new_libs" ;;
+ -R*) ;;
+ *)
+ # And here is the reason: when a library appears more
+ # than once as an explicit dependence of a library, or
+ # is implicitly linked in more than once by the
+ # compiler, it is considered special, and multiple
+ # occurrences thereof are not removed. Compare this
+ # with having the same library being listed as a
+ # dependency of multiple other libraries: in this case,
+ # we know (pedantically, we assume) the library does not
+ # need to be listed more than once, so we keep only the
+ # last copy. This is not always right, but it is rare
+ # enough that we require users that really mean to play
+ # such unportable linking tricks to link the library
+ # using -Wl,-lname, so that libtool does not consider it
+ # for duplicate removal.
+ case " $specialdeplibs " in
+ *" $deplib "*) new_libs="$deplib $new_libs" ;;
+ *)
+ case " $new_libs " in
+ *" $deplib "*) ;;
+ *) new_libs="$deplib $new_libs" ;;
+ esac
+ ;;
+ esac
+ ;;
+ esac
+ done
+ tmp_libs=
+ for deplib in $new_libs; do
+ case $deplib in
+ -L*)
+ case " $tmp_libs " in
+ *" $deplib "*) ;;
+ *) tmp_libs="$tmp_libs $deplib" ;;
+ esac
+ ;;
+ *) tmp_libs="$tmp_libs $deplib" ;;
+ esac
+ done
+ eval $var=\"$tmp_libs\"
+ done # for var
+ fi
+ # Last step: remove runtime libs from dependency_libs
+ # (they stay in deplibs)
+ tmp_libs=
+ for i in $dependency_libs ; do
+ case " $predeps $postdeps $compiler_lib_search_path " in
+ *" $i "*)
+ i=""
+ ;;
+ esac
+ if test -n "$i" ; then
+ tmp_libs="$tmp_libs $i"
+ fi
+ done
+ dependency_libs=$tmp_libs
+ done # for pass
+ if test "$linkmode" = prog; then
+ dlfiles="$newdlfiles"
+ fi
+ if test "$linkmode" = prog || test "$linkmode" = lib; then
+ dlprefiles="$newdlprefiles"
+ fi
+
+ case $linkmode in
+ oldlib)
+ if test -n "$dlfiles$dlprefiles" || test "$dlself" != no; then
+ func_warning "\`-dlopen' is ignored for archives"
+ fi
+
+ case " $deplibs" in
+ *\ -l* | *\ -L*)
+ func_warning "\`-l' and \`-L' are ignored for archives" ;;
+ esac
+
+ test -n "$rpath" && \
+ func_warning "\`-rpath' is ignored for archives"
+
+ test -n "$xrpath" && \
+ func_warning "\`-R' is ignored for archives"
+
+ test -n "$vinfo" && \
+ func_warning "\`-version-info/-version-number' is ignored for archives"
+
+ test -n "$release" && \
+ func_warning "\`-release' is ignored for archives"
+
+ test -n "$export_symbols$export_symbols_regex" && \
+ func_warning "\`-export-symbols' is ignored for archives"
+
+ # Now set the variables for building old libraries.
+ build_libtool_libs=no
+ oldlibs="$output"
+ objs="$objs$old_deplibs"
+ ;;
+
+ lib)
+ # Make sure we only generate libraries of the form `libNAME.la'.
+ case $outputname in
+ lib*)
+ func_stripname 'lib' '.la' "$outputname"
+ name=$func_stripname_result
+ eval shared_ext=\"$shrext_cmds\"
+ eval libname=\"$libname_spec\"
+ ;;
+ *)
+ test "$module" = no && \
+ func_fatal_help "libtool library \`$output' must begin with \`lib'"
+
+ if test "$need_lib_prefix" != no; then
+ # Add the "lib" prefix for modules if required
+ func_stripname '' '.la' "$outputname"
+ name=$func_stripname_result
+ eval shared_ext=\"$shrext_cmds\"
+ eval libname=\"$libname_spec\"
+ else
+ func_stripname '' '.la' "$outputname"
+ libname=$func_stripname_result
+ fi
+ ;;
+ esac
+
+ if test -n "$objs"; then
+ if test "$deplibs_check_method" != pass_all; then
+ func_fatal_error "cannot build libtool library \`$output' from non-libtool objects on this host:$objs"
+ else
+ $ECHO
+ $ECHO "*** Warning: Linking the shared library $output against the non-libtool"
+ $ECHO "*** objects $objs is not portable!"
+ libobjs="$libobjs $objs"
+ fi
+ fi
+
+ test "$dlself" != no && \
+ func_warning "\`-dlopen self' is ignored for libtool libraries"
+
+ set dummy $rpath
+ shift
+ test "$#" -gt 1 && \
+ func_warning "ignoring multiple \`-rpath's for a libtool library"
+
+ install_libdir="$1"
+
+ oldlibs=
+ if test -z "$rpath"; then
+ if test "$build_libtool_libs" = yes; then
+ # Building a libtool convenience library.
+ # Some compilers have problems with a `.al' extension so
+ # convenience libraries should have the same extension an
+ # archive normally would.
+ oldlibs="$output_objdir/$libname.$libext $oldlibs"
+ build_libtool_libs=convenience
+ build_old_libs=yes
+ fi
+
+ test -n "$vinfo" && \
+ func_warning "\`-version-info/-version-number' is ignored for convenience libraries"
+
+ test -n "$release" && \
+ func_warning "\`-release' is ignored for convenience libraries"
+ else
+
+ # Parse the version information argument.
+ save_ifs="$IFS"; IFS=':'
+ set dummy $vinfo 0 0 0
+ shift
+ IFS="$save_ifs"
+
+ test -n "$7" && \
+ func_fatal_help "too many parameters to \`-version-info'"
+
+ # convert absolute version numbers to libtool ages
+ # this retains compatibility with .la files and attempts
+ # to make the code below a bit more comprehensible
+
+ case $vinfo_number in
+ yes)
+ number_major="$1"
+ number_minor="$2"
+ number_revision="$3"
+ #
+ # There are really only two kinds -- those that
+ # use the current revision as the major version
+ # and those that subtract age and use age as
+ # a minor version. But, then there is irix
+ # which has an extra 1 added just for fun
+ #
+ case $version_type in
+ darwin|linux|osf|windows|none)
+ func_arith $number_major + $number_minor
+ current=$func_arith_result
+ age="$number_minor"
+ revision="$number_revision"
+ ;;
+ freebsd-aout|freebsd-elf|sunos)
+ current="$number_major"
+ revision="$number_minor"
+ age="0"
+ ;;
+ irix|nonstopux)
+ func_arith $number_major + $number_minor
+ current=$func_arith_result
+ age="$number_minor"
+ revision="$number_minor"
+ lt_irix_increment=no
+ ;;
+ esac
+ ;;
+ no)
+ current="$1"
+ revision="$2"
+ age="$3"
+ ;;
+ esac
+
+ # Check that each of the things are valid numbers.
+ case $current in
+ 0|[1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-9][0-9][0-9][0-9]|[1-9][0-9][0-9][0-9][0-9]) ;;
+ *)
+ func_error "CURRENT \`$current' must be a nonnegative integer"
+ func_fatal_error "\`$vinfo' is not valid version information"
+ ;;
+ esac
+
+ case $revision in
+ 0|[1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-9][0-9][0-9][0-9]|[1-9][0-9][0-9][0-9][0-9]) ;;
+ *)
+ func_error "REVISION \`$revision' must be a nonnegative integer"
+ func_fatal_error "\`$vinfo' is not valid version information"
+ ;;
+ esac
+
+ case $age in
+ 0|[1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-9][0-9][0-9][0-9]|[1-9][0-9][0-9][0-9][0-9]) ;;
+ *)
+ func_error "AGE \`$age' must be a nonnegative integer"
+ func_fatal_error "\`$vinfo' is not valid version information"
+ ;;
+ esac
+
+ if test "$age" -gt "$current"; then
+ func_error "AGE \`$age' is greater than the current interface number \`$current'"
+ func_fatal_error "\`$vinfo' is not valid version information"
+ fi
+
+ # Calculate the version variables.
+ major=
+ versuffix=
+ verstring=
+ case $version_type in
+ none) ;;
+
+ darwin)
+ # Like Linux, but with the current version available in
+ # verstring for coding it into the library header
+ func_arith $current - $age
+ major=.$func_arith_result
+ versuffix="$major.$age.$revision"
+ # Darwin ld doesn't like 0 for these options...
+ func_arith $current + 1
+ minor_current=$func_arith_result
+ xlcverstring="${wl}-compatibility_version ${wl}$minor_current ${wl}-current_version ${wl}$minor_current.$revision"
+ verstring="-compatibility_version $minor_current -current_version $minor_current.$revision"
+ ;;
+
+ freebsd-aout)
+ major=".$current"
+ versuffix=".$current.$revision";
+ ;;
+
+ freebsd-elf)
+ major=".$current"
+ versuffix=".$current"
+ ;;
+
+ irix | nonstopux)
+ if test "X$lt_irix_increment" = "Xno"; then
+ func_arith $current - $age
+ else
+ func_arith $current - $age + 1
+ fi
+ major=$func_arith_result
+
+ case $version_type in
+ nonstopux) verstring_prefix=nonstopux ;;
+ *) verstring_prefix=sgi ;;
+ esac
+ verstring="$verstring_prefix$major.$revision"
+
+ # Add in all the interfaces that we are compatible with.
+ loop=$revision
+ while test "$loop" -ne 0; do
+ func_arith $revision - $loop
+ iface=$func_arith_result
+ func_arith $loop - 1
+ loop=$func_arith_result
+ verstring="$verstring_prefix$major.$iface:$verstring"
+ done
+
+ # Before this point, $major must not contain `.'.
+ major=.$major
+ versuffix="$major.$revision"
+ ;;
+
+ linux)
+ func_arith $current - $age
+ major=.$func_arith_result
+ versuffix="$major.$age.$revision"
+ ;;
+
+ osf)
+ func_arith $current - $age
+ major=.$func_arith_result
+ versuffix=".$current.$age.$revision"
+ verstring="$current.$age.$revision"
+
+ # Add in all the interfaces that we are compatible with.
+ loop=$age
+ while test "$loop" -ne 0; do
+ func_arith $current - $loop
+ iface=$func_arith_result
+ func_arith $loop - 1
+ loop=$func_arith_result
+ verstring="$verstring:${iface}.0"
+ done
+
+ # Make executables depend on our current version.
+ verstring="$verstring:${current}.0"
+ ;;
+
+ qnx)
+ major=".$current"
+ versuffix=".$current"
+ ;;
+
+ sunos)
+ major=".$current"
+ versuffix=".$current.$revision"
+ ;;
+
+ windows)
+ # Use '-' rather than '.', since we only want one
+ # extension on DOS 8.3 filesystems.
+ func_arith $current - $age
+ major=$func_arith_result
+ versuffix="-$major"
+ ;;
+
+ *)
+ func_fatal_configuration "unknown library version type \`$version_type'"
+ ;;
+ esac
+
+ # Clear the version info if we defaulted, and they specified a release.
+ if test -z "$vinfo" && test -n "$release"; then
+ major=
+ case $version_type in
+ darwin)
+ # we can't check for "0.0" in archive_cmds due to quoting
+ # problems, so we reset it completely
+ verstring=
+ ;;
+ *)
+ verstring="0.0"
+ ;;
+ esac
+ if test "$need_version" = no; then
+ versuffix=
+ else
+ versuffix=".0.0"
+ fi
+ fi
+
+ # Remove version info from name if versioning should be avoided
+ if test "$avoid_version" = yes && test "$need_version" = no; then
+ major=
+ versuffix=
+ verstring=""
+ fi
+
+ # Check to see if the archive will have undefined symbols.
+ if test "$allow_undefined" = yes; then
+ if test "$allow_undefined_flag" = unsupported; then
+ func_warning "undefined symbols not allowed in $host shared libraries"
+ build_libtool_libs=no
+ build_old_libs=yes
+ fi
+ else
+ # Don't allow undefined symbols.
+ allow_undefined_flag="$no_undefined_flag"
+ fi
+
+ fi
+
+ func_generate_dlsyms "$libname" "$libname" "yes"
+ libobjs="$libobjs $symfileobj"
+ test "X$libobjs" = "X " && libobjs=
+
+ if test "$mode" != relink; then
+ # Remove our outputs, but don't remove object files since they
+ # may have been created when compiling PIC objects.
+ removelist=
+ tempremovelist=`$ECHO "$output_objdir/*"`
+ for p in $tempremovelist; do
+ case $p in
+ *.$objext | *.gcno)
+ ;;
+ $output_objdir/$outputname | $output_objdir/$libname.* | $output_objdir/${libname}${release}.*)
+ if test "X$precious_files_regex" != "X"; then
+ if $ECHO "$p" | $EGREP -e "$precious_files_regex" >/dev/null 2>&1
+ then
+ continue
+ fi
+ fi
+ removelist="$removelist $p"
+ ;;
+ *) ;;
+ esac
+ done
+ test -n "$removelist" && \
+ func_show_eval "${RM}r \$removelist"
+ fi
+
+ # Now set the variables for building old libraries.
+ if test "$build_old_libs" = yes && test "$build_libtool_libs" != convenience ; then
+ oldlibs="$oldlibs $output_objdir/$libname.$libext"
+
+ # Transform .lo files to .o files.
+ oldobjs="$objs "`$ECHO "X$libobjs" | $SP2NL | $Xsed -e '/\.'${libext}'$/d' -e "$lo2o" | $NL2SP`
+ fi
+
+ # Eliminate all temporary directories.
+ #for path in $notinst_path; do
+ # lib_search_path=`$ECHO "X$lib_search_path " | $Xsed -e "s% $path % %g"`
+ # deplibs=`$ECHO "X$deplibs " | $Xsed -e "s% -L$path % %g"`
+ # dependency_libs=`$ECHO "X$dependency_libs " | $Xsed -e "s% -L$path % %g"`
+ #done
+
+ if test -n "$xrpath"; then
+ # If the user specified any rpath flags, then add them.
+ temp_xrpath=
+ for libdir in $xrpath; do
+ temp_xrpath="$temp_xrpath -R$libdir"
+ case "$finalize_rpath " in
+ *" $libdir "*) ;;
+ *) finalize_rpath="$finalize_rpath $libdir" ;;
+ esac
+ done
+ if test "$hardcode_into_libs" != yes || test "$build_old_libs" = yes; then
+ dependency_libs="$temp_xrpath $dependency_libs"
+ fi
+ fi
+
+ # Make sure dlfiles contains only unique files that won't be dlpreopened
+ old_dlfiles="$dlfiles"
+ dlfiles=
+ for lib in $old_dlfiles; do
+ case " $dlprefiles $dlfiles " in
+ *" $lib "*) ;;
+ *) dlfiles="$dlfiles $lib" ;;
+ esac
+ done
+
+ # Make sure dlprefiles contains only unique files
+ old_dlprefiles="$dlprefiles"
+ dlprefiles=
+ for lib in $old_dlprefiles; do
+ case "$dlprefiles " in
+ *" $lib "*) ;;
+ *) dlprefiles="$dlprefiles $lib" ;;
+ esac
+ done
+
+ if test "$build_libtool_libs" = yes; then
+ if test -n "$rpath"; then
+ case $host in
+ *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-*-beos* | *-cegcc*)
+ # these systems don't actually have a c library (as such)!
+ ;;
+ *-*-rhapsody* | *-*-darwin1.[012])
+ # Rhapsody C library is in the System framework
+ deplibs="$deplibs System.ltframework"
+ ;;
+ *-*-netbsd*)
+ # Don't link with libc until the a.out ld.so is fixed.
+ ;;
+ *-*-openbsd* | *-*-freebsd* | *-*-dragonfly*)
+ # Do not include libc due to us having libc/libc_r.
+ ;;
+ *-*-sco3.2v5* | *-*-sco5v6*)
+ # Causes problems with __ctype
+ ;;
+ *-*-sysv4.2uw2* | *-*-sysv5* | *-*-unixware* | *-*-OpenUNIX*)
+ # Compiler inserts libc in the correct place for threads to work
+ ;;
+ *)
+ # Add libc to deplibs on all other systems if necessary.
+ if test "$build_libtool_need_lc" = "yes"; then
+ deplibs="$deplibs -lc"
+ fi
+ ;;
+ esac
+ fi
+
+ # Transform deplibs into only deplibs that can be linked in shared.
+ name_save=$name
+ libname_save=$libname
+ release_save=$release
+ versuffix_save=$versuffix
+ major_save=$major
+ # I'm not sure if I'm treating the release correctly. I think
+ # release should show up in the -l (ie -lgmp5) so we don't want to
+ # add it in twice. Is that correct?
+ release=""
+ versuffix=""
+ major=""
+ newdeplibs=
+ droppeddeps=no
+ case $deplibs_check_method in
+ pass_all)
+ # Don't check for shared/static. Everything works.
+ # This might be a little naive. We might want to check
+ # whether the library exists or not. But this is on
+ # osf3 & osf4 and I'm not really sure... Just
+ # implementing what was already the behavior.
+ newdeplibs=$deplibs
+ ;;
+ test_compile)
+ # This code stresses the "libraries are programs" paradigm to its
+ # limits. Maybe even breaks it. We compile a program, linking it
+ # against the deplibs as a proxy for the library. Then we can check
+ # whether they linked in statically or dynamically with ldd.
+ $opt_dry_run || $RM conftest.c
+ cat > conftest.c <<EOF
+ int main() { return 0; }
+EOF
+ $opt_dry_run || $RM conftest
+ if $LTCC $LTCFLAGS -o conftest conftest.c $deplibs; then
+ ldd_output=`ldd conftest`
+ for i in $deplibs; do
+ case $i in
+ -l*)
+ func_stripname -l '' "$i"
+ name=$func_stripname_result
+ if test "X$allow_libtool_libs_with_static_runtimes" = "Xyes" ; then
+ case " $predeps $postdeps " in
+ *" $i "*)
+ newdeplibs="$newdeplibs $i"
+ i=""
+ ;;
+ esac
+ fi
+ if test -n "$i" ; then
+ libname=`eval "\\$ECHO \"$libname_spec\""`
+ deplib_matches=`eval "\\$ECHO \"$library_names_spec\""`
+ set dummy $deplib_matches; shift
+ deplib_match=$1
+ if test `expr "$ldd_output" : ".*$deplib_match"` -ne 0 ; then
+ newdeplibs="$newdeplibs $i"
+ else
+ droppeddeps=yes
+ $ECHO
+ $ECHO "*** Warning: dynamic linker does not accept needed library $i."
+ $ECHO "*** I have the capability to make that library automatically link in when"
+ $ECHO "*** you link to this library. But I can only do this if you have a"
+ $ECHO "*** shared version of the library, which I believe you do not have"
+ $ECHO "*** because a test_compile did reveal that the linker did not use it for"
+ $ECHO "*** its dynamic dependency list that programs get resolved with at runtime."
+ fi
+ fi
+ ;;
+ *)
+ newdeplibs="$newdeplibs $i"
+ ;;
+ esac
+ done
+ else
+ # Error occurred in the first compile. Let's try to salvage
+ # the situation: Compile a separate program for each library.
+ for i in $deplibs; do
+ case $i in
+ -l*)
+ func_stripname -l '' "$i"
+ name=$func_stripname_result
+ $opt_dry_run || $RM conftest
+ if $LTCC $LTCFLAGS -o conftest conftest.c $i; then
+ ldd_output=`ldd conftest`
+ if test "X$allow_libtool_libs_with_static_runtimes" = "Xyes" ; then
+ case " $predeps $postdeps " in
+ *" $i "*)
+ newdeplibs="$newdeplibs $i"
+ i=""
+ ;;
+ esac
+ fi
+ if test -n "$i" ; then
+ libname=`eval "\\$ECHO \"$libname_spec\""`
+ deplib_matches=`eval "\\$ECHO \"$library_names_spec\""`
+ set dummy $deplib_matches; shift
+ deplib_match=$1
+ if test `expr "$ldd_output" : ".*$deplib_match"` -ne 0 ; then
+ newdeplibs="$newdeplibs $i"
+ else
+ droppeddeps=yes
+ $ECHO
+ $ECHO "*** Warning: dynamic linker does not accept needed library $i."
+ $ECHO "*** I have the capability to make that library automatically link in when"
+ $ECHO "*** you link to this library. But I can only do this if you have a"
+ $ECHO "*** shared version of the library, which you do not appear to have"
+ $ECHO "*** because a test_compile did reveal that the linker did not use this one"
+ $ECHO "*** as a dynamic dependency that programs can get resolved with at runtime."
+ fi
+ fi
+ else
+ droppeddeps=yes
+ $ECHO
+ $ECHO "*** Warning! Library $i is needed by this library but I was not able to"
+ $ECHO "*** make it link in! You will probably need to install it or some"
+ $ECHO "*** library that it depends on before this library will be fully"
+ $ECHO "*** functional. Installing it before continuing would be even better."
+ fi
+ ;;
+ *)
+ newdeplibs="$newdeplibs $i"
+ ;;
+ esac
+ done
+ fi
+ ;;
+ file_magic*)
+ set dummy $deplibs_check_method; shift
+ file_magic_regex=`expr "$deplibs_check_method" : "$1 \(.*\)"`
+ for a_deplib in $deplibs; do
+ case $a_deplib in
+ -l*)
+ func_stripname -l '' "$a_deplib"
+ name=$func_stripname_result
+ if test "X$allow_libtool_libs_with_static_runtimes" = "Xyes" ; then
+ case " $predeps $postdeps " in
+ *" $a_deplib "*)
+ newdeplibs="$newdeplibs $a_deplib"
+ a_deplib=""
+ ;;
+ esac
+ fi
+ if test -n "$a_deplib" ; then
+ libname=`eval "\\$ECHO \"$libname_spec\""`
+ for i in $lib_search_path $sys_lib_search_path $shlib_search_path; do
+ potential_libs=`ls $i/$libname[.-]* 2>/dev/null`
+ for potent_lib in $potential_libs; do
+ # Follow soft links.
+ if ls -lLd "$potent_lib" 2>/dev/null |
+ $GREP " -> " >/dev/null; then
+ continue
+ fi
+ # The statement above tries to avoid entering an
+ # endless loop below, in case of cyclic links.
+ # We might still enter an endless loop, since a link
+ # loop can be closed while we follow links,
+ # but so what?
+ potlib="$potent_lib"
+ while test -h "$potlib" 2>/dev/null; do
+ potliblink=`ls -ld $potlib | ${SED} 's/.* -> //'`
+ case $potliblink in
+ [\\/]* | [A-Za-z]:[\\/]*) potlib="$potliblink";;
+ *) potlib=`$ECHO "X$potlib" | $Xsed -e 's,[^/]*$,,'`"$potliblink";;
+ esac
+ done
+ if eval $file_magic_cmd \"\$potlib\" 2>/dev/null |
+ $SED -e 10q |
+ $EGREP "$file_magic_regex" > /dev/null; then
+ newdeplibs="$newdeplibs $a_deplib"
+ a_deplib=""
+ break 2
+ fi
+ done
+ done
+ fi
+ if test -n "$a_deplib" ; then
+ droppeddeps=yes
+ $ECHO
+ $ECHO "*** Warning: linker path does not have real file for library $a_deplib."
+ $ECHO "*** I have the capability to make that library automatically link in when"
+ $ECHO "*** you link to this library. But I can only do this if you have a"
+ $ECHO "*** shared version of the library, which you do not appear to have"
+ $ECHO "*** because I did check the linker path looking for a file starting"
+ if test -z "$potlib" ; then
+ $ECHO "*** with $libname but no candidates were found. (...for file magic test)"
+ else
+ $ECHO "*** with $libname and none of the candidates passed a file format test"
+ $ECHO "*** using a file magic. Last file checked: $potlib"
+ fi
+ fi
+ ;;
+ *)
+ # Add a -L argument.
+ newdeplibs="$newdeplibs $a_deplib"
+ ;;
+ esac
+ done # Gone through all deplibs.
+ ;;
+ match_pattern*)
+ set dummy $deplibs_check_method; shift
+ match_pattern_regex=`expr "$deplibs_check_method" : "$1 \(.*\)"`
+ for a_deplib in $deplibs; do
+ case $a_deplib in
+ -l*)
+ func_stripname -l '' "$a_deplib"
+ name=$func_stripname_result
+ if test "X$allow_libtool_libs_with_static_runtimes" = "Xyes" ; then
+ case " $predeps $postdeps " in
+ *" $a_deplib "*)
+ newdeplibs="$newdeplibs $a_deplib"
+ a_deplib=""
+ ;;
+ esac
+ fi
+ if test -n "$a_deplib" ; then
+ libname=`eval "\\$ECHO \"$libname_spec\""`
+ for i in $lib_search_path $sys_lib_search_path $shlib_search_path; do
+ potential_libs=`ls $i/$libname[.-]* 2>/dev/null`
+ for potent_lib in $potential_libs; do
+ potlib="$potent_lib" # see symlink-check above in file_magic test
+ if eval "\$ECHO \"X$potent_lib\"" 2>/dev/null | $Xsed -e 10q | \
+ $EGREP "$match_pattern_regex" > /dev/null; then
+ newdeplibs="$newdeplibs $a_deplib"
+ a_deplib=""
+ break 2
+ fi
+ done
+ done
+ fi
+ if test -n "$a_deplib" ; then
+ droppeddeps=yes
+ $ECHO
+ $ECHO "*** Warning: linker path does not have real file for library $a_deplib."
+ $ECHO "*** I have the capability to make that library automatically link in when"
+ $ECHO "*** you link to this library. But I can only do this if you have a"
+ $ECHO "*** shared version of the library, which you do not appear to have"
+ $ECHO "*** because I did check the linker path looking for a file starting"
+ if test -z "$potlib" ; then
+ $ECHO "*** with $libname but no candidates were found. (...for regex pattern test)"
+ else
+ $ECHO "*** with $libname and none of the candidates passed a file format test"
+ $ECHO "*** using a regex pattern. Last file checked: $potlib"
+ fi
+ fi
+ ;;
+ *)
+ # Add a -L argument.
+ newdeplibs="$newdeplibs $a_deplib"
+ ;;
+ esac
+ done # Gone through all deplibs.
+ ;;
+ none | unknown | *)
+ newdeplibs=""
+ tmp_deplibs=`$ECHO "X $deplibs" | $Xsed \
+ -e 's/ -lc$//' -e 's/ -[LR][^ ]*//g'`
+ if test "X$allow_libtool_libs_with_static_runtimes" = "Xyes" ; then
+ for i in $predeps $postdeps ; do
+ # can't use Xsed below, because $i might contain '/'
+ tmp_deplibs=`$ECHO "X $tmp_deplibs" | $Xsed -e "s,$i,,"`
+ done
+ fi
+ if $ECHO "X $tmp_deplibs" | $Xsed -e 's/[ ]//g' |
+ $GREP . >/dev/null; then
+ $ECHO
+ if test "X$deplibs_check_method" = "Xnone"; then
+ $ECHO "*** Warning: inter-library dependencies are not supported in this platform."
+ else
+ $ECHO "*** Warning: inter-library dependencies are not known to be supported."
+ fi
+ $ECHO "*** All declared inter-library dependencies are being dropped."
+ droppeddeps=yes
+ fi
+ ;;
+ esac
+ versuffix=$versuffix_save
+ major=$major_save
+ release=$release_save
+ libname=$libname_save
+ name=$name_save
+
+ case $host in
+ *-*-rhapsody* | *-*-darwin1.[012])
+ # On Rhapsody replace the C library with the System framework
+ newdeplibs=`$ECHO "X $newdeplibs" | $Xsed -e 's/ -lc / System.ltframework /'`
+ ;;
+ esac
+
+ if test "$droppeddeps" = yes; then
+ if test "$module" = yes; then
+ $ECHO
+ $ECHO "*** Warning: libtool could not satisfy all declared inter-library"
+ $ECHO "*** dependencies of module $libname. Therefore, libtool will create"
+ $ECHO "*** a static module, that should work as long as the dlopening"
+ $ECHO "*** application is linked with the -dlopen flag."
+ if test -z "$global_symbol_pipe"; then
+ $ECHO
+ $ECHO "*** However, this would only work if libtool was able to extract symbol"
+ $ECHO "*** lists from a program, using \`nm' or equivalent, but libtool could"
+ $ECHO "*** not find such a program. So, this module is probably useless."
+ $ECHO "*** \`nm' from GNU binutils and a full rebuild may help."
+ fi
+ if test "$build_old_libs" = no; then
+ oldlibs="$output_objdir/$libname.$libext"
+ build_libtool_libs=module
+ build_old_libs=yes
+ else
+ build_libtool_libs=no
+ fi
+ else
+ $ECHO "*** The inter-library dependencies that have been dropped here will be"
+ $ECHO "*** automatically added whenever a program is linked with this library"
+ $ECHO "*** or is declared to -dlopen it."
+
+ if test "$allow_undefined" = no; then
+ $ECHO
+ $ECHO "*** Since this library must not contain undefined symbols,"
+ $ECHO "*** because either the platform does not support them or"
+ $ECHO "*** it was explicitly requested with -no-undefined,"
+ $ECHO "*** libtool will only create a static version of it."
+ if test "$build_old_libs" = no; then
+ oldlibs="$output_objdir/$libname.$libext"
+ build_libtool_libs=module
+ build_old_libs=yes
+ else
+ build_libtool_libs=no
+ fi
+ fi
+ fi
+ fi
+ # Done checking deplibs!
+ deplibs=$newdeplibs
+ fi
+ # Time to change all our "foo.ltframework" stuff back to "-framework foo"
+ case $host in
+ *-*-darwin*)
+ newdeplibs=`$ECHO "X $newdeplibs" | $Xsed -e 's% \([^ $]*\).ltframework% -framework \1%g'`
+ new_inherited_linker_flags=`$ECHO "X $new_inherited_linker_flags" | $Xsed -e 's% \([^ $]*\).ltframework% -framework \1%g'`
+ deplibs=`$ECHO "X $deplibs" | $Xsed -e 's% \([^ $]*\).ltframework% -framework \1%g'`
+ ;;
+ esac
+
+ # move library search paths that coincide with paths to not yet
+ # installed libraries to the beginning of the library search list
+ new_libs=
+ for path in $notinst_path; do
+ case " $new_libs " in
+ *" -L$path/$objdir "*) ;;
+ *)
+ case " $deplibs " in
+ *" -L$path/$objdir "*)
+ new_libs="$new_libs -L$path/$objdir" ;;
+ esac
+ ;;
+ esac
+ done
+ for deplib in $deplibs; do
+ case $deplib in
+ -L*)
+ case " $new_libs " in
+ *" $deplib "*) ;;
+ *) new_libs="$new_libs $deplib" ;;
+ esac
+ ;;
+ *) new_libs="$new_libs $deplib" ;;
+ esac
+ done
+ deplibs="$new_libs"
+
+ # All the library-specific variables (install_libdir is set above).
+ library_names=
+ old_library=
+ dlname=
+
+ # Test again, we may have decided not to build it any more
+ if test "$build_libtool_libs" = yes; then
+ if test "$hardcode_into_libs" = yes; then
+ # Hardcode the library paths
+ hardcode_libdirs=
+ dep_rpath=
+ rpath="$finalize_rpath"
+ test "$mode" != relink && rpath="$compile_rpath$rpath"
+ for libdir in $rpath; do
+ if test -n "$hardcode_libdir_flag_spec"; then
+ if test -n "$hardcode_libdir_separator"; then
+ if test -z "$hardcode_libdirs"; then
+ hardcode_libdirs="$libdir"
+ else
+ # Just accumulate the unique libdirs.
+ case $hardcode_libdir_separator$hardcode_libdirs$hardcode_libdir_separator in
+ *"$hardcode_libdir_separator$libdir$hardcode_libdir_separator"*)
+ ;;
+ *)
+ hardcode_libdirs="$hardcode_libdirs$hardcode_libdir_separator$libdir"
+ ;;
+ esac
+ fi
+ else
+ eval flag=\"$hardcode_libdir_flag_spec\"
+ dep_rpath="$dep_rpath $flag"
+ fi
+ elif test -n "$runpath_var"; then
+ case "$perm_rpath " in
+ *" $libdir "*) ;;
+ *) perm_rpath="$perm_rpath $libdir" ;;
+ esac
+ fi
+ done
+ # Substitute the hardcoded libdirs into the rpath.
+ if test -n "$hardcode_libdir_separator" &&
+ test -n "$hardcode_libdirs"; then
+ libdir="$hardcode_libdirs"
+ if test -n "$hardcode_libdir_flag_spec_ld"; then
+ eval dep_rpath=\"$hardcode_libdir_flag_spec_ld\"
+ else
+ eval dep_rpath=\"$hardcode_libdir_flag_spec\"
+ fi
+ fi
+ if test -n "$runpath_var" && test -n "$perm_rpath"; then
+ # We should set the runpath_var.
+ rpath=
+ for dir in $perm_rpath; do
+ rpath="$rpath$dir:"
+ done
+ eval "$runpath_var='$rpath\$$runpath_var'; export $runpath_var"
+ fi
+ test -n "$dep_rpath" && deplibs="$dep_rpath $deplibs"
+ fi
+
+ shlibpath="$finalize_shlibpath"
+ test "$mode" != relink && shlibpath="$compile_shlibpath$shlibpath"
+ if test -n "$shlibpath"; then
+ eval "$shlibpath_var='$shlibpath\$$shlibpath_var'; export $shlibpath_var"
+ fi
+
+ # Get the real and link names of the library.
+ eval shared_ext=\"$shrext_cmds\"
+ eval library_names=\"$library_names_spec\"
+ set dummy $library_names
+ shift
+ realname="$1"
+ shift
+
+ if test -n "$soname_spec"; then
+ eval soname=\"$soname_spec\"
+ else
+ soname="$realname"
+ fi
+ if test -z "$dlname"; then
+ dlname=$soname
+ fi
+
+ lib="$output_objdir/$realname"
+ linknames=
+ for link
+ do
+ linknames="$linknames $link"
+ done
+
+ # Use standard objects if they are pic
+ test -z "$pic_flag" && libobjs=`$ECHO "X$libobjs" | $SP2NL | $Xsed -e "$lo2o" | $NL2SP`
+ test "X$libobjs" = "X " && libobjs=
+
+ delfiles=
+ if test -n "$export_symbols" && test -n "$include_expsyms"; then
+ $opt_dry_run || cp "$export_symbols" "$output_objdir/$libname.uexp"
+ export_symbols="$output_objdir/$libname.uexp"
+ delfiles="$delfiles $export_symbols"
+ fi
+
+ orig_export_symbols=
+ case $host_os in
+ cygwin* | mingw* | cegcc*)
+ if test -n "$export_symbols" && test -z "$export_symbols_regex"; then
+ # exporting using user supplied symfile
+ if test "x`$SED 1q $export_symbols`" != xEXPORTS; then
+ # and it's NOT already a .def file. Must figure out
+ # which of the given symbols are data symbols and tag
+ # them as such. So, trigger use of export_symbols_cmds.
+ # export_symbols gets reassigned inside the "prepare
+ # the list of exported symbols" if statement, so the
+ # include_expsyms logic still works.
+ orig_export_symbols="$export_symbols"
+ export_symbols=
+ always_export_symbols=yes
+ fi
+ fi
+ ;;
+ esac
+
+ # Prepare the list of exported symbols
+ if test -z "$export_symbols"; then
+ if test "$always_export_symbols" = yes || test -n "$export_symbols_regex"; then
+ func_verbose "generating symbol list for \`$libname.la'"
+ export_symbols="$output_objdir/$libname.exp"
+ $opt_dry_run || $RM $export_symbols
+ cmds=$export_symbols_cmds
+ save_ifs="$IFS"; IFS='~'
+ for cmd in $cmds; do
+ IFS="$save_ifs"
+ eval cmd=\"$cmd\"
+ func_len " $cmd"
+ len=$func_len_result
+ if test "$len" -lt "$max_cmd_len" || test "$max_cmd_len" -le -1; then
+ func_show_eval "$cmd" 'exit $?'
+ skipped_export=false
+ else
+ # The command line is too long to execute in one step.
+ func_verbose "using reloadable object file for export list..."
+ skipped_export=:
+ # Break out early, otherwise skipped_export may be
+ # set to false by a later but shorter cmd.
+ break
+ fi
+ done
+ IFS="$save_ifs"
+ if test -n "$export_symbols_regex" && test "X$skipped_export" != "X:"; then
+ func_show_eval '$EGREP -e "$export_symbols_regex" "$export_symbols" > "${export_symbols}T"'
+ func_show_eval '$MV "${export_symbols}T" "$export_symbols"'
+ fi
+ fi
+ fi
+
+ if test -n "$export_symbols" && test -n "$include_expsyms"; then
+ tmp_export_symbols="$export_symbols"
+ test -n "$orig_export_symbols" && tmp_export_symbols="$orig_export_symbols"
+ $opt_dry_run || eval '$ECHO "X$include_expsyms" | $Xsed | $SP2NL >> "$tmp_export_symbols"'
+ fi
+
+ if test "X$skipped_export" != "X:" && test -n "$orig_export_symbols"; then
+ # The given exports_symbols file has to be filtered, so filter it.
+ func_verbose "filter symbol list for \`$libname.la' to tag DATA exports"
+ # FIXME: $output_objdir/$libname.filter potentially contains lots of
+ # 's' commands which not all seds can handle. GNU sed should be fine
+ # though. Also, the filter scales superlinearly with the number of
+ # global variables. join(1) would be nice here, but unfortunately
+ # isn't a blessed tool.
+ $opt_dry_run || $SED -e '/[ ,]DATA/!d;s,\(.*\)\([ \,].*\),s|^\1$|\1\2|,' < $export_symbols > $output_objdir/$libname.filter
+ delfiles="$delfiles $export_symbols $output_objdir/$libname.filter"
+ export_symbols=$output_objdir/$libname.def
+ $opt_dry_run || $SED -f $output_objdir/$libname.filter < $orig_export_symbols > $export_symbols
+ fi
+
+ tmp_deplibs=
+ for test_deplib in $deplibs; do
+ case " $convenience " in
+ *" $test_deplib "*) ;;
+ *)
+ tmp_deplibs="$tmp_deplibs $test_deplib"
+ ;;
+ esac
+ done
+ deplibs="$tmp_deplibs"
+
+ if test -n "$convenience"; then
+ if test -n "$whole_archive_flag_spec" &&
+ test "$compiler_needs_object" = yes &&
+ test -z "$libobjs"; then
+ # extract the archives, so we have objects to list.
+ # TODO: could optimize this to just extract one archive.
+ whole_archive_flag_spec=
+ fi
+ if test -n "$whole_archive_flag_spec"; then
+ save_libobjs=$libobjs
+ eval libobjs=\"\$libobjs $whole_archive_flag_spec\"
+ test "X$libobjs" = "X " && libobjs=
+ else
+ gentop="$output_objdir/${outputname}x"
+ generated="$generated $gentop"
+
+ func_extract_archives $gentop $convenience
+ libobjs="$libobjs $func_extract_archives_result"
+ test "X$libobjs" = "X " && libobjs=
+ fi
+ fi
+
+ if test "$thread_safe" = yes && test -n "$thread_safe_flag_spec"; then
+ eval flag=\"$thread_safe_flag_spec\"
+ linker_flags="$linker_flags $flag"
+ fi
+
+ # Make a backup of the uninstalled library when relinking
+ if test "$mode" = relink; then
+ $opt_dry_run || eval '(cd $output_objdir && $RM ${realname}U && $MV $realname ${realname}U)' || exit $?
+ fi
+
+ # Do each of the archive commands.
+ if test "$module" = yes && test -n "$module_cmds" ; then
+ if test -n "$export_symbols" && test -n "$module_expsym_cmds"; then
+ eval test_cmds=\"$module_expsym_cmds\"
+ cmds=$module_expsym_cmds
+ else
+ eval test_cmds=\"$module_cmds\"
+ cmds=$module_cmds
+ fi
+ else
+ if test -n "$export_symbols" && test -n "$archive_expsym_cmds"; then
+ eval test_cmds=\"$archive_expsym_cmds\"
+ cmds=$archive_expsym_cmds
+ else
+ eval test_cmds=\"$archive_cmds\"
+ cmds=$archive_cmds
+ fi
+ fi
+
+ if test "X$skipped_export" != "X:" &&
+ func_len " $test_cmds" &&
+ len=$func_len_result &&
+ test "$len" -lt "$max_cmd_len" || test "$max_cmd_len" -le -1; then
+ :
+ else
+ # The command line is too long to link in one step, link piecewise
+ # or, if using GNU ld and skipped_export is not :, use a linker
+ # script.
+
+ # Save the value of $output and $libobjs because we want to
+ # use them later. If we have whole_archive_flag_spec, we
+ # want to use save_libobjs as it was before
+ # whole_archive_flag_spec was expanded, because we can't
+ # assume the linker understands whole_archive_flag_spec.
+ # This may have to be revisited, in case too many
+ # convenience libraries get linked in and end up exceeding
+ # the spec.
+ if test -z "$convenience" || test -z "$whole_archive_flag_spec"; then
+ save_libobjs=$libobjs
+ fi
+ save_output=$output
+ output_la=`$ECHO "X$output" | $Xsed -e "$basename"`
+
+ # Clear the reloadable object creation command queue and
+ # initialize k to one.
+ test_cmds=
+ concat_cmds=
+ objlist=
+ last_robj=
+ k=1
+
+ if test -n "$save_libobjs" && test "X$skipped_export" != "X:" && test "$with_gnu_ld" = yes; then
+ output=${output_objdir}/${output_la}.lnkscript
+ func_verbose "creating GNU ld script: $output"
+ $ECHO 'INPUT (' > $output
+ for obj in $save_libobjs
+ do
+ $ECHO "$obj" >> $output
+ done
+ $ECHO ')' >> $output
+ delfiles="$delfiles $output"
+ elif test -n "$save_libobjs" && test "X$skipped_export" != "X:" && test "X$file_list_spec" != X; then
+ output=${output_objdir}/${output_la}.lnk
+ func_verbose "creating linker input file list: $output"
+ : > $output
+ set x $save_libobjs
+ shift
+ firstobj=
+ if test "$compiler_needs_object" = yes; then
+ firstobj="$1 "
+ shift
+ fi
+ for obj
+ do
+ $ECHO "$obj" >> $output
+ done
+ delfiles="$delfiles $output"
+ output=$firstobj\"$file_list_spec$output\"
+ else
+ if test -n "$save_libobjs"; then
+ func_verbose "creating reloadable object files..."
+ output=$output_objdir/$output_la-${k}.$objext
+ eval test_cmds=\"$reload_cmds\"
+ func_len " $test_cmds"
+ len0=$func_len_result
+ len=$len0
+
+ # Loop over the list of objects to be linked.
+ for obj in $save_libobjs
+ do
+ func_len " $obj"
+ func_arith $len + $func_len_result
+ len=$func_arith_result
+ if test "X$objlist" = X ||
+ test "$len" -lt "$max_cmd_len"; then
+ func_append objlist " $obj"
+ else
+ # The command $test_cmds is almost too long, add a
+ # command to the queue.
+ if test "$k" -eq 1 ; then
+ # The first file doesn't have a previous command to add.
+ eval concat_cmds=\"$reload_cmds $objlist $last_robj\"
+ else
+ # All subsequent reloadable object files will link in
+ # the last one created.
+ eval concat_cmds=\"\$concat_cmds~$reload_cmds $objlist $last_robj~\$RM $last_robj\"
+ fi
+ last_robj=$output_objdir/$output_la-${k}.$objext
+ func_arith $k + 1
+ k=$func_arith_result
+ output=$output_objdir/$output_la-${k}.$objext
+ objlist=$obj
+ func_len " $last_robj"
+ func_arith $len0 + $func_len_result
+ len=$func_arith_result
+ fi
+ done
+ # Handle the remaining objects by creating one last
+ # reloadable object file. All subsequent reloadable object
+ # files will link in the last one created.
+ test -z "$concat_cmds" || concat_cmds=$concat_cmds~
+ eval concat_cmds=\"\${concat_cmds}$reload_cmds $objlist $last_robj\"
+ if test -n "$last_robj"; then
+ eval concat_cmds=\"\${concat_cmds}~\$RM $last_robj\"
+ fi
+ delfiles="$delfiles $output"
+
+ else
+ output=
+ fi
+
+ if ${skipped_export-false}; then
+ func_verbose "generating symbol list for \`$libname.la'"
+ export_symbols="$output_objdir/$libname.exp"
+ $opt_dry_run || $RM $export_symbols
+ libobjs=$output
+ # Append the command to create the export file.
+ test -z "$concat_cmds" || concat_cmds=$concat_cmds~
+ eval concat_cmds=\"\$concat_cmds$export_symbols_cmds\"
+ if test -n "$last_robj"; then
+ eval concat_cmds=\"\$concat_cmds~\$RM $last_robj\"
+ fi
+ fi
+
+ test -n "$save_libobjs" &&
+ func_verbose "creating a temporary reloadable object file: $output"
+
+ # Loop through the commands generated above and execute them.
+ save_ifs="$IFS"; IFS='~'
+ for cmd in $concat_cmds; do
+ IFS="$save_ifs"
+ $opt_silent || {
+ func_quote_for_expand "$cmd"
+ eval "func_echo $func_quote_for_expand_result"
+ }
+ $opt_dry_run || eval "$cmd" || {
+ lt_exit=$?
+
+ # Restore the uninstalled library and exit
+ if test "$mode" = relink; then
+ ( cd "$output_objdir" && \
+ $RM "${realname}T" && \
+ $MV "${realname}U" "$realname" )
+ fi
+
+ exit $lt_exit
+ }
+ done
+ IFS="$save_ifs"
+
+ if test -n "$export_symbols_regex" && ${skipped_export-false}; then
+ func_show_eval '$EGREP -e "$export_symbols_regex" "$export_symbols" > "${export_symbols}T"'
+ func_show_eval '$MV "${export_symbols}T" "$export_symbols"'
+ fi
+ fi
+
+ if ${skipped_export-false}; then
+ if test -n "$export_symbols" && test -n "$include_expsyms"; then
+ tmp_export_symbols="$export_symbols"
+ test -n "$orig_export_symbols" && tmp_export_symbols="$orig_export_symbols"
+ $opt_dry_run || eval '$ECHO "X$include_expsyms" | $Xsed | $SP2NL >> "$tmp_export_symbols"'
+ fi
+
+ if test -n "$orig_export_symbols"; then
+ # The given exports_symbols file has to be filtered, so filter it.
+ func_verbose "filter symbol list for \`$libname.la' to tag DATA exports"
+ # FIXME: $output_objdir/$libname.filter potentially contains lots of
+ # 's' commands which not all seds can handle. GNU sed should be fine
+ # though. Also, the filter scales superlinearly with the number of
+ # global variables. join(1) would be nice here, but unfortunately
+ # isn't a blessed tool.
+ $opt_dry_run || $SED -e '/[ ,]DATA/!d;s,\(.*\)\([ \,].*\),s|^\1$|\1\2|,' < $export_symbols > $output_objdir/$libname.filter
+ delfiles="$delfiles $export_symbols $output_objdir/$libname.filter"
+ export_symbols=$output_objdir/$libname.def
+ $opt_dry_run || $SED -f $output_objdir/$libname.filter < $orig_export_symbols > $export_symbols
+ fi
+ fi
+
+ libobjs=$output
+ # Restore the value of output.
+ output=$save_output
+
+ if test -n "$convenience" && test -n "$whole_archive_flag_spec"; then
+ eval libobjs=\"\$libobjs $whole_archive_flag_spec\"
+ test "X$libobjs" = "X " && libobjs=
+ fi
+ # Expand the library linking commands again to reset the
+ # value of $libobjs for piecewise linking.
+
+ # Do each of the archive commands.
+ if test "$module" = yes && test -n "$module_cmds" ; then
+ if test -n "$export_symbols" && test -n "$module_expsym_cmds"; then
+ cmds=$module_expsym_cmds
+ else
+ cmds=$module_cmds
+ fi
+ else
+ if test -n "$export_symbols" && test -n "$archive_expsym_cmds"; then
+ cmds=$archive_expsym_cmds
+ else
+ cmds=$archive_cmds
+ fi
+ fi
+ fi
+
+ if test -n "$delfiles"; then
+ # Append the command to remove temporary files to $cmds.
+ eval cmds=\"\$cmds~\$RM $delfiles\"
+ fi
+
+ # Add any objects from preloaded convenience libraries
+ if test -n "$dlprefiles"; then
+ gentop="$output_objdir/${outputname}x"
+ generated="$generated $gentop"
+
+ func_extract_archives $gentop $dlprefiles
+ libobjs="$libobjs $func_extract_archives_result"
+ test "X$libobjs" = "X " && libobjs=
+ fi
+
+ save_ifs="$IFS"; IFS='~'
+ for cmd in $cmds; do
+ IFS="$save_ifs"
+ eval cmd=\"$cmd\"
+ $opt_silent || {
+ func_quote_for_expand "$cmd"
+ eval "func_echo $func_quote_for_expand_result"
+ }
+ $opt_dry_run || eval "$cmd" || {
+ lt_exit=$?
+
+ # Restore the uninstalled library and exit
+ if test "$mode" = relink; then
+ ( cd "$output_objdir" && \
+ $RM "${realname}T" && \
+ $MV "${realname}U" "$realname" )
+ fi
+
+ exit $lt_exit
+ }
+ done
+ IFS="$save_ifs"
+
+ # Restore the uninstalled library and exit
+ if test "$mode" = relink; then
+ $opt_dry_run || eval '(cd $output_objdir && $RM ${realname}T && $MV $realname ${realname}T && $MV ${realname}U $realname)' || exit $?
+
+ if test -n "$convenience"; then
+ if test -z "$whole_archive_flag_spec"; then
+ func_show_eval '${RM}r "$gentop"'
+ fi
+ fi
+
+ exit $EXIT_SUCCESS
+ fi
+
+ # Create links to the real library.
+ for linkname in $linknames; do
+ if test "$realname" != "$linkname"; then
+ func_show_eval '(cd "$output_objdir" && $RM "$linkname" && $LN_S "$realname" "$linkname")' 'exit $?'
+ fi
+ done
+
+ # If -module or -export-dynamic was specified, set the dlname.
+ if test "$module" = yes || test "$export_dynamic" = yes; then
+ # On all known operating systems, these are identical.
+ dlname="$soname"
+ fi
+ fi
+ ;;
+
+ obj)
+ if test -n "$dlfiles$dlprefiles" || test "$dlself" != no; then
+ func_warning "\`-dlopen' is ignored for objects"
+ fi
+
+ case " $deplibs" in
+ *\ -l* | *\ -L*)
+ func_warning "\`-l' and \`-L' are ignored for objects" ;;
+ esac
+
+ test -n "$rpath" && \
+ func_warning "\`-rpath' is ignored for objects"
+
+ test -n "$xrpath" && \
+ func_warning "\`-R' is ignored for objects"
+
+ test -n "$vinfo" && \
+ func_warning "\`-version-info' is ignored for objects"
+
+ test -n "$release" && \
+ func_warning "\`-release' is ignored for objects"
+
+ case $output in
+ *.lo)
+ test -n "$objs$old_deplibs" && \
+ func_fatal_error "cannot build library object \`$output' from non-libtool objects"
+
+ libobj=$output
+ func_lo2o "$libobj"
+ obj=$func_lo2o_result
+ ;;
+ *)
+ libobj=
+ obj="$output"
+ ;;
+ esac
+
+ # Delete the old objects.
+ $opt_dry_run || $RM $obj $libobj
+
+ # Objects from convenience libraries. This assumes
+ # single-version convenience libraries. Whenever we create
+ # different ones for PIC/non-PIC, this we'll have to duplicate
+ # the extraction.
+ reload_conv_objs=
+ gentop=
+ # reload_cmds runs $LD directly, so let us get rid of
+ # -Wl from whole_archive_flag_spec and hope we can get by with
+ # turning comma into space..
+ wl=
+
+ if test -n "$convenience"; then
+ if test -n "$whole_archive_flag_spec"; then
+ eval tmp_whole_archive_flags=\"$whole_archive_flag_spec\"
+ reload_conv_objs=$reload_objs\ `$ECHO "X$tmp_whole_archive_flags" | $Xsed -e 's|,| |g'`
+ else
+ gentop="$output_objdir/${obj}x"
+ generated="$generated $gentop"
+
+ func_extract_archives $gentop $convenience
+ reload_conv_objs="$reload_objs $func_extract_archives_result"
+ fi
+ fi
+
+ # Create the old-style object.
+ reload_objs="$objs$old_deplibs "`$ECHO "X$libobjs" | $SP2NL | $Xsed -e '/\.'${libext}$'/d' -e '/\.lib$/d' -e "$lo2o" | $NL2SP`" $reload_conv_objs" ### testsuite: skip nested quoting test
+
+ output="$obj"
+ func_execute_cmds "$reload_cmds" 'exit $?'
+
+ # Exit if we aren't doing a library object file.
+ if test -z "$libobj"; then
+ if test -n "$gentop"; then
+ func_show_eval '${RM}r "$gentop"'
+ fi
+
+ exit $EXIT_SUCCESS
+ fi
+
+ if test "$build_libtool_libs" != yes; then
+ if test -n "$gentop"; then
+ func_show_eval '${RM}r "$gentop"'
+ fi
+
+ # Create an invalid libtool object if no PIC, so that we don't
+ # accidentally link it into a program.
+ # $show "echo timestamp > $libobj"
+ # $opt_dry_run || eval "echo timestamp > $libobj" || exit $?
+ exit $EXIT_SUCCESS
+ fi
+
+ if test -n "$pic_flag" || test "$pic_mode" != default; then
+ # Only do commands if we really have different PIC objects.
+ reload_objs="$libobjs $reload_conv_objs"
+ output="$libobj"
+ func_execute_cmds "$reload_cmds" 'exit $?'
+ fi
+
+ if test -n "$gentop"; then
+ func_show_eval '${RM}r "$gentop"'
+ fi
+
+ exit $EXIT_SUCCESS
+ ;;
+
+ prog)
+ case $host in
+ *cygwin*) func_stripname '' '.exe' "$output"
+ output=$func_stripname_result.exe;;
+ esac
+ test -n "$vinfo" && \
+ func_warning "\`-version-info' is ignored for programs"
+
+ test -n "$release" && \
+ func_warning "\`-release' is ignored for programs"
+
+ test "$preload" = yes \
+ && test "$dlopen_support" = unknown \
+ && test "$dlopen_self" = unknown \
+ && test "$dlopen_self_static" = unknown && \
+ func_warning "\`LT_INIT([dlopen])' not used. Assuming no dlopen support."
+
+ case $host in
+ *-*-rhapsody* | *-*-darwin1.[012])
+ # On Rhapsody replace the C library is the System framework
+ compile_deplibs=`$ECHO "X $compile_deplibs" | $Xsed -e 's/ -lc / System.ltframework /'`
+ finalize_deplibs=`$ECHO "X $finalize_deplibs" | $Xsed -e 's/ -lc / System.ltframework /'`
+ ;;
+ esac
+
+ case $host in
+ *-*-darwin*)
+ # Don't allow lazy linking, it breaks C++ global constructors
+ # But is supposedly fixed on 10.4 or later (yay!).
+ if test "$tagname" = CXX ; then
+ case ${MACOSX_DEPLOYMENT_TARGET-10.0} in
+ 10.[0123])
+ compile_command="$compile_command ${wl}-bind_at_load"
+ finalize_command="$finalize_command ${wl}-bind_at_load"
+ ;;
+ esac
+ fi
+ # Time to change all our "foo.ltframework" stuff back to "-framework foo"
+ compile_deplibs=`$ECHO "X $compile_deplibs" | $Xsed -e 's% \([^ $]*\).ltframework% -framework \1%g'`
+ finalize_deplibs=`$ECHO "X $finalize_deplibs" | $Xsed -e 's% \([^ $]*\).ltframework% -framework \1%g'`
+ ;;
+ esac
+
+
+ # move library search paths that coincide with paths to not yet
+ # installed libraries to the beginning of the library search list
+ new_libs=
+ for path in $notinst_path; do
+ case " $new_libs " in
+ *" -L$path/$objdir "*) ;;
+ *)
+ case " $compile_deplibs " in
+ *" -L$path/$objdir "*)
+ new_libs="$new_libs -L$path/$objdir" ;;
+ esac
+ ;;
+ esac
+ done
+ for deplib in $compile_deplibs; do
+ case $deplib in
+ -L*)
+ case " $new_libs " in
+ *" $deplib "*) ;;
+ *) new_libs="$new_libs $deplib" ;;
+ esac
+ ;;
+ *) new_libs="$new_libs $deplib" ;;
+ esac
+ done
+ compile_deplibs="$new_libs"
+
+
+ compile_command="$compile_command $compile_deplibs"
+ finalize_command="$finalize_command $finalize_deplibs"
+
+ if test -n "$rpath$xrpath"; then
+ # If the user specified any rpath flags, then add them.
+ for libdir in $rpath $xrpath; do
+ # This is the magic to use -rpath.
+ case "$finalize_rpath " in
+ *" $libdir "*) ;;
+ *) finalize_rpath="$finalize_rpath $libdir" ;;
+ esac
+ done
+ fi
+
+ # Now hardcode the library paths
+ rpath=
+ hardcode_libdirs=
+ for libdir in $compile_rpath $finalize_rpath; do
+ if test -n "$hardcode_libdir_flag_spec"; then
+ if test -n "$hardcode_libdir_separator"; then
+ if test -z "$hardcode_libdirs"; then
+ hardcode_libdirs="$libdir"
+ else
+ # Just accumulate the unique libdirs.
+ case $hardcode_libdir_separator$hardcode_libdirs$hardcode_libdir_separator in
+ *"$hardcode_libdir_separator$libdir$hardcode_libdir_separator"*)
+ ;;
+ *)
+ hardcode_libdirs="$hardcode_libdirs$hardcode_libdir_separator$libdir"
+ ;;
+ esac
+ fi
+ else
+ eval flag=\"$hardcode_libdir_flag_spec\"
+ rpath="$rpath $flag"
+ fi
+ elif test -n "$runpath_var"; then
+ case "$perm_rpath " in
+ *" $libdir "*) ;;
+ *) perm_rpath="$perm_rpath $libdir" ;;
+ esac
+ fi
+ case $host in
+ *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-cegcc*)
+ testbindir=`${ECHO} "$libdir" | ${SED} -e 's*/lib$*/bin*'`
+ case :$dllsearchpath: in
+ *":$libdir:"*) ;;
+ ::) dllsearchpath=$libdir;;
+ *) dllsearchpath="$dllsearchpath:$libdir";;
+ esac
+ case :$dllsearchpath: in
+ *":$testbindir:"*) ;;
+ ::) dllsearchpath=$testbindir;;
+ *) dllsearchpath="$dllsearchpath:$testbindir";;
+ esac
+ ;;
+ esac
+ done
+ # Substitute the hardcoded libdirs into the rpath.
+ if test -n "$hardcode_libdir_separator" &&
+ test -n "$hardcode_libdirs"; then
+ libdir="$hardcode_libdirs"
+ eval rpath=\" $hardcode_libdir_flag_spec\"
+ fi
+ compile_rpath="$rpath"
+
+ rpath=
+ hardcode_libdirs=
+ for libdir in $finalize_rpath; do
+ if test -n "$hardcode_libdir_flag_spec"; then
+ if test -n "$hardcode_libdir_separator"; then
+ if test -z "$hardcode_libdirs"; then
+ hardcode_libdirs="$libdir"
+ else
+ # Just accumulate the unique libdirs.
+ case $hardcode_libdir_separator$hardcode_libdirs$hardcode_libdir_separator in
+ *"$hardcode_libdir_separator$libdir$hardcode_libdir_separator"*)
+ ;;
+ *)
+ hardcode_libdirs="$hardcode_libdirs$hardcode_libdir_separator$libdir"
+ ;;
+ esac
+ fi
+ else
+ eval flag=\"$hardcode_libdir_flag_spec\"
+ rpath="$rpath $flag"
+ fi
+ elif test -n "$runpath_var"; then
+ case "$finalize_perm_rpath " in
+ *" $libdir "*) ;;
+ *) finalize_perm_rpath="$finalize_perm_rpath $libdir" ;;
+ esac
+ fi
+ done
+ # Substitute the hardcoded libdirs into the rpath.
+ if test -n "$hardcode_libdir_separator" &&
+ test -n "$hardcode_libdirs"; then
+ libdir="$hardcode_libdirs"
+ eval rpath=\" $hardcode_libdir_flag_spec\"
+ fi
+ finalize_rpath="$rpath"
+
+ if test -n "$libobjs" && test "$build_old_libs" = yes; then
+ # Transform all the library objects into standard objects.
+ compile_command=`$ECHO "X$compile_command" | $SP2NL | $Xsed -e "$lo2o" | $NL2SP`
+ finalize_command=`$ECHO "X$finalize_command" | $SP2NL | $Xsed -e "$lo2o" | $NL2SP`
+ fi
+
+ func_generate_dlsyms "$outputname" "@PROGRAM@" "no"
+
+ # template prelinking step
+ if test -n "$prelink_cmds"; then
+ func_execute_cmds "$prelink_cmds" 'exit $?'
+ fi
+
+ wrappers_required=yes
+ case $host in
+ *cygwin* | *mingw* )
+ if test "$build_libtool_libs" != yes; then
+ wrappers_required=no
+ fi
+ ;;
+ *cegcc)
+ # Disable wrappers for cegcc, we are cross compiling anyway.
+ wrappers_required=no
+ ;;
+ *)
+ if test "$need_relink" = no || test "$build_libtool_libs" != yes; then
+ wrappers_required=no
+ fi
+ ;;
+ esac
+ if test "$wrappers_required" = no; then
+ # Replace the output file specification.
+ compile_command=`$ECHO "X$compile_command" | $Xsed -e 's%@OUTPUT@%'"$output"'%g'`
+ link_command="$compile_command$compile_rpath"
+
+ # We have no uninstalled library dependencies, so finalize right now.
+ exit_status=0
+ func_show_eval "$link_command" 'exit_status=$?'
+
+ # Delete the generated files.
+ if test -f "$output_objdir/${outputname}S.${objext}"; then
+ func_show_eval '$RM "$output_objdir/${outputname}S.${objext}"'
+ fi
+
+ exit $exit_status
+ fi
+
+ if test -n "$compile_shlibpath$finalize_shlibpath"; then
+ compile_command="$shlibpath_var=\"$compile_shlibpath$finalize_shlibpath\$$shlibpath_var\" $compile_command"
+ fi
+ if test -n "$finalize_shlibpath"; then
+ finalize_command="$shlibpath_var=\"$finalize_shlibpath\$$shlibpath_var\" $finalize_command"
+ fi
+
+ compile_var=
+ finalize_var=
+ if test -n "$runpath_var"; then
+ if test -n "$perm_rpath"; then
+ # We should set the runpath_var.
+ rpath=
+ for dir in $perm_rpath; do
+ rpath="$rpath$dir:"
+ done
+ compile_var="$runpath_var=\"$rpath\$$runpath_var\" "
+ fi
+ if test -n "$finalize_perm_rpath"; then
+ # We should set the runpath_var.
+ rpath=
+ for dir in $finalize_perm_rpath; do
+ rpath="$rpath$dir:"
+ done
+ finalize_var="$runpath_var=\"$rpath\$$runpath_var\" "
+ fi
+ fi
+
+ if test "$no_install" = yes; then
+ # We don't need to create a wrapper script.
+ link_command="$compile_var$compile_command$compile_rpath"
+ # Replace the output file specification.
+ link_command=`$ECHO "X$link_command" | $Xsed -e 's%@OUTPUT@%'"$output"'%g'`
+ # Delete the old output file.
+ $opt_dry_run || $RM $output
+ # Link the executable and exit
+ func_show_eval "$link_command" 'exit $?'
+ exit $EXIT_SUCCESS
+ fi
+
+ if test "$hardcode_action" = relink; then
+ # Fast installation is not supported
+ link_command="$compile_var$compile_command$compile_rpath"
+ relink_command="$finalize_var$finalize_command$finalize_rpath"
+
+ func_warning "this platform does not like uninstalled shared libraries"
+ func_warning "\`$output' will be relinked during installation"
+ else
+ if test "$fast_install" != no; then
+ link_command="$finalize_var$compile_command$finalize_rpath"
+ if test "$fast_install" = yes; then
+ relink_command=`$ECHO "X$compile_var$compile_command$compile_rpath" | $Xsed -e 's%@OUTPUT@%\$progdir/\$file%g'`
+ else
+ # fast_install is set to needless
+ relink_command=
+ fi
+ else
+ link_command="$compile_var$compile_command$compile_rpath"
+ relink_command="$finalize_var$finalize_command$finalize_rpath"
+ fi
+ fi
+
+ # Replace the output file specification.
+ link_command=`$ECHO "X$link_command" | $Xsed -e 's%@OUTPUT@%'"$output_objdir/$outputname"'%g'`
+
+ # Delete the old output files.
+ $opt_dry_run || $RM $output $output_objdir/$outputname $output_objdir/lt-$outputname
+
+ func_show_eval "$link_command" 'exit $?'
+
+ # Now create the wrapper script.
+ func_verbose "creating $output"
+
+ # Quote the relink command for shipping.
+ if test -n "$relink_command"; then
+ # Preserve any variables that may affect compiler behavior
+ for var in $variables_saved_for_relink; do
+ if eval test -z \"\${$var+set}\"; then
+ relink_command="{ test -z \"\${$var+set}\" || $lt_unset $var || { $var=; export $var; }; }; $relink_command"
+ elif eval var_value=\$$var; test -z "$var_value"; then
+ relink_command="$var=; export $var; $relink_command"
+ else
+ func_quote_for_eval "$var_value"
+ relink_command="$var=$func_quote_for_eval_result; export $var; $relink_command"
+ fi
+ done
+ relink_command="(cd `pwd`; $relink_command)"
+ relink_command=`$ECHO "X$relink_command" | $Xsed -e "$sed_quote_subst"`
+ fi
+
+ # Quote $ECHO for shipping.
+ if test "X$ECHO" = "X$SHELL $progpath --fallback-echo"; then
+ case $progpath in
+ [\\/]* | [A-Za-z]:[\\/]*) qecho="$SHELL $progpath --fallback-echo";;
+ *) qecho="$SHELL `pwd`/$progpath --fallback-echo";;
+ esac
+ qecho=`$ECHO "X$qecho" | $Xsed -e "$sed_quote_subst"`
+ else
+ qecho=`$ECHO "X$ECHO" | $Xsed -e "$sed_quote_subst"`
+ fi
+
+ # Only actually do things if not in dry run mode.
+ $opt_dry_run || {
+ # win32 will think the script is a binary if it has
+ # a .exe suffix, so we strip it off here.
+ case $output in
+ *.exe) func_stripname '' '.exe' "$output"
+ output=$func_stripname_result ;;
+ esac
+ # test for cygwin because mv fails w/o .exe extensions
+ case $host in
+ *cygwin*)
+ exeext=.exe
+ func_stripname '' '.exe' "$outputname"
+ outputname=$func_stripname_result ;;
+ *) exeext= ;;
+ esac
+ case $host in
+ *cygwin* | *mingw* )
+ func_dirname_and_basename "$output" "" "."
+ output_name=$func_basename_result
+ output_path=$func_dirname_result
+ cwrappersource="$output_path/$objdir/lt-$output_name.c"
+ cwrapper="$output_path/$output_name.exe"
+ $RM $cwrappersource $cwrapper
+ trap "$RM $cwrappersource $cwrapper; exit $EXIT_FAILURE" 1 2 15
+
+ func_emit_cwrapperexe_src > $cwrappersource
+
+ # The wrapper executable is built using the $host compiler,
+ # because it contains $host paths and files. If cross-
+ # compiling, it, like the target executable, must be
+ # executed on the $host or under an emulation environment.
+ $opt_dry_run || {
+ $LTCC $LTCFLAGS -o $cwrapper $cwrappersource
+ $STRIP $cwrapper
+ }
+
+ # Now, create the wrapper script for func_source use:
+ func_ltwrapper_scriptname $cwrapper
+ $RM $func_ltwrapper_scriptname_result
+ trap "$RM $func_ltwrapper_scriptname_result; exit $EXIT_FAILURE" 1 2 15
+ $opt_dry_run || {
+ # note: this script will not be executed, so do not chmod.
+ if test "x$build" = "x$host" ; then
+ $cwrapper --lt-dump-script > $func_ltwrapper_scriptname_result
+ else
+ func_emit_wrapper no > $func_ltwrapper_scriptname_result
+ fi
+ }
+ ;;
+ * )
+ $RM $output
+ trap "$RM $output; exit $EXIT_FAILURE" 1 2 15
+
+ func_emit_wrapper no > $output
+ chmod +x $output
+ ;;
+ esac
+ }
+ exit $EXIT_SUCCESS
+ ;;
+ esac
+
+ # See if we need to build an old-fashioned archive.
+ for oldlib in $oldlibs; do
+
+ if test "$build_libtool_libs" = convenience; then
+ oldobjs="$libobjs_save $symfileobj"
+ addlibs="$convenience"
+ build_libtool_libs=no
+ else
+ if test "$build_libtool_libs" = module; then
+ oldobjs="$libobjs_save"
+ build_libtool_libs=no
+ else
+ oldobjs="$old_deplibs $non_pic_objects"
+ if test "$preload" = yes && test -f "$symfileobj"; then
+ oldobjs="$oldobjs $symfileobj"
+ fi
+ fi
+ addlibs="$old_convenience"
+ fi
+
+ if test -n "$addlibs"; then
+ gentop="$output_objdir/${outputname}x"
+ generated="$generated $gentop"
+
+ func_extract_archives $gentop $addlibs
+ oldobjs="$oldobjs $func_extract_archives_result"
+ fi
+
+ # Do each command in the archive commands.
+ if test -n "$old_archive_from_new_cmds" && test "$build_libtool_libs" = yes; then
+ cmds=$old_archive_from_new_cmds
+ else
+
+ # Add any objects from preloaded convenience libraries
+ if test -n "$dlprefiles"; then
+ gentop="$output_objdir/${outputname}x"
+ generated="$generated $gentop"
+
+ func_extract_archives $gentop $dlprefiles
+ oldobjs="$oldobjs $func_extract_archives_result"
+ fi
+
+ # POSIX demands no paths to be encoded in archives. We have
+ # to avoid creating archives with duplicate basenames if we
+ # might have to extract them afterwards, e.g., when creating a
+ # static archive out of a convenience library, or when linking
+ # the entirety of a libtool archive into another (currently
+ # not supported by libtool).
+ if (for obj in $oldobjs
+ do
+ func_basename "$obj"
+ $ECHO "$func_basename_result"
+ done | sort | sort -uc >/dev/null 2>&1); then
+ :
+ else
+ $ECHO "copying selected object files to avoid basename conflicts..."
+ gentop="$output_objdir/${outputname}x"
+ generated="$generated $gentop"
+ func_mkdir_p "$gentop"
+ save_oldobjs=$oldobjs
+ oldobjs=
+ counter=1
+ for obj in $save_oldobjs
+ do
+ func_basename "$obj"
+ objbase="$func_basename_result"
+ case " $oldobjs " in
+ " ") oldobjs=$obj ;;
+ *[\ /]"$objbase "*)
+ while :; do
+ # Make sure we don't pick an alternate name that also
+ # overlaps.
+ newobj=lt$counter-$objbase
+ func_arith $counter + 1
+ counter=$func_arith_result
+ case " $oldobjs " in
+ *[\ /]"$newobj "*) ;;
+ *) if test ! -f "$gentop/$newobj"; then break; fi ;;
+ esac
+ done
+ func_show_eval "ln $obj $gentop/$newobj || cp $obj $gentop/$newobj"
+ oldobjs="$oldobjs $gentop/$newobj"
+ ;;
+ *) oldobjs="$oldobjs $obj" ;;
+ esac
+ done
+ fi
+ eval cmds=\"$old_archive_cmds\"
+
+ func_len " $cmds"
+ len=$func_len_result
+ if test "$len" -lt "$max_cmd_len" || test "$max_cmd_len" -le -1; then
+ cmds=$old_archive_cmds
+ else
+ # the command line is too long to link in one step, link in parts
+ func_verbose "using piecewise archive linking..."
+ save_RANLIB=$RANLIB
+ RANLIB=:
+ objlist=
+ concat_cmds=
+ save_oldobjs=$oldobjs
+ oldobjs=
+ # Is there a better way of finding the last object in the list?
+ for obj in $save_oldobjs
+ do
+ last_oldobj=$obj
+ done
+ eval test_cmds=\"$old_archive_cmds\"
+ func_len " $test_cmds"
+ len0=$func_len_result
+ len=$len0
+ for obj in $save_oldobjs
+ do
+ func_len " $obj"
+ func_arith $len + $func_len_result
+ len=$func_arith_result
+ func_append objlist " $obj"
+ if test "$len" -lt "$max_cmd_len"; then
+ :
+ else
+ # the above command should be used before it gets too long
+ oldobjs=$objlist
+ if test "$obj" = "$last_oldobj" ; then
+ RANLIB=$save_RANLIB
+ fi
+ test -z "$concat_cmds" || concat_cmds=$concat_cmds~
+ eval concat_cmds=\"\${concat_cmds}$old_archive_cmds\"
+ objlist=
+ len=$len0
+ fi
+ done
+ RANLIB=$save_RANLIB
+ oldobjs=$objlist
+ if test "X$oldobjs" = "X" ; then
+ eval cmds=\"\$concat_cmds\"
+ else
+ eval cmds=\"\$concat_cmds~\$old_archive_cmds\"
+ fi
+ fi
+ fi
+ func_execute_cmds "$cmds" 'exit $?'
+ done
+
+ test -n "$generated" && \
+ func_show_eval "${RM}r$generated"
+
+ # Now create the libtool archive.
+ case $output in
+ *.la)
+ old_library=
+ test "$build_old_libs" = yes && old_library="$libname.$libext"
+ func_verbose "creating $output"
+
+ # Preserve any variables that may affect compiler behavior
+ for var in $variables_saved_for_relink; do
+ if eval test -z \"\${$var+set}\"; then
+ relink_command="{ test -z \"\${$var+set}\" || $lt_unset $var || { $var=; export $var; }; }; $relink_command"
+ elif eval var_value=\$$var; test -z "$var_value"; then
+ relink_command="$var=; export $var; $relink_command"
+ else
+ func_quote_for_eval "$var_value"
+ relink_command="$var=$func_quote_for_eval_result; export $var; $relink_command"
+ fi
+ done
+ # Quote the link command for shipping.
+ relink_command="(cd `pwd`; $SHELL $progpath $preserve_args --mode=relink $libtool_args @inst_prefix_dir@)"
+ relink_command=`$ECHO "X$relink_command" | $Xsed -e "$sed_quote_subst"`
+ if test "$hardcode_automatic" = yes ; then
+ relink_command=
+ fi
+
+ # Only create the output if not a dry run.
+ $opt_dry_run || {
+ for installed in no yes; do
+ if test "$installed" = yes; then
+ if test -z "$install_libdir"; then
+ break
+ fi
+ output="$output_objdir/$outputname"i
+ # Replace all uninstalled libtool libraries with the installed ones
+ newdependency_libs=
+ for deplib in $dependency_libs; do
+ case $deplib in
+ *.la)
+ func_basename "$deplib"
+ name="$func_basename_result"
+ eval libdir=`${SED} -n -e 's/^libdir=\(.*\)$/\1/p' $deplib`
+ test -z "$libdir" && \
+ func_fatal_error "\`$deplib' is not a valid libtool archive"
+ newdependency_libs="$newdependency_libs $libdir/$name"
+ ;;
+ *) newdependency_libs="$newdependency_libs $deplib" ;;
+ esac
+ done
+ dependency_libs="$newdependency_libs"
+ newdlfiles=
+
+ for lib in $dlfiles; do
+ case $lib in
+ *.la)
+ func_basename "$lib"
+ name="$func_basename_result"
+ eval libdir=`${SED} -n -e 's/^libdir=\(.*\)$/\1/p' $lib`
+ test -z "$libdir" && \
+ func_fatal_error "\`$lib' is not a valid libtool archive"
+ newdlfiles="$newdlfiles $libdir/$name"
+ ;;
+ *) newdlfiles="$newdlfiles $lib" ;;
+ esac
+ done
+ dlfiles="$newdlfiles"
+ newdlprefiles=
+ for lib in $dlprefiles; do
+ case $lib in
+ *.la)
+ # Only pass preopened files to the pseudo-archive (for
+ # eventual linking with the app. that links it) if we
+ # didn't already link the preopened objects directly into
+ # the library:
+ func_basename "$lib"
+ name="$func_basename_result"
+ eval libdir=`${SED} -n -e 's/^libdir=\(.*\)$/\1/p' $lib`
+ test -z "$libdir" && \
+ func_fatal_error "\`$lib' is not a valid libtool archive"
+ newdlprefiles="$newdlprefiles $libdir/$name"
+ ;;
+ esac
+ done
+ dlprefiles="$newdlprefiles"
+ else
+ newdlfiles=
+ for lib in $dlfiles; do
+ case $lib in
+ [\\/]* | [A-Za-z]:[\\/]*) abs="$lib" ;;
+ *) abs=`pwd`"/$lib" ;;
+ esac
+ newdlfiles="$newdlfiles $abs"
+ done
+ dlfiles="$newdlfiles"
+ newdlprefiles=
+ for lib in $dlprefiles; do
+ case $lib in
+ [\\/]* | [A-Za-z]:[\\/]*) abs="$lib" ;;
+ *) abs=`pwd`"/$lib" ;;
+ esac
+ newdlprefiles="$newdlprefiles $abs"
+ done
+ dlprefiles="$newdlprefiles"
+ fi
+ $RM $output
+ # place dlname in correct position for cygwin
+ tdlname=$dlname
+ case $host,$output,$installed,$module,$dlname in
+ *cygwin*,*lai,yes,no,*.dll | *mingw*,*lai,yes,no,*.dll | *cegcc*,*lai,yes,no,*.dll) tdlname=../bin/$dlname ;;
+ esac
+ $ECHO > $output "\
+# $outputname - a libtool library file
+# Generated by $PROGRAM (GNU $PACKAGE$TIMESTAMP) $VERSION
+#
+# Please DO NOT delete this file!
+# It is necessary for linking the library.
+
+# The name that we can dlopen(3).
+dlname='$tdlname'
+
+# Names of this library.
+library_names='$library_names'
+
+# The name of the static archive.
+old_library='$old_library'
+
+# Linker flags that can not go in dependency_libs.
+inherited_linker_flags='$new_inherited_linker_flags'
+
+# Libraries that this one depends upon.
+dependency_libs='$dependency_libs'
+
+# Names of additional weak libraries provided by this library
+weak_library_names='$weak_libs'
+
+# Version information for $libname.
+current=$current
+age=$age
+revision=$revision
+
+# Is this an already installed library?
+installed=$installed
+
+# Should we warn about portability when linking against -modules?
+shouldnotlink=$module
+
+# Files to dlopen/dlpreopen
+dlopen='$dlfiles'
+dlpreopen='$dlprefiles'
+
+# Directory that this library needs to be installed in:
+libdir='$install_libdir'"
+ if test "$installed" = no && test "$need_relink" = yes; then
+ $ECHO >> $output "\
+relink_command=\"$relink_command\""
+ fi
+ done
+ }
+
+ # Do a symbolic link so that the libtool archive can be found in
+ # LD_LIBRARY_PATH before the program is installed.
+ func_show_eval '( cd "$output_objdir" && $RM "$outputname" && $LN_S "../$outputname" "$outputname" )' 'exit $?'
+ ;;
+ esac
+ exit $EXIT_SUCCESS
+}
+
+{ test "$mode" = link || test "$mode" = relink; } &&
+ func_mode_link ${1+"$@"}
+
+
+# func_mode_uninstall arg...
+func_mode_uninstall ()
+{
+ $opt_debug
+ RM="$nonopt"
+ files=
+ rmforce=
+ exit_status=0
+
+ # This variable tells wrapper scripts just to set variables rather
+ # than running their programs.
+ libtool_install_magic="$magic"
+
+ for arg
+ do
+ case $arg in
+ -f) RM="$RM $arg"; rmforce=yes ;;
+ -*) RM="$RM $arg" ;;
+ *) files="$files $arg" ;;
+ esac
+ done
+
+ test -z "$RM" && \
+ func_fatal_help "you must specify an RM program"
+
+ rmdirs=
+
+ origobjdir="$objdir"
+ for file in $files; do
+ func_dirname "$file" "" "."
+ dir="$func_dirname_result"
+ if test "X$dir" = X.; then
+ objdir="$origobjdir"
+ else
+ objdir="$dir/$origobjdir"
+ fi
+ func_basename "$file"
+ name="$func_basename_result"
+ test "$mode" = uninstall && objdir="$dir"
+
+ # Remember objdir for removal later, being careful to avoid duplicates
+ if test "$mode" = clean; then
+ case " $rmdirs " in
+ *" $objdir "*) ;;
+ *) rmdirs="$rmdirs $objdir" ;;
+ esac
+ fi
+
+ # Don't error if the file doesn't exist and rm -f was used.
+ if { test -L "$file"; } >/dev/null 2>&1 ||
+ { test -h "$file"; } >/dev/null 2>&1 ||
+ test -f "$file"; then
+ :
+ elif test -d "$file"; then
+ exit_status=1
+ continue
+ elif test "$rmforce" = yes; then
+ continue
+ fi
+
+ rmfiles="$file"
+
+ case $name in
+ *.la)
+ # Possibly a libtool archive, so verify it.
+ if func_lalib_p "$file"; then
+ func_source $dir/$name
+
+ # Delete the libtool libraries and symlinks.
+ for n in $library_names; do
+ rmfiles="$rmfiles $objdir/$n"
+ done
+ test -n "$old_library" && rmfiles="$rmfiles $objdir/$old_library"
+
+ case "$mode" in
+ clean)
+ case " $library_names " in
+ # " " in the beginning catches empty $dlname
+ *" $dlname "*) ;;
+ *) rmfiles="$rmfiles $objdir/$dlname" ;;
+ esac
+ test -n "$libdir" && rmfiles="$rmfiles $objdir/$name $objdir/${name}i"
+ ;;
+ uninstall)
+ if test -n "$library_names"; then
+ # Do each command in the postuninstall commands.
+ func_execute_cmds "$postuninstall_cmds" 'test "$rmforce" = yes || exit_status=1'
+ fi
+
+ if test -n "$old_library"; then
+ # Do each command in the old_postuninstall commands.
+ func_execute_cmds "$old_postuninstall_cmds" 'test "$rmforce" = yes || exit_status=1'
+ fi
+ # FIXME: should reinstall the best remaining shared library.
+ ;;
+ esac
+ fi
+ ;;
+
+ *.lo)
+ # Possibly a libtool object, so verify it.
+ if func_lalib_p "$file"; then
+
+ # Read the .lo file
+ func_source $dir/$name
+
+ # Add PIC object to the list of files to remove.
+ if test -n "$pic_object" &&
+ test "$pic_object" != none; then
+ rmfiles="$rmfiles $dir/$pic_object"
+ fi
+
+ # Add non-PIC object to the list of files to remove.
+ if test -n "$non_pic_object" &&
+ test "$non_pic_object" != none; then
+ rmfiles="$rmfiles $dir/$non_pic_object"
+ fi
+ fi
+ ;;
+
+ *)
+ if test "$mode" = clean ; then
+ noexename=$name
+ case $file in
+ *.exe)
+ func_stripname '' '.exe' "$file"
+ file=$func_stripname_result
+ func_stripname '' '.exe' "$name"
+ noexename=$func_stripname_result
+ # $file with .exe has already been added to rmfiles,
+ # add $file without .exe
+ rmfiles="$rmfiles $file"
+ ;;
+ esac
+ # Do a test to see if this is a libtool program.
+ if func_ltwrapper_p "$file"; then
+ if func_ltwrapper_executable_p "$file"; then
+ func_ltwrapper_scriptname "$file"
+ relink_command=
+ func_source $func_ltwrapper_scriptname_result
+ rmfiles="$rmfiles $func_ltwrapper_scriptname_result"
+ else
+ relink_command=
+ func_source $dir/$noexename
+ fi
+
+ # note $name still contains .exe if it was in $file originally
+ # as does the version of $file that was added into $rmfiles
+ rmfiles="$rmfiles $objdir/$name $objdir/${name}S.${objext}"
+ if test "$fast_install" = yes && test -n "$relink_command"; then
+ rmfiles="$rmfiles $objdir/lt-$name"
+ fi
+ if test "X$noexename" != "X$name" ; then
+ rmfiles="$rmfiles $objdir/lt-${noexename}.c"
+ fi
+ fi
+ fi
+ ;;
+ esac
+ func_show_eval "$RM $rmfiles" 'exit_status=1'
+ done
+ objdir="$origobjdir"
+
+ # Try to remove the ${objdir}s in the directories where we deleted files
+ for dir in $rmdirs; do
+ if test -d "$dir"; then
+ func_show_eval "rmdir $dir >/dev/null 2>&1"
+ fi
+ done
+
+ exit $exit_status
+}
+
+{ test "$mode" = uninstall || test "$mode" = clean; } &&
+ func_mode_uninstall ${1+"$@"}
+
+test -z "$mode" && {
+ help="$generic_help"
+ func_fatal_help "you must specify a MODE"
+}
+
+test -z "$exec_cmd" && \
+ func_fatal_help "invalid operation mode \`$mode'"
+
+if test -n "$exec_cmd"; then
+ eval exec "$exec_cmd"
+ exit $EXIT_FAILURE
+fi
+
+exit $exit_status
+
+
+# The TAGs below are defined such that we never get into a situation
+# in which we disable both kinds of libraries. Given conflicting
+# choices, we go for a static library, that is the most portable,
+# since we can't tell whether shared libraries were disabled because
+# the user asked for that or because the platform doesn't support
+# them. This is particularly important on AIX, because we don't
+# support having both static and shared libraries enabled at the same
+# time on that platform, so we default to a shared-only configuration.
+# If a disable-shared tag is given, we'll fallback to a static-only
+# configuration. But we'll never go from static-only to shared-only.
+
+# ### BEGIN LIBTOOL TAG CONFIG: disable-shared
+build_libtool_libs=no
+build_old_libs=yes
+# ### END LIBTOOL TAG CONFIG: disable-shared
+
+# ### BEGIN LIBTOOL TAG CONFIG: disable-static
+build_old_libs=`case $build_libtool_libs in yes) echo no;; *) echo yes;; esac`
+# ### END LIBTOOL TAG CONFIG: disable-static
+
+# Local Variables:
+# mode:shell-script
+# sh-indentation:2
+# End:
+# vi:sw=2
+
diff --git a/config/ltoptions.m4 b/config/ltoptions.m4
new file mode 100644
index 0000000..34151a3
--- /dev/null
+++ b/config/ltoptions.m4
@@ -0,0 +1,368 @@
+# Helper functions for option handling. -*- Autoconf -*-
+#
+# Copyright (C) 2004, 2005, 2007, 2008 Free Software Foundation, Inc.
+# Written by Gary V. Vaughan, 2004
+#
+# This file is free software; the Free Software Foundation gives
+# unlimited permission to copy and/or distribute it, with or without
+# modifications, as long as this notice is preserved.
+
+# serial 6 ltoptions.m4
+
+# This is to help aclocal find these macros, as it can't see m4_define.
+AC_DEFUN([LTOPTIONS_VERSION], [m4_if([1])])
+
+
+# _LT_MANGLE_OPTION(MACRO-NAME, OPTION-NAME)
+# ------------------------------------------
+m4_define([_LT_MANGLE_OPTION],
+[[_LT_OPTION_]m4_bpatsubst($1__$2, [[^a-zA-Z0-9_]], [_])])
+
+
+# _LT_SET_OPTION(MACRO-NAME, OPTION-NAME)
+# ---------------------------------------
+# Set option OPTION-NAME for macro MACRO-NAME, and if there is a
+# matching handler defined, dispatch to it. Other OPTION-NAMEs are
+# saved as a flag.
+m4_define([_LT_SET_OPTION],
+[m4_define(_LT_MANGLE_OPTION([$1], [$2]))dnl
+m4_ifdef(_LT_MANGLE_DEFUN([$1], [$2]),
+ _LT_MANGLE_DEFUN([$1], [$2]),
+ [m4_warning([Unknown $1 option `$2'])])[]dnl
+])
+
+
+# _LT_IF_OPTION(MACRO-NAME, OPTION-NAME, IF-SET, [IF-NOT-SET])
+# ------------------------------------------------------------
+# Execute IF-SET if OPTION is set, IF-NOT-SET otherwise.
+m4_define([_LT_IF_OPTION],
+[m4_ifdef(_LT_MANGLE_OPTION([$1], [$2]), [$3], [$4])])
+
+
+# _LT_UNLESS_OPTIONS(MACRO-NAME, OPTION-LIST, IF-NOT-SET)
+# -------------------------------------------------------
+# Execute IF-NOT-SET unless all options in OPTION-LIST for MACRO-NAME
+# are set.
+m4_define([_LT_UNLESS_OPTIONS],
+[m4_foreach([_LT_Option], m4_split(m4_normalize([$2])),
+ [m4_ifdef(_LT_MANGLE_OPTION([$1], _LT_Option),
+ [m4_define([$0_found])])])[]dnl
+m4_ifdef([$0_found], [m4_undefine([$0_found])], [$3
+])[]dnl
+])
+
+
+# _LT_SET_OPTIONS(MACRO-NAME, OPTION-LIST)
+# ----------------------------------------
+# OPTION-LIST is a space-separated list of Libtool options associated
+# with MACRO-NAME. If any OPTION has a matching handler declared with
+# LT_OPTION_DEFINE, dispatch to that macro; otherwise complain about
+# the unknown option and exit.
+m4_defun([_LT_SET_OPTIONS],
+[# Set options
+m4_foreach([_LT_Option], m4_split(m4_normalize([$2])),
+ [_LT_SET_OPTION([$1], _LT_Option)])
+
+m4_if([$1],[LT_INIT],[
+ dnl
+ dnl Simply set some default values (i.e off) if boolean options were not
+ dnl specified:
+ _LT_UNLESS_OPTIONS([LT_INIT], [dlopen], [enable_dlopen=no
+ ])
+ _LT_UNLESS_OPTIONS([LT_INIT], [win32-dll], [enable_win32_dll=no
+ ])
+ dnl
+ dnl If no reference was made to various pairs of opposing options, then
+ dnl we run the default mode handler for the pair. For example, if neither
+ dnl `shared' nor `disable-shared' was passed, we enable building of shared
+ dnl archives by default:
+ _LT_UNLESS_OPTIONS([LT_INIT], [shared disable-shared], [_LT_ENABLE_SHARED])
+ _LT_UNLESS_OPTIONS([LT_INIT], [static disable-static], [_LT_ENABLE_STATIC])
+ _LT_UNLESS_OPTIONS([LT_INIT], [pic-only no-pic], [_LT_WITH_PIC])
+ _LT_UNLESS_OPTIONS([LT_INIT], [fast-install disable-fast-install],
+ [_LT_ENABLE_FAST_INSTALL])
+ ])
+])# _LT_SET_OPTIONS
+
+
+## --------------------------------- ##
+## Macros to handle LT_INIT options. ##
+## --------------------------------- ##
+
+# _LT_MANGLE_DEFUN(MACRO-NAME, OPTION-NAME)
+# -----------------------------------------
+m4_define([_LT_MANGLE_DEFUN],
+[[_LT_OPTION_DEFUN_]m4_bpatsubst(m4_toupper([$1__$2]), [[^A-Z0-9_]], [_])])
+
+
+# LT_OPTION_DEFINE(MACRO-NAME, OPTION-NAME, CODE)
+# -----------------------------------------------
+m4_define([LT_OPTION_DEFINE],
+[m4_define(_LT_MANGLE_DEFUN([$1], [$2]), [$3])[]dnl
+])# LT_OPTION_DEFINE
+
+
+# dlopen
+# ------
+LT_OPTION_DEFINE([LT_INIT], [dlopen], [enable_dlopen=yes
+])
+
+AU_DEFUN([AC_LIBTOOL_DLOPEN],
+[_LT_SET_OPTION([LT_INIT], [dlopen])
+AC_DIAGNOSE([obsolete],
+[$0: Remove this warning and the call to _LT_SET_OPTION when you
+put the `dlopen' option into LT_INIT's first parameter.])
+])
+
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AC_LIBTOOL_DLOPEN], [])
+
+
+# win32-dll
+# ---------
+# Declare package support for building win32 dll's.
+LT_OPTION_DEFINE([LT_INIT], [win32-dll],
+[enable_win32_dll=yes
+
+case $host in
+*-*-cygwin* | *-*-mingw* | *-*-pw32* | *-cegcc*)
+ AC_CHECK_TOOL(AS, as, false)
+ AC_CHECK_TOOL(DLLTOOL, dlltool, false)
+ AC_CHECK_TOOL(OBJDUMP, objdump, false)
+ ;;
+esac
+
+test -z "$AS" && AS=as
+_LT_DECL([], [AS], [0], [Assembler program])dnl
+
+test -z "$DLLTOOL" && DLLTOOL=dlltool
+_LT_DECL([], [DLLTOOL], [0], [DLL creation program])dnl
+
+test -z "$OBJDUMP" && OBJDUMP=objdump
+_LT_DECL([], [OBJDUMP], [0], [Object dumper program])dnl
+])# win32-dll
+
+AU_DEFUN([AC_LIBTOOL_WIN32_DLL],
+[AC_REQUIRE([AC_CANONICAL_HOST])dnl
+_LT_SET_OPTION([LT_INIT], [win32-dll])
+AC_DIAGNOSE([obsolete],
+[$0: Remove this warning and the call to _LT_SET_OPTION when you
+put the `win32-dll' option into LT_INIT's first parameter.])
+])
+
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AC_LIBTOOL_WIN32_DLL], [])
+
+
+# _LT_ENABLE_SHARED([DEFAULT])
+# ----------------------------
+# implement the --enable-shared flag, and supports the `shared' and
+# `disable-shared' LT_INIT options.
+# DEFAULT is either `yes' or `no'. If omitted, it defaults to `yes'.
+m4_define([_LT_ENABLE_SHARED],
+[m4_define([_LT_ENABLE_SHARED_DEFAULT], [m4_if($1, no, no, yes)])dnl
+AC_ARG_ENABLE([shared],
+ [AS_HELP_STRING([--enable-shared@<:@=PKGS@:>@],
+ [build shared libraries @<:@default=]_LT_ENABLE_SHARED_DEFAULT[@:>@])],
+ [p=${PACKAGE-default}
+ case $enableval in
+ yes) enable_shared=yes ;;
+ no) enable_shared=no ;;
+ *)
+ enable_shared=no
+ # Look at the argument we got. We use all the common list separators.
+ lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR,"
+ for pkg in $enableval; do
+ IFS="$lt_save_ifs"
+ if test "X$pkg" = "X$p"; then
+ enable_shared=yes
+ fi
+ done
+ IFS="$lt_save_ifs"
+ ;;
+ esac],
+ [enable_shared=]_LT_ENABLE_SHARED_DEFAULT)
+
+ _LT_DECL([build_libtool_libs], [enable_shared], [0],
+ [Whether or not to build shared libraries])
+])# _LT_ENABLE_SHARED
+
+LT_OPTION_DEFINE([LT_INIT], [shared], [_LT_ENABLE_SHARED([yes])])
+LT_OPTION_DEFINE([LT_INIT], [disable-shared], [_LT_ENABLE_SHARED([no])])
+
+# Old names:
+AC_DEFUN([AC_ENABLE_SHARED],
+[_LT_SET_OPTION([LT_INIT], m4_if([$1], [no], [disable-])[shared])
+])
+
+AC_DEFUN([AC_DISABLE_SHARED],
+[_LT_SET_OPTION([LT_INIT], [disable-shared])
+])
+
+AU_DEFUN([AM_ENABLE_SHARED], [AC_ENABLE_SHARED($@)])
+AU_DEFUN([AM_DISABLE_SHARED], [AC_DISABLE_SHARED($@)])
+
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AM_ENABLE_SHARED], [])
+dnl AC_DEFUN([AM_DISABLE_SHARED], [])
+
+
+
+# _LT_ENABLE_STATIC([DEFAULT])
+# ----------------------------
+# implement the --enable-static flag, and support the `static' and
+# `disable-static' LT_INIT options.
+# DEFAULT is either `yes' or `no'. If omitted, it defaults to `yes'.
+m4_define([_LT_ENABLE_STATIC],
+[m4_define([_LT_ENABLE_STATIC_DEFAULT], [m4_if($1, no, no, yes)])dnl
+AC_ARG_ENABLE([static],
+ [AS_HELP_STRING([--enable-static@<:@=PKGS@:>@],
+ [build static libraries @<:@default=]_LT_ENABLE_STATIC_DEFAULT[@:>@])],
+ [p=${PACKAGE-default}
+ case $enableval in
+ yes) enable_static=yes ;;
+ no) enable_static=no ;;
+ *)
+ enable_static=no
+ # Look at the argument we got. We use all the common list separators.
+ lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR,"
+ for pkg in $enableval; do
+ IFS="$lt_save_ifs"
+ if test "X$pkg" = "X$p"; then
+ enable_static=yes
+ fi
+ done
+ IFS="$lt_save_ifs"
+ ;;
+ esac],
+ [enable_static=]_LT_ENABLE_STATIC_DEFAULT)
+
+ _LT_DECL([build_old_libs], [enable_static], [0],
+ [Whether or not to build static libraries])
+])# _LT_ENABLE_STATIC
+
+LT_OPTION_DEFINE([LT_INIT], [static], [_LT_ENABLE_STATIC([yes])])
+LT_OPTION_DEFINE([LT_INIT], [disable-static], [_LT_ENABLE_STATIC([no])])
+
+# Old names:
+AC_DEFUN([AC_ENABLE_STATIC],
+[_LT_SET_OPTION([LT_INIT], m4_if([$1], [no], [disable-])[static])
+])
+
+AC_DEFUN([AC_DISABLE_STATIC],
+[_LT_SET_OPTION([LT_INIT], [disable-static])
+])
+
+AU_DEFUN([AM_ENABLE_STATIC], [AC_ENABLE_STATIC($@)])
+AU_DEFUN([AM_DISABLE_STATIC], [AC_DISABLE_STATIC($@)])
+
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AM_ENABLE_STATIC], [])
+dnl AC_DEFUN([AM_DISABLE_STATIC], [])
+
+
+
+# _LT_ENABLE_FAST_INSTALL([DEFAULT])
+# ----------------------------------
+# implement the --enable-fast-install flag, and support the `fast-install'
+# and `disable-fast-install' LT_INIT options.
+# DEFAULT is either `yes' or `no'. If omitted, it defaults to `yes'.
+m4_define([_LT_ENABLE_FAST_INSTALL],
+[m4_define([_LT_ENABLE_FAST_INSTALL_DEFAULT], [m4_if($1, no, no, yes)])dnl
+AC_ARG_ENABLE([fast-install],
+ [AS_HELP_STRING([--enable-fast-install@<:@=PKGS@:>@],
+ [optimize for fast installation @<:@default=]_LT_ENABLE_FAST_INSTALL_DEFAULT[@:>@])],
+ [p=${PACKAGE-default}
+ case $enableval in
+ yes) enable_fast_install=yes ;;
+ no) enable_fast_install=no ;;
+ *)
+ enable_fast_install=no
+ # Look at the argument we got. We use all the common list separators.
+ lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR,"
+ for pkg in $enableval; do
+ IFS="$lt_save_ifs"
+ if test "X$pkg" = "X$p"; then
+ enable_fast_install=yes
+ fi
+ done
+ IFS="$lt_save_ifs"
+ ;;
+ esac],
+ [enable_fast_install=]_LT_ENABLE_FAST_INSTALL_DEFAULT)
+
+_LT_DECL([fast_install], [enable_fast_install], [0],
+ [Whether or not to optimize for fast installation])dnl
+])# _LT_ENABLE_FAST_INSTALL
+
+LT_OPTION_DEFINE([LT_INIT], [fast-install], [_LT_ENABLE_FAST_INSTALL([yes])])
+LT_OPTION_DEFINE([LT_INIT], [disable-fast-install], [_LT_ENABLE_FAST_INSTALL([no])])
+
+# Old names:
+AU_DEFUN([AC_ENABLE_FAST_INSTALL],
+[_LT_SET_OPTION([LT_INIT], m4_if([$1], [no], [disable-])[fast-install])
+AC_DIAGNOSE([obsolete],
+[$0: Remove this warning and the call to _LT_SET_OPTION when you put
+the `fast-install' option into LT_INIT's first parameter.])
+])
+
+AU_DEFUN([AC_DISABLE_FAST_INSTALL],
+[_LT_SET_OPTION([LT_INIT], [disable-fast-install])
+AC_DIAGNOSE([obsolete],
+[$0: Remove this warning and the call to _LT_SET_OPTION when you put
+the `disable-fast-install' option into LT_INIT's first parameter.])
+])
+
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AC_ENABLE_FAST_INSTALL], [])
+dnl AC_DEFUN([AM_DISABLE_FAST_INSTALL], [])
+
+
+# _LT_WITH_PIC([MODE])
+# --------------------
+# implement the --with-pic flag, and support the `pic-only' and `no-pic'
+# LT_INIT options.
+# MODE is either `yes' or `no'. If omitted, it defaults to `both'.
+m4_define([_LT_WITH_PIC],
+[AC_ARG_WITH([pic],
+ [AS_HELP_STRING([--with-pic],
+ [try to use only PIC/non-PIC objects @<:@default=use both@:>@])],
+ [pic_mode="$withval"],
+ [pic_mode=default])
+
+test -z "$pic_mode" && pic_mode=m4_default([$1], [default])
+
+_LT_DECL([], [pic_mode], [0], [What type of objects to build])dnl
+])# _LT_WITH_PIC
+
+LT_OPTION_DEFINE([LT_INIT], [pic-only], [_LT_WITH_PIC([yes])])
+LT_OPTION_DEFINE([LT_INIT], [no-pic], [_LT_WITH_PIC([no])])
+
+# Old name:
+AU_DEFUN([AC_LIBTOOL_PICMODE],
+[_LT_SET_OPTION([LT_INIT], [pic-only])
+AC_DIAGNOSE([obsolete],
+[$0: Remove this warning and the call to _LT_SET_OPTION when you
+put the `pic-only' option into LT_INIT's first parameter.])
+])
+
+dnl aclocal-1.4 backwards compatibility:
+dnl AC_DEFUN([AC_LIBTOOL_PICMODE], [])
+
+## ----------------- ##
+## LTDL_INIT Options ##
+## ----------------- ##
+
+m4_define([_LTDL_MODE], [])
+LT_OPTION_DEFINE([LTDL_INIT], [nonrecursive],
+ [m4_define([_LTDL_MODE], [nonrecursive])])
+LT_OPTION_DEFINE([LTDL_INIT], [recursive],
+ [m4_define([_LTDL_MODE], [recursive])])
+LT_OPTION_DEFINE([LTDL_INIT], [subproject],
+ [m4_define([_LTDL_MODE], [subproject])])
+
+m4_define([_LTDL_TYPE], [])
+LT_OPTION_DEFINE([LTDL_INIT], [installable],
+ [m4_define([_LTDL_TYPE], [installable])])
+LT_OPTION_DEFINE([LTDL_INIT], [convenience],
+ [m4_define([_LTDL_TYPE], [convenience])])
diff --git a/config/ltsugar.m4 b/config/ltsugar.m4
new file mode 100644
index 0000000..9000a05
--- /dev/null
+++ b/config/ltsugar.m4
@@ -0,0 +1,123 @@
+# ltsugar.m4 -- libtool m4 base layer. -*-Autoconf-*-
+#
+# Copyright (C) 2004, 2005, 2007, 2008 Free Software Foundation, Inc.
+# Written by Gary V. Vaughan, 2004
+#
+# This file is free software; the Free Software Foundation gives
+# unlimited permission to copy and/or distribute it, with or without
+# modifications, as long as this notice is preserved.
+
+# serial 6 ltsugar.m4
+
+# This is to help aclocal find these macros, as it can't see m4_define.
+AC_DEFUN([LTSUGAR_VERSION], [m4_if([0.1])])
+
+
+# lt_join(SEP, ARG1, [ARG2...])
+# -----------------------------
+# Produce ARG1SEPARG2...SEPARGn, omitting [] arguments and their
+# associated separator.
+# Needed until we can rely on m4_join from Autoconf 2.62, since all earlier
+# versions in m4sugar had bugs.
+m4_define([lt_join],
+[m4_if([$#], [1], [],
+ [$#], [2], [[$2]],
+ [m4_if([$2], [], [], [[$2]_])$0([$1], m4_shift(m4_shift($@)))])])
+m4_define([_lt_join],
+[m4_if([$#$2], [2], [],
+ [m4_if([$2], [], [], [[$1$2]])$0([$1], m4_shift(m4_shift($@)))])])
+
+
+# lt_car(LIST)
+# lt_cdr(LIST)
+# ------------
+# Manipulate m4 lists.
+# These macros are necessary as long as will still need to support
+# Autoconf-2.59 which quotes differently.
+m4_define([lt_car], [[$1]])
+m4_define([lt_cdr],
+[m4_if([$#], 0, [m4_fatal([$0: cannot be called without arguments])],
+ [$#], 1, [],
+ [m4_dquote(m4_shift($@))])])
+m4_define([lt_unquote], $1)
+
+
+# lt_append(MACRO-NAME, STRING, [SEPARATOR])
+# ------------------------------------------
+# Redefine MACRO-NAME to hold its former content plus `SEPARATOR'`STRING'.
+# Note that neither SEPARATOR nor STRING are expanded; they are appended
+# to MACRO-NAME as is (leaving the expansion for when MACRO-NAME is invoked).
+# No SEPARATOR is output if MACRO-NAME was previously undefined (different
+# than defined and empty).
+#
+# This macro is needed until we can rely on Autoconf 2.62, since earlier
+# versions of m4sugar mistakenly expanded SEPARATOR but not STRING.
+m4_define([lt_append],
+[m4_define([$1],
+ m4_ifdef([$1], [m4_defn([$1])[$3]])[$2])])
+
+
+
+# lt_combine(SEP, PREFIX-LIST, INFIX, SUFFIX1, [SUFFIX2...])
+# ----------------------------------------------------------
+# Produce a SEP delimited list of all paired combinations of elements of
+# PREFIX-LIST with SUFFIX1 through SUFFIXn. Each element of the list
+# has the form PREFIXmINFIXSUFFIXn.
+# Needed until we can rely on m4_combine added in Autoconf 2.62.
+m4_define([lt_combine],
+[m4_if(m4_eval([$# > 3]), [1],
+ [m4_pushdef([_Lt_sep], [m4_define([_Lt_sep], m4_defn([lt_car]))])]]dnl
+[[m4_foreach([_Lt_prefix], [$2],
+ [m4_foreach([_Lt_suffix],
+ ]m4_dquote(m4_dquote(m4_shift(m4_shift(m4_shift($@)))))[,
+ [_Lt_sep([$1])[]m4_defn([_Lt_prefix])[$3]m4_defn([_Lt_suffix])])])])])
+
+
+# lt_if_append_uniq(MACRO-NAME, VARNAME, [SEPARATOR], [UNIQ], [NOT-UNIQ])
+# -----------------------------------------------------------------------
+# Iff MACRO-NAME does not yet contain VARNAME, then append it (delimited
+# by SEPARATOR if supplied) and expand UNIQ, else NOT-UNIQ.
+m4_define([lt_if_append_uniq],
+[m4_ifdef([$1],
+ [m4_if(m4_index([$3]m4_defn([$1])[$3], [$3$2$3]), [-1],
+ [lt_append([$1], [$2], [$3])$4],
+ [$5])],
+ [lt_append([$1], [$2], [$3])$4])])
+
+
+# lt_dict_add(DICT, KEY, VALUE)
+# -----------------------------
+m4_define([lt_dict_add],
+[m4_define([$1($2)], [$3])])
+
+
+# lt_dict_add_subkey(DICT, KEY, SUBKEY, VALUE)
+# --------------------------------------------
+m4_define([lt_dict_add_subkey],
+[m4_define([$1($2:$3)], [$4])])
+
+
+# lt_dict_fetch(DICT, KEY, [SUBKEY])
+# ----------------------------------
+m4_define([lt_dict_fetch],
+[m4_ifval([$3],
+ m4_ifdef([$1($2:$3)], [m4_defn([$1($2:$3)])]),
+ m4_ifdef([$1($2)], [m4_defn([$1($2)])]))])
+
+
+# lt_if_dict_fetch(DICT, KEY, [SUBKEY], VALUE, IF-TRUE, [IF-FALSE])
+# -----------------------------------------------------------------
+m4_define([lt_if_dict_fetch],
+[m4_if(lt_dict_fetch([$1], [$2], [$3]), [$4],
+ [$5],
+ [$6])])
+
+
+# lt_dict_filter(DICT, [SUBKEY], VALUE, [SEPARATOR], KEY, [...])
+# --------------------------------------------------------------
+m4_define([lt_dict_filter],
+[m4_if([$5], [], [],
+ [lt_join(m4_quote(m4_default([$4], [[, ]])),
+ lt_unquote(m4_split(m4_normalize(m4_foreach(_Lt_key, lt_car([m4_shiftn(4, $@)]),
+ [lt_if_dict_fetch([$1], _Lt_key, [$2], [$3], [_Lt_key ])])))))])[]dnl
+])
diff --git a/config/ltversion.m4 b/config/ltversion.m4
new file mode 100644
index 0000000..f3c5309
--- /dev/null
+++ b/config/ltversion.m4
@@ -0,0 +1,23 @@
+# ltversion.m4 -- version numbers -*- Autoconf -*-
+#
+# Copyright (C) 2004 Free Software Foundation, Inc.
+# Written by Scott James Remnant, 2004
+#
+# This file is free software; the Free Software Foundation gives
+# unlimited permission to copy and/or distribute it, with or without
+# modifications, as long as this notice is preserved.
+
+# Generated from ltversion.in.
+
+# serial 3017 ltversion.m4
+# This file is part of GNU Libtool
+
+m4_define([LT_PACKAGE_VERSION], [2.2.6b])
+m4_define([LT_PACKAGE_REVISION], [1.3017])
+
+AC_DEFUN([LTVERSION_VERSION],
+[macro_version='2.2.6b'
+macro_revision='1.3017'
+_LT_DECL(, macro_version, 0, [Which release of libtool.m4 was used?])
+_LT_DECL(, macro_revision, 0)
+])
diff --git a/config/lt~obsolete.m4 b/config/lt~obsolete.m4
new file mode 100644
index 0000000..637bb20
--- /dev/null
+++ b/config/lt~obsolete.m4
@@ -0,0 +1,92 @@
+# lt~obsolete.m4 -- aclocal satisfying obsolete definitions. -*-Autoconf-*-
+#
+# Copyright (C) 2004, 2005, 2007 Free Software Foundation, Inc.
+# Written by Scott James Remnant, 2004.
+#
+# This file is free software; the Free Software Foundation gives
+# unlimited permission to copy and/or distribute it, with or without
+# modifications, as long as this notice is preserved.
+
+# serial 4 lt~obsolete.m4
+
+# These exist entirely to fool aclocal when bootstrapping libtool.
+#
+# In the past libtool.m4 has provided macros via AC_DEFUN (or AU_DEFUN)
+# which have later been changed to m4_define as they aren't part of the
+# exported API, or moved to Autoconf or Automake where they belong.
+#
+# The trouble is, aclocal is a bit thick. It'll see the old AC_DEFUN
+# in /usr/share/aclocal/libtool.m4 and remember it, then when it sees us
+# using a macro with the same name in our local m4/libtool.m4 it'll
+# pull the old libtool.m4 in (it doesn't see our shiny new m4_define
+# and doesn't know about Autoconf macros at all.)
+#
+# So we provide this file, which has a silly filename so it's always
+# included after everything else. This provides aclocal with the
+# AC_DEFUNs it wants, but when m4 processes it, it doesn't do anything
+# because those macros already exist, or will be overwritten later.
+# We use AC_DEFUN over AU_DEFUN for compatibility with aclocal-1.6.
+#
+# Anytime we withdraw an AC_DEFUN or AU_DEFUN, remember to add it here.
+# Yes, that means every name once taken will need to remain here until
+# we give up compatibility with versions before 1.7, at which point
+# we need to keep only those names which we still refer to.
+
+# This is to help aclocal find these macros, as it can't see m4_define.
+AC_DEFUN([LTOBSOLETE_VERSION], [m4_if([1])])
+
+m4_ifndef([AC_LIBTOOL_LINKER_OPTION], [AC_DEFUN([AC_LIBTOOL_LINKER_OPTION])])
+m4_ifndef([AC_PROG_EGREP], [AC_DEFUN([AC_PROG_EGREP])])
+m4_ifndef([_LT_AC_PROG_ECHO_BACKSLASH], [AC_DEFUN([_LT_AC_PROG_ECHO_BACKSLASH])])
+m4_ifndef([_LT_AC_SHELL_INIT], [AC_DEFUN([_LT_AC_SHELL_INIT])])
+m4_ifndef([_LT_AC_SYS_LIBPATH_AIX], [AC_DEFUN([_LT_AC_SYS_LIBPATH_AIX])])
+m4_ifndef([_LT_PROG_LTMAIN], [AC_DEFUN([_LT_PROG_LTMAIN])])
+m4_ifndef([_LT_AC_TAGVAR], [AC_DEFUN([_LT_AC_TAGVAR])])
+m4_ifndef([AC_LTDL_ENABLE_INSTALL], [AC_DEFUN([AC_LTDL_ENABLE_INSTALL])])
+m4_ifndef([AC_LTDL_PREOPEN], [AC_DEFUN([AC_LTDL_PREOPEN])])
+m4_ifndef([_LT_AC_SYS_COMPILER], [AC_DEFUN([_LT_AC_SYS_COMPILER])])
+m4_ifndef([_LT_AC_LOCK], [AC_DEFUN([_LT_AC_LOCK])])
+m4_ifndef([AC_LIBTOOL_SYS_OLD_ARCHIVE], [AC_DEFUN([AC_LIBTOOL_SYS_OLD_ARCHIVE])])
+m4_ifndef([_LT_AC_TRY_DLOPEN_SELF], [AC_DEFUN([_LT_AC_TRY_DLOPEN_SELF])])
+m4_ifndef([AC_LIBTOOL_PROG_CC_C_O], [AC_DEFUN([AC_LIBTOOL_PROG_CC_C_O])])
+m4_ifndef([AC_LIBTOOL_SYS_HARD_LINK_LOCKS], [AC_DEFUN([AC_LIBTOOL_SYS_HARD_LINK_LOCKS])])
+m4_ifndef([AC_LIBTOOL_OBJDIR], [AC_DEFUN([AC_LIBTOOL_OBJDIR])])
+m4_ifndef([AC_LTDL_OBJDIR], [AC_DEFUN([AC_LTDL_OBJDIR])])
+m4_ifndef([AC_LIBTOOL_PROG_LD_HARDCODE_LIBPATH], [AC_DEFUN([AC_LIBTOOL_PROG_LD_HARDCODE_LIBPATH])])
+m4_ifndef([AC_LIBTOOL_SYS_LIB_STRIP], [AC_DEFUN([AC_LIBTOOL_SYS_LIB_STRIP])])
+m4_ifndef([AC_PATH_MAGIC], [AC_DEFUN([AC_PATH_MAGIC])])
+m4_ifndef([AC_PROG_LD_GNU], [AC_DEFUN([AC_PROG_LD_GNU])])
+m4_ifndef([AC_PROG_LD_RELOAD_FLAG], [AC_DEFUN([AC_PROG_LD_RELOAD_FLAG])])
+m4_ifndef([AC_DEPLIBS_CHECK_METHOD], [AC_DEFUN([AC_DEPLIBS_CHECK_METHOD])])
+m4_ifndef([AC_LIBTOOL_PROG_COMPILER_NO_RTTI], [AC_DEFUN([AC_LIBTOOL_PROG_COMPILER_NO_RTTI])])
+m4_ifndef([AC_LIBTOOL_SYS_GLOBAL_SYMBOL_PIPE], [AC_DEFUN([AC_LIBTOOL_SYS_GLOBAL_SYMBOL_PIPE])])
+m4_ifndef([AC_LIBTOOL_PROG_COMPILER_PIC], [AC_DEFUN([AC_LIBTOOL_PROG_COMPILER_PIC])])
+m4_ifndef([AC_LIBTOOL_PROG_LD_SHLIBS], [AC_DEFUN([AC_LIBTOOL_PROG_LD_SHLIBS])])
+m4_ifndef([AC_LIBTOOL_POSTDEP_PREDEP], [AC_DEFUN([AC_LIBTOOL_POSTDEP_PREDEP])])
+m4_ifndef([LT_AC_PROG_EGREP], [AC_DEFUN([LT_AC_PROG_EGREP])])
+m4_ifndef([LT_AC_PROG_SED], [AC_DEFUN([LT_AC_PROG_SED])])
+m4_ifndef([_LT_CC_BASENAME], [AC_DEFUN([_LT_CC_BASENAME])])
+m4_ifndef([_LT_COMPILER_BOILERPLATE], [AC_DEFUN([_LT_COMPILER_BOILERPLATE])])
+m4_ifndef([_LT_LINKER_BOILERPLATE], [AC_DEFUN([_LT_LINKER_BOILERPLATE])])
+m4_ifndef([_AC_PROG_LIBTOOL], [AC_DEFUN([_AC_PROG_LIBTOOL])])
+m4_ifndef([AC_LIBTOOL_SETUP], [AC_DEFUN([AC_LIBTOOL_SETUP])])
+m4_ifndef([_LT_AC_CHECK_DLFCN], [AC_DEFUN([_LT_AC_CHECK_DLFCN])])
+m4_ifndef([AC_LIBTOOL_SYS_DYNAMIC_LINKER], [AC_DEFUN([AC_LIBTOOL_SYS_DYNAMIC_LINKER])])
+m4_ifndef([_LT_AC_TAGCONFIG], [AC_DEFUN([_LT_AC_TAGCONFIG])])
+m4_ifndef([AC_DISABLE_FAST_INSTALL], [AC_DEFUN([AC_DISABLE_FAST_INSTALL])])
+m4_ifndef([_LT_AC_LANG_CXX], [AC_DEFUN([_LT_AC_LANG_CXX])])
+m4_ifndef([_LT_AC_LANG_F77], [AC_DEFUN([_LT_AC_LANG_F77])])
+m4_ifndef([_LT_AC_LANG_GCJ], [AC_DEFUN([_LT_AC_LANG_GCJ])])
+m4_ifndef([AC_LIBTOOL_RC], [AC_DEFUN([AC_LIBTOOL_RC])])
+m4_ifndef([AC_LIBTOOL_LANG_C_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_C_CONFIG])])
+m4_ifndef([_LT_AC_LANG_C_CONFIG], [AC_DEFUN([_LT_AC_LANG_C_CONFIG])])
+m4_ifndef([AC_LIBTOOL_LANG_CXX_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_CXX_CONFIG])])
+m4_ifndef([_LT_AC_LANG_CXX_CONFIG], [AC_DEFUN([_LT_AC_LANG_CXX_CONFIG])])
+m4_ifndef([AC_LIBTOOL_LANG_F77_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_F77_CONFIG])])
+m4_ifndef([_LT_AC_LANG_F77_CONFIG], [AC_DEFUN([_LT_AC_LANG_F77_CONFIG])])
+m4_ifndef([AC_LIBTOOL_LANG_GCJ_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_GCJ_CONFIG])])
+m4_ifndef([_LT_AC_LANG_GCJ_CONFIG], [AC_DEFUN([_LT_AC_LANG_GCJ_CONFIG])])
+m4_ifndef([AC_LIBTOOL_LANG_RC_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_RC_CONFIG])])
+m4_ifndef([_LT_AC_LANG_RC_CONFIG], [AC_DEFUN([_LT_AC_LANG_RC_CONFIG])])
+m4_ifndef([AC_LIBTOOL_CONFIG], [AC_DEFUN([AC_LIBTOOL_CONFIG])])
+m4_ifndef([_LT_AC_FILE_LTDLL_C], [AC_DEFUN([_LT_AC_FILE_LTDLL_C])])
diff --git a/config/md2nroff.pl b/config/md2nroff.pl
deleted file mode 100755
index 18ce667..0000000
--- a/config/md2nroff.pl
+++ /dev/null
@@ -1,164 +0,0 @@
-#!/usr/bin/env perl
-
-# Script to convert markdown to nroff man pages.
-#
-# The main conversion work is done via pandoc. But pandoc doesn't do
-# everything exactly the way we want it, so use some perl regular
-# expressions to fix up what pandoc doesn't get right.
-#
-# Do a "smart" write of the resulting output man page -- only write to
-# the output file if the contents have actually changed compared to
-# what was already there.
-
-use strict;
-use warnings;
-
-use POSIX;
-use File::Basename;
-use Getopt::Long;
-use File::Temp qw/tempfile/;
-
-my $source_arg;
-my $target_arg;
-my $help_arg;
-
-my $ok = Getopt::Long::GetOptions("source=s" => \$source_arg,
- "target=s" => \$target_arg,
- "help|h" => \$help_arg,
- );
-
-if ($help_arg) {
- print "$0 --source input_MD_file --target output_nroff_file\n";
- exit(0);
-}
-
-# Sanity checks
-die "Must specify a source file"
- if (!defined($source_arg));
-die "Source file does not exist ($source_arg)"
- if (! -r $source_arg);
-
-my $pandoc = `which pandoc`;
-die "Cannot find pandoc executable"
- if ($pandoc eq "");
-
-#####################################################################
-
-my $file = $source_arg;
-$file =~ m/(\d+).md/;
-my $section = $1;
-die "Could not figure out the man page section: $source_arg"
- if (!defined($section));
-my $shortfile = basename($file);
-$shortfile =~ s/\.$section\.md$//;
-
-# If the target file was not specified, derive it from the source file
-my $target;
-if (!defined($target_arg)) {
- $target_arg = $source_arg;
-
- $target_arg =~ m/\.(\d)\.md$/;
- my $section = $1;
-
- my $dirname = dirname($target_arg);
- my $basename = basename($target_arg);
- $basename =~ s/\.md$//;
-
- $target = "$dirname/man$section/$basename";
-} else {
- $target = $target_arg;
-}
-
-print "*** Processing: $file -> $target\n";
-
-# Read in the file
-my $pandoc_input;
-open(IN, $file)
- || die "Can't open $file";
-$pandoc_input .= $_
- while (<IN>);
-close(IN);
-
-# Remove the Jekyll header
-$pandoc_input =~ s/.*---\n.+?---\n//s;
-
-# Remove the {% include ... %} directives
-$pandoc_input =~ s/\n{0,1}\s*{%\s+include .+?\s+%}\s*\n/\n/g;
-
-# Change {% highlight c %} to ```c
-$pandoc_input =~ s/^\s*{%\s+highlight\s+c\s+%}\s*$/\n```c/gmi;
-
-# Change {% endhighlight %} to ```
-$pandoc_input =~ s/^\s*\{\%\s+endhighlight\s+\%\}\s*$/```\n/gmi;
-
-# Pandoc does not handle markdown links in output nroff properly,
-# so just remove all links.
-while ($pandoc_input =~ m/\[(.+?)\]\(.+?\)/) {
- my $text = $1;
- $pandoc_input =~ s/\[(.+?)\]\(.+?\)/$text/;
-}
-
-# Add the pandoc header
-$pandoc_input = "% $shortfile($section) Libfabric Programmer's Manual | \@VERSION\@
-% OpenFabrics
-% \@DATE\@\n\n$pandoc_input";
-
-# Generate the nroff output
-my ($fh, $temp_filename) = tempfile();
-print $fh $pandoc_input;
-close($fh);
-
-open(IN, "pandoc -s --from=markdown --to=man $temp_filename|")
- || die "Can't run pandoc";
-my $pandoc_nroff;
-$pandoc_nroff .= $_
- while (<IN>);
-close(IN);
-unlink($temp_filename);
-
-# Now that we have the nroff string result, is it different than the
-# target file?
-my $write_nroff = 1;
-if (-r $target) {
- # If the target file exists, read it in
- open(IN, $target)
- || die "Can't open $target";
- my $target_nroff;
- $target_nroff .= $_
- while (<IN>);
- close(IN);
-
- # Remove the date from the target nroff string so that we can
- # compare and ignore if the date has changed. Note that some
- # versions of pandoc render dates as xxxx\-xx\-xx, and others
- # render it as xxxx-xx-xx. Handle both.
- $target_nroff =~ s/\"\d\d\d\d\\\-\d\d\\\-\d\d\"/\"\\\@DATE\\\@\"/;
- $target_nroff =~ s/\"\d\d\d\d\-\d\d\-\d\d\"/\"\\\@DATE\\\@\"/;
-
- $write_nroff = 0
- if ($pandoc_nroff eq $target_nroff);
-}
-
-# Do we need to write a new target nroff?
-if ($write_nroff) {
-
- # What's the date right now?
- my $now_string = strftime "%Y\\-%m\\-%d", localtime;
- $pandoc_nroff =~ s/\\\@DATE\\\@/$now_string/g;
-
- # Make sure the target directory exists
- my $dirname = dirname($target);
- mkdir($dirname)
- if (! -d $dirname);
-
- open(OUT, ">$target")
- || die "Can't write to $target";
- print OUT $pandoc_nroff;
- close(OUT);
-
- print "--> Wrote new $target\n";
-} else {
- print "--> $target unchanged; not written\n";
-}
-
-exit(0);
diff --git a/config/missing b/config/missing
new file mode 100755
index 0000000..28055d2
--- /dev/null
+++ b/config/missing
@@ -0,0 +1,376 @@
+#! /bin/sh
+# Common stub for a few missing GNU programs while installing.
+
+scriptversion=2009-04-28.21; # UTC
+
+# Copyright (C) 1996, 1997, 1999, 2000, 2002, 2003, 2004, 2005, 2006,
+# 2008, 2009 Free Software Foundation, Inc.
+# Originally by Fran,cois Pinard <pinard at iro.umontreal.ca>, 1996.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+if test $# -eq 0; then
+ echo 1>&2 "Try \`$0 --help' for more information"
+ exit 1
+fi
+
+run=:
+sed_output='s/.* --output[ =]\([^ ]*\).*/\1/p'
+sed_minuso='s/.* -o \([^ ]*\).*/\1/p'
+
+# In the cases where this matters, `missing' is being run in the
+# srcdir already.
+if test -f configure.ac; then
+ configure_ac=configure.ac
+else
+ configure_ac=configure.in
+fi
+
+msg="missing on your system"
+
+case $1 in
+--run)
+ # Try to run requested program, and just exit if it succeeds.
+ run=
+ shift
+ "$@" && exit 0
+ # Exit code 63 means version mismatch. This often happens
+ # when the user try to use an ancient version of a tool on
+ # a file that requires a minimum version. In this case we
+ # we should proceed has if the program had been absent, or
+ # if --run hadn't been passed.
+ if test $? = 63; then
+ run=:
+ msg="probably too old"
+ fi
+ ;;
+
+ -h|--h|--he|--hel|--help)
+ echo "\
+$0 [OPTION]... PROGRAM [ARGUMENT]...
+
+Handle \`PROGRAM [ARGUMENT]...' for when PROGRAM is missing, or return an
+error status if there is no known handling for PROGRAM.
+
+Options:
+ -h, --help display this help and exit
+ -v, --version output version information and exit
+ --run try to run the given command, and emulate it if it fails
+
+Supported PROGRAM values:
+ aclocal touch file \`aclocal.m4'
+ autoconf touch file \`configure'
+ autoheader touch file \`config.h.in'
+ autom4te touch the output file, or create a stub one
+ automake touch all \`Makefile.in' files
+ bison create \`y.tab.[ch]', if possible, from existing .[ch]
+ flex create \`lex.yy.c', if possible, from existing .c
+ help2man touch the output file
+ lex create \`lex.yy.c', if possible, from existing .c
+ makeinfo touch the output file
+ tar try tar, gnutar, gtar, then tar without non-portable flags
+ yacc create \`y.tab.[ch]', if possible, from existing .[ch]
+
+Version suffixes to PROGRAM as well as the prefixes \`gnu-', \`gnu', and
+\`g' are ignored when checking the name.
+
+Send bug reports to <bug-automake at gnu.org>."
+ exit $?
+ ;;
+
+ -v|--v|--ve|--ver|--vers|--versi|--versio|--version)
+ echo "missing $scriptversion (GNU Automake)"
+ exit $?
+ ;;
+
+ -*)
+ echo 1>&2 "$0: Unknown \`$1' option"
+ echo 1>&2 "Try \`$0 --help' for more information"
+ exit 1
+ ;;
+
+esac
+
+# normalize program name to check for.
+program=`echo "$1" | sed '
+ s/^gnu-//; t
+ s/^gnu//; t
+ s/^g//; t'`
+
+# Now exit if we have it, but it failed. Also exit now if we
+# don't have it and --version was passed (most likely to detect
+# the program). This is about non-GNU programs, so use $1 not
+# $program.
+case $1 in
+ lex*|yacc*)
+ # Not GNU programs, they don't have --version.
+ ;;
+
+ tar*)
+ if test -n "$run"; then
+ echo 1>&2 "ERROR: \`tar' requires --run"
+ exit 1
+ elif test "x$2" = "x--version" || test "x$2" = "x--help"; then
+ exit 1
+ fi
+ ;;
+
+ *)
+ if test -z "$run" && ($1 --version) > /dev/null 2>&1; then
+ # We have it, but it failed.
+ exit 1
+ elif test "x$2" = "x--version" || test "x$2" = "x--help"; then
+ # Could not run --version or --help. This is probably someone
+ # running `$TOOL --version' or `$TOOL --help' to check whether
+ # $TOOL exists and not knowing $TOOL uses missing.
+ exit 1
+ fi
+ ;;
+esac
+
+# If it does not exist, or fails to run (possibly an outdated version),
+# try to emulate it.
+case $program in
+ aclocal*)
+ echo 1>&2 "\
+WARNING: \`$1' is $msg. You should only need it if
+ you modified \`acinclude.m4' or \`${configure_ac}'. You might want
+ to install the \`Automake' and \`Perl' packages. Grab them from
+ any GNU archive site."
+ touch aclocal.m4
+ ;;
+
+ autoconf*)
+ echo 1>&2 "\
+WARNING: \`$1' is $msg. You should only need it if
+ you modified \`${configure_ac}'. You might want to install the
+ \`Autoconf' and \`GNU m4' packages. Grab them from any GNU
+ archive site."
+ touch configure
+ ;;
+
+ autoheader*)
+ echo 1>&2 "\
+WARNING: \`$1' is $msg. You should only need it if
+ you modified \`acconfig.h' or \`${configure_ac}'. You might want
+ to install the \`Autoconf' and \`GNU m4' packages. Grab them
+ from any GNU archive site."
+ files=`sed -n 's/^[ ]*A[CM]_CONFIG_HEADER(\([^)]*\)).*/\1/p' ${configure_ac}`
+ test -z "$files" && files="config.h"
+ touch_files=
+ for f in $files; do
+ case $f in
+ *:*) touch_files="$touch_files "`echo "$f" |
+ sed -e 's/^[^:]*://' -e 's/:.*//'`;;
+ *) touch_files="$touch_files $f.in";;
+ esac
+ done
+ touch $touch_files
+ ;;
+
+ automake*)
+ echo 1>&2 "\
+WARNING: \`$1' is $msg. You should only need it if
+ you modified \`Makefile.am', \`acinclude.m4' or \`${configure_ac}'.
+ You might want to install the \`Automake' and \`Perl' packages.
+ Grab them from any GNU archive site."
+ find . -type f -name Makefile.am -print |
+ sed 's/\.am$/.in/' |
+ while read f; do touch "$f"; done
+ ;;
+
+ autom4te*)
+ echo 1>&2 "\
+WARNING: \`$1' is needed, but is $msg.
+ You might have modified some files without having the
+ proper tools for further handling them.
+ You can get \`$1' as part of \`Autoconf' from any GNU
+ archive site."
+
+ file=`echo "$*" | sed -n "$sed_output"`
+ test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
+ if test -f "$file"; then
+ touch $file
+ else
+ test -z "$file" || exec >$file
+ echo "#! /bin/sh"
+ echo "# Created by GNU Automake missing as a replacement of"
+ echo "# $ $@"
+ echo "exit 0"
+ chmod +x $file
+ exit 1
+ fi
+ ;;
+
+ bison*|yacc*)
+ echo 1>&2 "\
+WARNING: \`$1' $msg. You should only need it if
+ you modified a \`.y' file. You may need the \`Bison' package
+ in order for those modifications to take effect. You can get
+ \`Bison' from any GNU archive site."
+ rm -f y.tab.c y.tab.h
+ if test $# -ne 1; then
+ eval LASTARG="\${$#}"
+ case $LASTARG in
+ *.y)
+ SRCFILE=`echo "$LASTARG" | sed 's/y$/c/'`
+ if test -f "$SRCFILE"; then
+ cp "$SRCFILE" y.tab.c
+ fi
+ SRCFILE=`echo "$LASTARG" | sed 's/y$/h/'`
+ if test -f "$SRCFILE"; then
+ cp "$SRCFILE" y.tab.h
+ fi
+ ;;
+ esac
+ fi
+ if test ! -f y.tab.h; then
+ echo >y.tab.h
+ fi
+ if test ! -f y.tab.c; then
+ echo 'main() { return 0; }' >y.tab.c
+ fi
+ ;;
+
+ lex*|flex*)
+ echo 1>&2 "\
+WARNING: \`$1' is $msg. You should only need it if
+ you modified a \`.l' file. You may need the \`Flex' package
+ in order for those modifications to take effect. You can get
+ \`Flex' from any GNU archive site."
+ rm -f lex.yy.c
+ if test $# -ne 1; then
+ eval LASTARG="\${$#}"
+ case $LASTARG in
+ *.l)
+ SRCFILE=`echo "$LASTARG" | sed 's/l$/c/'`
+ if test -f "$SRCFILE"; then
+ cp "$SRCFILE" lex.yy.c
+ fi
+ ;;
+ esac
+ fi
+ if test ! -f lex.yy.c; then
+ echo 'main() { return 0; }' >lex.yy.c
+ fi
+ ;;
+
+ help2man*)
+ echo 1>&2 "\
+WARNING: \`$1' is $msg. You should only need it if
+ you modified a dependency of a manual page. You may need the
+ \`Help2man' package in order for those modifications to take
+ effect. You can get \`Help2man' from any GNU archive site."
+
+ file=`echo "$*" | sed -n "$sed_output"`
+ test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
+ if test -f "$file"; then
+ touch $file
+ else
+ test -z "$file" || exec >$file
+ echo ".ab help2man is required to generate this page"
+ exit $?
+ fi
+ ;;
+
+ makeinfo*)
+ echo 1>&2 "\
+WARNING: \`$1' is $msg. You should only need it if
+ you modified a \`.texi' or \`.texinfo' file, or any other file
+ indirectly affecting the aspect of the manual. The spurious
+ call might also be the consequence of using a buggy \`make' (AIX,
+ DU, IRIX). You might want to install the \`Texinfo' package or
+ the \`GNU make' package. Grab either from any GNU archive site."
+ # The file to touch is that specified with -o ...
+ file=`echo "$*" | sed -n "$sed_output"`
+ test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
+ if test -z "$file"; then
+ # ... or it is the one specified with @setfilename ...
+ infile=`echo "$*" | sed 's/.* \([^ ]*\) *$/\1/'`
+ file=`sed -n '
+ /^@setfilename/{
+ s/.* \([^ ]*\) *$/\1/
+ p
+ q
+ }' $infile`
+ # ... or it is derived from the source name (dir/f.texi becomes f.info)
+ test -z "$file" && file=`echo "$infile" | sed 's,.*/,,;s,.[^.]*$,,'`.info
+ fi
+ # If the file does not exist, the user really needs makeinfo;
+ # let's fail without touching anything.
+ test -f $file || exit 1
+ touch $file
+ ;;
+
+ tar*)
+ shift
+
+ # We have already tried tar in the generic part.
+ # Look for gnutar/gtar before invocation to avoid ugly error
+ # messages.
+ if (gnutar --version > /dev/null 2>&1); then
+ gnutar "$@" && exit 0
+ fi
+ if (gtar --version > /dev/null 2>&1); then
+ gtar "$@" && exit 0
+ fi
+ firstarg="$1"
+ if shift; then
+ case $firstarg in
+ *o*)
+ firstarg=`echo "$firstarg" | sed s/o//`
+ tar "$firstarg" "$@" && exit 0
+ ;;
+ esac
+ case $firstarg in
+ *h*)
+ firstarg=`echo "$firstarg" | sed s/h//`
+ tar "$firstarg" "$@" && exit 0
+ ;;
+ esac
+ fi
+
+ echo 1>&2 "\
+WARNING: I can't seem to be able to run \`tar' with the given arguments.
+ You may want to install GNU tar or Free paxutils, or check the
+ command line arguments."
+ exit 1
+ ;;
+
+ *)
+ echo 1>&2 "\
+WARNING: \`$1' is needed, and is $msg.
+ You might have modified some files without having the
+ proper tools for further handling them. Check the \`README' file,
+ it often tells you about the needed prerequisites for installing
+ this package. You may also peek at any GNU archive site, in case
+ some other package would contain this missing \`$1' program."
+ exit 1
+ ;;
+esac
+
+exit 0
+
+# Local variables:
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "scriptversion="
+# time-stamp-format: "%:y-%02m-%02d.%02H"
+# time-stamp-time-zone: "UTC"
+# time-stamp-end: "; # UTC"
+# End:
diff --git a/config/ofiwg-bot at flatbed-.bashrc b/config/ofiwg-bot at flatbed-.bashrc
deleted file mode 100644
index 5869bd4..0000000
--- a/config/ofiwg-bot at flatbed-.bashrc
+++ /dev/null
@@ -1,12 +0,0 @@
-# .bashrc
-
-# Source global definitions
-if [ -f /etc/bashrc ]; then
- . /etc/bashrc
-fi
-
-# User specific aliases and functions
-export PATH=$HOME/cron/local/bin:$PATH
-export MANPATH=$HOME/cron/local/share/man:$MANPATH
-
-umask 2
diff --git a/config/ofiwg-bot at flatbed-crontab.txt b/config/ofiwg-bot at flatbed-crontab.txt
deleted file mode 100644
index 3359d73..0000000
--- a/config/ofiwg-bot at flatbed-crontab.txt
+++ /dev/null
@@ -1,41 +0,0 @@
-#
-# Update the git clone of libfabric
-#
-# Split this into two lines so that we can have it run every 15 minutes
-# *except* 3:00am (when the nightly tarball is created, which does
-# its own git pull).
-#
-# Update master:
-0,15,30,45 0-2,4-23 * * * (cd $HOME/cron/git/libfabric; git pull --rebase > /dev/null 2>&1; cd $HOME/cron/git/fabtests; git pull --rebase > /dev/null 2>&1)
-15,30,45 3 * * * (cd $HOME/cron/git/libfabric; git pull --rebase > /dev/null 2>&1; cd $HOME/cron/git/fabtests; git pull --rebase > /dev/null 2>&1)
-#
-# Update v1.1.x:
-# (disabled on 26 Feb 2016 -- no more work is occurring on v1.1.x)
-####1,16,31,46 0-2,4-23 * * * (cd $HOME/cron/git/libfabric-1.1.x; git pull --rebase > /dev/null 2>&1)
-####15,30,45 3 * * * (cd $HOME/cron/git/libfabric-1.1.x; git pull --rebase > /dev/null 2>&1)
-#
-# Using the script from newly-updated libfabric master clone,
-# generate new man pages
-#
-5,20,35,50 * * * * (PATH=$HOME/cron/local/bin:$PATH; /home/ofiwg-bot/cron/git/libfabric/config/cron-run-all-md2nroff.pl --repo git at github.com:ofiwg/libfabric.git --source-branch master --pages-branch gh-pages --logfile-dir ~/cron/build-logs)
-#
-# Also update man pages on v1.x branch
-# (disabled on 26 Feb 2016 -- no more work is occurring on v1.1.x)
-# Do not auto-publish the new pages to gh-pages
-#
-####6,21,36,51 * * * * (PATH=$HOME/cron/local/bin:$PATH; /home/ofiwg-bot/cron/git/libfabric/config/cron-run-all-md2nroff.pl --repo git at github.com:ofiwg/libfabric.git --source-branch v1.1.x --logfile-dir ~/cron/build-logs/v1.1.x)
-#
-# Make a nightly tarball snapshot from:
-# - libfabric master
-# - fabtests master
-#
-0 3 * * * (PATH=$HOME/cron/local/bin:$PATH; $HOME/cron/git/libfabric/config/cron-make-nightly-tarball.pl --libfabric-source-dir $HOME/cron/git/libfabric --fabtests-source-dir $HOME/cron/git/fabtests --download-dir=/var/www/html/downloads/ofiwg/nightly_tarballs/v1.2.x --logfile-dir ~/cron/build-logs/v1.2.x --libfabric-coverity-token `cat ~/coverity-libfabric-token.txt` --fabtests-coverity-token `cat ~/coverity-fabtests-token.txt`)
-#
-# Make a nightly tarball snapshot from:
-# - libfabric v1.1.x branch
-# - fabtests master
-# (disabled on 26 Feb 2016 -- no more work is occurring on v1.1.x)
-#
-# No Coverity submissions for this build.
-#
-####15 3 * * * (PATH=$HOME/cron/local/bin:$PATH; $HOME/cron/git/libfabric/config/cron-make-nightly-tarball.pl --libfabric-source-dir $HOME/cron/git/libfabric-1.1.x --fabtests-source-dir $HOME/cron/git/fabtests --download-dir=/var/www/html/downloads/ofiwg/nightly_tarballs/v1.1.x --logfile-dir ~/cron/build-logs/v1.1.x)
diff --git a/configure b/configure
new file mode 100755
index 0000000..d4fbc27
--- /dev/null
+++ b/configure
@@ -0,0 +1,24112 @@
+#! /bin/sh
+# Guess values for system-dependent variables and create Makefiles.
+# Generated by GNU Autoconf 2.67 for libfabric 1.4.0rc2.
+#
+# Report bugs to <ofiwg at lists.openfabrics.org>.
+#
+#
+# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
+# 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free Software
+# Foundation, Inc.
+#
+#
+# This configure script is free software; the Free Software Foundation
+# gives unlimited permission to copy, distribute and modify it.
+## -------------------- ##
+## M4sh Initialization. ##
+## -------------------- ##
+
+# Be more Bourne compatible
+DUALCASE=1; export DUALCASE # for MKS sh
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then :
+ emulate sh
+ NULLCMD=:
+ # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which
+ # is contrary to our usage. Disable this feature.
+ alias -g '${1+"$@"}'='"$@"'
+ setopt NO_GLOB_SUBST
+else
+ case `(set -o) 2>/dev/null` in #(
+ *posix*) :
+ set -o posix ;; #(
+ *) :
+ ;;
+esac
+fi
+
+
+as_nl='
+'
+export as_nl
+# Printing a long string crashes Solaris 7 /usr/bin/printf.
+as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'
+as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo
+as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo
+# Prefer a ksh shell builtin over an external printf program on Solaris,
+# but without wasting forks for bash or zsh.
+if test -z "$BASH_VERSION$ZSH_VERSION" \
+ && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then
+ as_echo='print -r --'
+ as_echo_n='print -rn --'
+elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then
+ as_echo='printf %s\n'
+ as_echo_n='printf %s'
+else
+ if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then
+ as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"'
+ as_echo_n='/usr/ucb/echo -n'
+ else
+ as_echo_body='eval expr "X$1" : "X\\(.*\\)"'
+ as_echo_n_body='eval
+ arg=$1;
+ case $arg in #(
+ *"$as_nl"*)
+ expr "X$arg" : "X\\(.*\\)$as_nl";
+ arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;;
+ esac;
+ expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl"
+ '
+ export as_echo_n_body
+ as_echo_n='sh -c $as_echo_n_body as_echo'
+ fi
+ export as_echo_body
+ as_echo='sh -c $as_echo_body as_echo'
+fi
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+ PATH_SEPARATOR=:
+ (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && {
+ (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 ||
+ PATH_SEPARATOR=';'
+ }
+fi
+
+
+# IFS
+# We need space, tab and new line, in precisely that order. Quoting is
+# there to prevent editors from complaining about space-tab.
+# (If _AS_PATH_WALK were called with IFS unset, it would disable word
+# splitting by setting IFS to empty value.)
+IFS=" "" $as_nl"
+
+# Find who we are. Look in the path if we contain no directory separator.
+case $0 in #((
+ *[\\/]* ) as_myself=$0 ;;
+ *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+ done
+IFS=$as_save_IFS
+
+ ;;
+esac
+# We did not find ourselves, most probably we were run as `sh COMMAND'
+# in which case we are not to be found in the path.
+if test "x$as_myself" = x; then
+ as_myself=$0
+fi
+if test ! -f "$as_myself"; then
+ $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2
+ exit 1
+fi
+
+# Unset variables that we do not need and which cause bugs (e.g. in
+# pre-3.0 UWIN ksh). But do not cause bugs in bash 2.01; the "|| exit 1"
+# suppresses any "Segmentation fault" message there. '((' could
+# trigger a bug in pdksh 5.2.14.
+for as_var in BASH_ENV ENV MAIL MAILPATH
+do eval test x\${$as_var+set} = xset \
+ && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || :
+done
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+LC_ALL=C
+export LC_ALL
+LANGUAGE=C
+export LANGUAGE
+
+# CDPATH.
+(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
+
+if test "x$CONFIG_SHELL" = x; then
+ as_bourne_compatible="if test -n \"\${ZSH_VERSION+set}\" && (emulate sh) >/dev/null 2>&1; then :
+ emulate sh
+ NULLCMD=:
+ # Pre-4.2 versions of Zsh do word splitting on \${1+\"\$@\"}, which
+ # is contrary to our usage. Disable this feature.
+ alias -g '\${1+\"\$@\"}'='\"\$@\"'
+ setopt NO_GLOB_SUBST
+else
+ case \`(set -o) 2>/dev/null\` in #(
+ *posix*) :
+ set -o posix ;; #(
+ *) :
+ ;;
+esac
+fi
+"
+ as_required="as_fn_return () { (exit \$1); }
+as_fn_success () { as_fn_return 0; }
+as_fn_failure () { as_fn_return 1; }
+as_fn_ret_success () { return 0; }
+as_fn_ret_failure () { return 1; }
+
+exitcode=0
+as_fn_success || { exitcode=1; echo as_fn_success failed.; }
+as_fn_failure && { exitcode=1; echo as_fn_failure succeeded.; }
+as_fn_ret_success || { exitcode=1; echo as_fn_ret_success failed.; }
+as_fn_ret_failure && { exitcode=1; echo as_fn_ret_failure succeeded.; }
+if ( set x; as_fn_ret_success y && test x = \"\$1\" ); then :
+
+else
+ exitcode=1; echo positional parameters were not saved.
+fi
+test x\$exitcode = x0 || exit 1"
+ as_suggested=" as_lineno_1=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_1a=\$LINENO
+ as_lineno_2=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_2a=\$LINENO
+ eval 'test \"x\$as_lineno_1'\$as_run'\" != \"x\$as_lineno_2'\$as_run'\" &&
+ test \"x\`expr \$as_lineno_1'\$as_run' + 1\`\" = \"x\$as_lineno_2'\$as_run'\"' || exit 1
+test \$(( 1 + 1 )) = 2 || exit 1"
+ if (eval "$as_required") 2>/dev/null; then :
+ as_have_required=yes
+else
+ as_have_required=no
+fi
+ if test x$as_have_required = xyes && (eval "$as_suggested") 2>/dev/null; then :
+
+else
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+as_found=false
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ as_found=:
+ case $as_dir in #(
+ /*)
+ for as_base in sh bash ksh sh5; do
+ # Try only shells that exist, to save several forks.
+ as_shell=$as_dir/$as_base
+ if { test -f "$as_shell" || test -f "$as_shell.exe"; } &&
+ { $as_echo "$as_bourne_compatible""$as_required" | as_run=a "$as_shell"; } 2>/dev/null; then :
+ CONFIG_SHELL=$as_shell as_have_required=yes
+ if { $as_echo "$as_bourne_compatible""$as_suggested" | as_run=a "$as_shell"; } 2>/dev/null; then :
+ break 2
+fi
+fi
+ done;;
+ esac
+ as_found=false
+done
+$as_found || { if { test -f "$SHELL" || test -f "$SHELL.exe"; } &&
+ { $as_echo "$as_bourne_compatible""$as_required" | as_run=a "$SHELL"; } 2>/dev/null; then :
+ CONFIG_SHELL=$SHELL as_have_required=yes
+fi; }
+IFS=$as_save_IFS
+
+
+ if test "x$CONFIG_SHELL" != x; then :
+ # We cannot yet assume a decent shell, so we have to provide a
+ # neutralization value for shells without unset; and this also
+ # works around shells that cannot unset nonexistent variables.
+ BASH_ENV=/dev/null
+ ENV=/dev/null
+ (unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV
+ export CONFIG_SHELL
+ exec "$CONFIG_SHELL" "$as_myself" ${1+"$@"}
+fi
+
+ if test x$as_have_required = xno; then :
+ $as_echo "$0: This script requires a shell more modern than all"
+ $as_echo "$0: the shells that I found on your system."
+ if test x${ZSH_VERSION+set} = xset ; then
+ $as_echo "$0: In particular, zsh $ZSH_VERSION has bugs and should"
+ $as_echo "$0: be upgraded to zsh 4.3.4 or later."
+ else
+ $as_echo "$0: Please tell bug-autoconf at gnu.org and
+$0: ofiwg at lists.openfabrics.org about your system,
+$0: including any error possibly output before this
+$0: message. Then install a modern shell, or manually run
+$0: the script under such a shell if you do have one."
+ fi
+ exit 1
+fi
+fi
+fi
+SHELL=${CONFIG_SHELL-/bin/sh}
+export SHELL
+# Unset more variables known to interfere with behavior of common tools.
+CLICOLOR_FORCE= GREP_OPTIONS=
+unset CLICOLOR_FORCE GREP_OPTIONS
+
+## --------------------- ##
+## M4sh Shell Functions. ##
+## --------------------- ##
+# as_fn_unset VAR
+# ---------------
+# Portably unset VAR.
+as_fn_unset ()
+{
+ { eval $1=; unset $1;}
+}
+as_unset=as_fn_unset
+
+# as_fn_set_status STATUS
+# -----------------------
+# Set $? to STATUS, without forking.
+as_fn_set_status ()
+{
+ return $1
+} # as_fn_set_status
+
+# as_fn_exit STATUS
+# -----------------
+# Exit the shell with STATUS, even in a "trap 0" or "set -e" context.
+as_fn_exit ()
+{
+ set +e
+ as_fn_set_status $1
+ exit $1
+} # as_fn_exit
+
+# as_fn_mkdir_p
+# -------------
+# Create "$as_dir" as a directory, including parents if necessary.
+as_fn_mkdir_p ()
+{
+
+ case $as_dir in #(
+ -*) as_dir=./$as_dir;;
+ esac
+ test -d "$as_dir" || eval $as_mkdir_p || {
+ as_dirs=
+ while :; do
+ case $as_dir in #(
+ *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'(
+ *) as_qdir=$as_dir;;
+ esac
+ as_dirs="'$as_qdir' $as_dirs"
+ as_dir=`$as_dirname -- "$as_dir" ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+ X"$as_dir" : 'X\(//\)[^/]' \| \
+ X"$as_dir" : 'X\(//\)$' \| \
+ X"$as_dir" : 'X\(/\)' \| . 2>/dev/null ||
+$as_echo X"$as_dir" |
+ sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)[^/].*/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\).*/{
+ s//\1/
+ q
+ }
+ s/.*/./; q'`
+ test -d "$as_dir" && break
+ done
+ test -z "$as_dirs" || eval "mkdir $as_dirs"
+ } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir"
+
+
+} # as_fn_mkdir_p
+# as_fn_append VAR VALUE
+# ----------------------
+# Append the text in VALUE to the end of the definition contained in VAR. Take
+# advantage of any shell optimizations that allow amortized linear growth over
+# repeated appends, instead of the typical quadratic growth present in naive
+# implementations.
+if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null; then :
+ eval 'as_fn_append ()
+ {
+ eval $1+=\$2
+ }'
+else
+ as_fn_append ()
+ {
+ eval $1=\$$1\$2
+ }
+fi # as_fn_append
+
+# as_fn_arith ARG...
+# ------------------
+# Perform arithmetic evaluation on the ARGs, and store the result in the
+# global $as_val. Take advantage of shells that can avoid forks. The arguments
+# must be portable across $(()) and expr.
+if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null; then :
+ eval 'as_fn_arith ()
+ {
+ as_val=$(( $* ))
+ }'
+else
+ as_fn_arith ()
+ {
+ as_val=`expr "$@" || test $? -eq 1`
+ }
+fi # as_fn_arith
+
+
+# as_fn_error STATUS ERROR [LINENO LOG_FD]
+# ----------------------------------------
+# Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are
+# provided, also output the error to LOG_FD, referencing LINENO. Then exit the
+# script with STATUS, using 1 if that was 0.
+as_fn_error ()
+{
+ as_status=$1; test $as_status -eq 0 && as_status=1
+ if test "$4"; then
+ as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+ $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4
+ fi
+ $as_echo "$as_me: error: $2" >&2
+ as_fn_exit $as_status
+} # as_fn_error
+
+if expr a : '\(a\)' >/dev/null 2>&1 &&
+ test "X`expr 00001 : '.*\(...\)'`" = X001; then
+ as_expr=expr
+else
+ as_expr=false
+fi
+
+if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then
+ as_basename=basename
+else
+ as_basename=false
+fi
+
+if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then
+ as_dirname=dirname
+else
+ as_dirname=false
+fi
+
+as_me=`$as_basename -- "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+ X"$0" : 'X\(//\)$' \| \
+ X"$0" : 'X\(/\)' \| . 2>/dev/null ||
+$as_echo X/"$0" |
+ sed '/^.*\/\([^/][^/]*\)\/*$/{
+ s//\1/
+ q
+ }
+ /^X\/\(\/\/\)$/{
+ s//\1/
+ q
+ }
+ /^X\/\(\/\).*/{
+ s//\1/
+ q
+ }
+ s/.*/./; q'`
+
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+
+ as_lineno_1=$LINENO as_lineno_1a=$LINENO
+ as_lineno_2=$LINENO as_lineno_2a=$LINENO
+ eval 'test "x$as_lineno_1'$as_run'" != "x$as_lineno_2'$as_run'" &&
+ test "x`expr $as_lineno_1'$as_run' + 1`" = "x$as_lineno_2'$as_run'"' || {
+ # Blame Lee E. McMahon (1931-1989) for sed's syntax. :-)
+ sed -n '
+ p
+ /[$]LINENO/=
+ ' <$as_myself |
+ sed '
+ s/[$]LINENO.*/&-/
+ t lineno
+ b
+ :lineno
+ N
+ :loop
+ s/[$]LINENO\([^'$as_cr_alnum'_].*\n\)\(.*\)/\2\1\2/
+ t loop
+ s/-\n.*//
+ ' >$as_me.lineno &&
+ chmod +x "$as_me.lineno" ||
+ { $as_echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2; as_fn_exit 1; }
+
+ # Don't try to exec as it changes $[0], causing all sort of problems
+ # (the dirname of $[0] is not the place where we might find the
+ # original and so on. Autoconf is especially sensitive to this).
+ . "./$as_me.lineno"
+ # Exit status is that of the last command.
+ exit
+}
+
+ECHO_C= ECHO_N= ECHO_T=
+case `echo -n x` in #(((((
+-n*)
+ case `echo 'xy\c'` in
+ *c*) ECHO_T=' ';; # ECHO_T is single tab character.
+ xy) ECHO_C='\c';;
+ *) echo `echo ksh88 bug on AIX 6.1` > /dev/null
+ ECHO_T=' ';;
+ esac;;
+*)
+ ECHO_N='-n';;
+esac
+
+rm -f conf$$ conf$$.exe conf$$.file
+if test -d conf$$.dir; then
+ rm -f conf$$.dir/conf$$.file
+else
+ rm -f conf$$.dir
+ mkdir conf$$.dir 2>/dev/null
+fi
+if (echo >conf$$.file) 2>/dev/null; then
+ if ln -s conf$$.file conf$$ 2>/dev/null; then
+ as_ln_s='ln -s'
+ # ... but there are two gotchas:
+ # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail.
+ # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable.
+ # In both cases, we have to default to `cp -p'.
+ ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe ||
+ as_ln_s='cp -p'
+ elif ln conf$$.file conf$$ 2>/dev/null; then
+ as_ln_s=ln
+ else
+ as_ln_s='cp -p'
+ fi
+else
+ as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file
+rmdir conf$$.dir 2>/dev/null
+
+if mkdir -p . 2>/dev/null; then
+ as_mkdir_p='mkdir -p "$as_dir"'
+else
+ test -d ./-p && rmdir ./-p
+ as_mkdir_p=false
+fi
+
+if test -x / >/dev/null 2>&1; then
+ as_test_x='test -x'
+else
+ if ls -dL / >/dev/null 2>&1; then
+ as_ls_L_option=L
+ else
+ as_ls_L_option=
+ fi
+ as_test_x='
+ eval sh -c '\''
+ if test -d "$1"; then
+ test -d "$1/.";
+ else
+ case $1 in #(
+ -*)set "./$1";;
+ esac;
+ case `ls -ld'$as_ls_L_option' "$1" 2>/dev/null` in #((
+ ???[sx]*):;;*)false;;esac;fi
+ '\'' sh
+ '
+fi
+as_executable_p=$as_test_x
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+
+# Check that we are running under the correct shell.
+SHELL=${CONFIG_SHELL-/bin/sh}
+
+case X$lt_ECHO in
+X*--fallback-echo)
+ # Remove one level of quotation (which was required for Make).
+ ECHO=`echo "$lt_ECHO" | sed 's,\\\\\$\\$0,'$0','`
+ ;;
+esac
+
+ECHO=${lt_ECHO-echo}
+if test "X$1" = X--no-reexec; then
+ # Discard the --no-reexec flag, and continue.
+ shift
+elif test "X$1" = X--fallback-echo; then
+ # Avoid inline document here, it may be left over
+ :
+elif test "X`{ $ECHO '\t'; } 2>/dev/null`" = 'X\t' ; then
+ # Yippee, $ECHO works!
+ :
+else
+ # Restart under the correct shell.
+ exec $SHELL "$0" --no-reexec ${1+"$@"}
+fi
+
+if test "X$1" = X--fallback-echo; then
+ # used as fallback echo
+ shift
+ cat <<_LT_EOF
+$*
+_LT_EOF
+ exit 0
+fi
+
+# The HP-UX ksh and POSIX shell print the target directory to stdout
+# if CDPATH is set.
+(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
+
+if test -z "$lt_ECHO"; then
+ if test "X${echo_test_string+set}" != Xset; then
+ # find a string as large as possible, as long as the shell can cope with it
+ for cmd in 'sed 50q "$0"' 'sed 20q "$0"' 'sed 10q "$0"' 'sed 2q "$0"' 'echo test'; do
+ # expected sizes: less than 2Kb, 1Kb, 512 bytes, 16 bytes, ...
+ if { echo_test_string=`eval $cmd`; } 2>/dev/null &&
+ { test "X$echo_test_string" = "X$echo_test_string"; } 2>/dev/null
+ then
+ break
+ fi
+ done
+ fi
+
+ if test "X`{ $ECHO '\t'; } 2>/dev/null`" = 'X\t' &&
+ echo_testing_string=`{ $ECHO "$echo_test_string"; } 2>/dev/null` &&
+ test "X$echo_testing_string" = "X$echo_test_string"; then
+ :
+ else
+ # The Solaris, AIX, and Digital Unix default echo programs unquote
+ # backslashes. This makes it impossible to quote backslashes using
+ # echo "$something" | sed 's/\\/\\\\/g'
+ #
+ # So, first we look for a working echo in the user's PATH.
+
+ lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR
+ for dir in $PATH /usr/ucb; do
+ IFS="$lt_save_ifs"
+ if (test -f $dir/echo || test -f $dir/echo$ac_exeext) &&
+ test "X`($dir/echo '\t') 2>/dev/null`" = 'X\t' &&
+ echo_testing_string=`($dir/echo "$echo_test_string") 2>/dev/null` &&
+ test "X$echo_testing_string" = "X$echo_test_string"; then
+ ECHO="$dir/echo"
+ break
+ fi
+ done
+ IFS="$lt_save_ifs"
+
+ if test "X$ECHO" = Xecho; then
+ # We didn't find a better echo, so look for alternatives.
+ if test "X`{ print -r '\t'; } 2>/dev/null`" = 'X\t' &&
+ echo_testing_string=`{ print -r "$echo_test_string"; } 2>/dev/null` &&
+ test "X$echo_testing_string" = "X$echo_test_string"; then
+ # This shell has a builtin print -r that does the trick.
+ ECHO='print -r'
+ elif { test -f /bin/ksh || test -f /bin/ksh$ac_exeext; } &&
+ test "X$CONFIG_SHELL" != X/bin/ksh; then
+ # If we have ksh, try running configure again with it.
+ ORIGINAL_CONFIG_SHELL=${CONFIG_SHELL-/bin/sh}
+ export ORIGINAL_CONFIG_SHELL
+ CONFIG_SHELL=/bin/ksh
+ export CONFIG_SHELL
+ exec $CONFIG_SHELL "$0" --no-reexec ${1+"$@"}
+ else
+ # Try using printf.
+ ECHO='printf %s\n'
+ if test "X`{ $ECHO '\t'; } 2>/dev/null`" = 'X\t' &&
+ echo_testing_string=`{ $ECHO "$echo_test_string"; } 2>/dev/null` &&
+ test "X$echo_testing_string" = "X$echo_test_string"; then
+ # Cool, printf works
+ :
+ elif echo_testing_string=`($ORIGINAL_CONFIG_SHELL "$0" --fallback-echo '\t') 2>/dev/null` &&
+ test "X$echo_testing_string" = 'X\t' &&
+ echo_testing_string=`($ORIGINAL_CONFIG_SHELL "$0" --fallback-echo "$echo_test_string") 2>/dev/null` &&
+ test "X$echo_testing_string" = "X$echo_test_string"; then
+ CONFIG_SHELL=$ORIGINAL_CONFIG_SHELL
+ export CONFIG_SHELL
+ SHELL="$CONFIG_SHELL"
+ export SHELL
+ ECHO="$CONFIG_SHELL $0 --fallback-echo"
+ elif echo_testing_string=`($CONFIG_SHELL "$0" --fallback-echo '\t') 2>/dev/null` &&
+ test "X$echo_testing_string" = 'X\t' &&
+ echo_testing_string=`($CONFIG_SHELL "$0" --fallback-echo "$echo_test_string") 2>/dev/null` &&
+ test "X$echo_testing_string" = "X$echo_test_string"; then
+ ECHO="$CONFIG_SHELL $0 --fallback-echo"
+ else
+ # maybe with a smaller string...
+ prev=:
+
+ for cmd in 'echo test' 'sed 2q "$0"' 'sed 10q "$0"' 'sed 20q "$0"' 'sed 50q "$0"'; do
+ if { test "X$echo_test_string" = "X`eval $cmd`"; } 2>/dev/null
+ then
+ break
+ fi
+ prev="$cmd"
+ done
+
+ if test "$prev" != 'sed 50q "$0"'; then
+ echo_test_string=`eval $prev`
+ export echo_test_string
+ exec ${ORIGINAL_CONFIG_SHELL-${CONFIG_SHELL-/bin/sh}} "$0" ${1+"$@"}
+ else
+ # Oops. We lost completely, so just stick with echo.
+ ECHO=echo
+ fi
+ fi
+ fi
+ fi
+ fi
+fi
+
+# Copy echo and quote the copy suitably for passing to libtool from
+# the Makefile, instead of quoting the original, which is used later.
+lt_ECHO=$ECHO
+if test "X$lt_ECHO" = "X$CONFIG_SHELL $0 --fallback-echo"; then
+ lt_ECHO="$CONFIG_SHELL \\\$\$0 --fallback-echo"
+fi
+
+
+
+
+test -n "$DJDIR" || exec 7<&0 </dev/null
+exec 6>&1
+
+# Name of the host.
+# hostname on some systems (SVR3.2, old GNU/Linux) returns a bogus exit status,
+# so uname gets run too.
+ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q`
+
+#
+# Initializations.
+#
+ac_default_prefix=/usr/local
+ac_clean_files=
+ac_config_libobj_dir=.
+LIBOBJS=
+cross_compiling=no
+subdirs=
+MFLAGS=
+MAKEFLAGS=
+
+# Identity of this package.
+PACKAGE_NAME='libfabric'
+PACKAGE_TARNAME='libfabric'
+PACKAGE_VERSION='1.4.0rc2'
+PACKAGE_STRING='libfabric 1.4.0rc2'
+PACKAGE_BUGREPORT='ofiwg at lists.openfabrics.org'
+PACKAGE_URL=''
+
+ac_unique_file="src/fabric.c"
+# Factoring default headers for most tests.
+ac_includes_default="\
+#include <stdio.h>
+#ifdef HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+# include <sys/stat.h>
+#endif
+#ifdef STDC_HEADERS
+# include <stdlib.h>
+# include <stddef.h>
+#else
+# ifdef HAVE_STDLIB_H
+# include <stdlib.h>
+# endif
+#endif
+#ifdef HAVE_STRING_H
+# if !defined STDC_HEADERS && defined HAVE_MEMORY_H
+# include <memory.h>
+# endif
+# include <string.h>
+#endif
+#ifdef HAVE_STRINGS_H
+# include <strings.h>
+#endif
+#ifdef HAVE_INTTYPES_H
+# include <inttypes.h>
+#endif
+#ifdef HAVE_STDINT_H
+# include <stdint.h>
+#endif
+#ifdef HAVE_UNISTD_H
+# include <unistd.h>
+#endif"
+
+ac_subst_vars='am__EXEEXT_FALSE
+am__EXEEXT_TRUE
+LTLIBOBJS
+LIBOBJS
+HAVE_DIRECT_FALSE
+HAVE_DIRECT_TRUE
+PROVIDER_DIRECT
+FI_PC_PRIVATE_LIBS
+FI_PC_PUBLIC_LIBS
+FI_PC_CFLAGS
+PROVIDERS_STATIC
+PROVIDERS_DL
+PROVIDERS_TO_BUILD
+HAVE_RXD_DL_FALSE
+HAVE_RXD_DL_TRUE
+HAVE_RXD_FALSE
+HAVE_RXD_TRUE
+HAVE_RXM_DL_FALSE
+HAVE_RXM_DL_TRUE
+HAVE_RXM_FALSE
+HAVE_RXM_TRUE
+HAVE_UDP_DL_FALSE
+HAVE_UDP_DL_TRUE
+HAVE_UDP_FALSE
+HAVE_UDP_TRUE
+udp_shm_LIBS
+udp_shm_LDFLAGS
+udp_shm_CPPFLAGS
+HAVE_GNI_DL_FALSE
+HAVE_GNI_DL_TRUE
+HAVE_GNI_FALSE
+HAVE_GNI_TRUE
+gnitest_LIBS
+gnitest_LDFLAGS
+gnitest_CPPFLAGS
+gni_LDFLAGS
+gni_CPPFLAGS
+HAVE_CRITERION_FALSE
+HAVE_CRITERION_TRUE
+CRAY_PMI_LIBS
+CRAY_PMI_CFLAGS
+CRAY_XPMEM_LIBS
+CRAY_XPMEM_CFLAGS
+CRAY_UDREG_LIBS
+CRAY_UDREG_CFLAGS
+CRAY_ALPS_UTIL_LIBS
+CRAY_ALPS_UTIL_CFLAGS
+CRAY_ALPS_LLI_STATIC_LIBS
+CRAY_ALPS_LLI_LIBS
+CRAY_ALPS_LLI_CFLAGS
+CRAY_UGNI_LIBS
+CRAY_UGNI_CFLAGS
+CRAY_GNI_HEADERS_LIBS
+CRAY_GNI_HEADERS_CFLAGS
+FI_PKG_CONFIG_LIBDIR
+FI_PKG_CONFIG_PATH
+FI_PKG_CONFIG
+HAVE_MXM_DL_FALSE
+HAVE_MXM_DL_TRUE
+HAVE_MXM_FALSE
+HAVE_MXM_TRUE
+mxm_LIBS
+mxm_LDFLAGS
+mxm_CPPFLAGS
+HAVE_USNIC_DL_FALSE
+HAVE_USNIC_DL_TRUE
+HAVE_USNIC_FALSE
+HAVE_USNIC_TRUE
+usnic_LIBS
+usnic_LDFLAGS
+usnic_CPPFLAGS
+HAVE_LIBNL3
+usnic_nl_LIBS
+usnic_nl_LDFLAGS
+usnic_nl_CPPFLAGS
+HAVE_VERBS_DL_FALSE
+HAVE_VERBS_DL_TRUE
+HAVE_VERBS_FALSE
+HAVE_VERBS_TRUE
+verbs_LIBS
+verbs_LDFLAGS
+verbs_CPPFLAGS
+verbs_rdmacm_LIBS
+verbs_rdmacm_LDFLAGS
+verbs_rdmacm_CPPFLAGS
+verbs_ibverbs_LIBS
+verbs_ibverbs_LDFLAGS
+verbs_ibverbs_CPPFLAGS
+HAVE_SOCKETS_DL_FALSE
+HAVE_SOCKETS_DL_TRUE
+HAVE_SOCKETS_FALSE
+HAVE_SOCKETS_TRUE
+sockets_LIBS
+sockets_LDFLAGS
+sockets_CPPFLAGS
+HAVE_PSM2_DL_FALSE
+HAVE_PSM2_DL_TRUE
+HAVE_PSM2_FALSE
+HAVE_PSM2_TRUE
+psm2_LIBS
+psm2_LDFLAGS
+psm2_CPPFLAGS
+HAVE_PSM_DL_FALSE
+HAVE_PSM_DL_TRUE
+HAVE_PSM_FALSE
+HAVE_PSM_TRUE
+psm_LIBS
+psm_LDFLAGS
+psm_CPPFLAGS
+HAVE_LD_VERSION_SCRIPT_FALSE
+HAVE_LD_VERSION_SCRIPT_TRUE
+EMBEDDED_FALSE
+EMBEDDED_TRUE
+OTOOL64
+OTOOL
+LIPO
+NMEDIT
+DSYMUTIL
+lt_ECHO
+RANLIB
+AR
+OBJDUMP
+LN_S
+NM
+ac_ct_DUMPBIN
+DUMPBIN
+LD
+FGREP
+SED
+LIBTOOL
+EGREP
+GREP
+CPP
+am__fastdepCC_FALSE
+am__fastdepCC_TRUE
+CCDEPMODE
+AMDEPBACKSLASH
+AMDEP_FALSE
+AMDEP_TRUE
+am__quote
+am__include
+DEPDIR
+OBJEXT
+EXEEXT
+ac_ct_CC
+CPPFLAGS
+LDFLAGS
+CFLAGS
+CC
+FREEBSD_FALSE
+FREEBSD_TRUE
+LINUX_FALSE
+LINUX_TRUE
+MACOS_FALSE
+MACOS_TRUE
+host_os
+host_vendor
+host_cpu
+host
+build_os
+build_vendor
+build_cpu
+build
+AM_BACKSLASH
+AM_DEFAULT_VERBOSITY
+am__untar
+am__tar
+AMTAR
+am__leading_dot
+SET_MAKE
+AWK
+mkdir_p
+MKDIR_P
+INSTALL_STRIP_PROGRAM
+STRIP
+install_sh
+MAKEINFO
+AUTOHEADER
+AUTOMAKE
+AUTOCONF
+ACLOCAL
+VERSION
+PACKAGE
+CYGPATH_W
+am__isrc
+INSTALL_DATA
+INSTALL_SCRIPT
+INSTALL_PROGRAM
+target_alias
+host_alias
+build_alias
+LIBS
+ECHO_T
+ECHO_N
+ECHO_C
+DEFS
+mandir
+localedir
+libdir
+psdir
+pdfdir
+dvidir
+htmldir
+infodir
+docdir
+oldincludedir
+includedir
+localstatedir
+sharedstatedir
+sysconfdir
+datadir
+datarootdir
+libexecdir
+sbindir
+bindir
+program_transform_name
+prefix
+exec_prefix
+PACKAGE_URL
+PACKAGE_BUGREPORT
+PACKAGE_STRING
+PACKAGE_VERSION
+PACKAGE_TARNAME
+PACKAGE_NAME
+PATH_SEPARATOR
+SHELL'
+ac_subst_files='FI_DIRECT_PROVIDER_API_10'
+ac_user_opts='
+enable_option_checking
+enable_silent_rules
+enable_debug
+with_valgrind
+enable_direct
+enable_dependency_tracking
+enable_shared
+enable_static
+with_pic
+enable_fast_install
+with_gnu_ld
+enable_libtool_lock
+with_dlopen
+enable_embedded
+enable_psm
+enable_psm2
+enable_sockets
+enable_verbs
+enable_usnic
+with_libnl
+enable_mxm
+enable_gni
+enable_xpmem
+with_criterion
+with_kdreg
+enable_udp
+enable_rxm
+enable_rxd
+'
+ ac_precious_vars='build_alias
+host_alias
+target_alias
+CC
+CFLAGS
+LDFLAGS
+LIBS
+CPPFLAGS
+CPP
+FI_PKG_CONFIG
+FI_PKG_CONFIG_PATH
+FI_PKG_CONFIG_LIBDIR
+CRAY_GNI_HEADERS_CFLAGS
+CRAY_GNI_HEADERS_LIBS
+CRAY_UGNI_CFLAGS
+CRAY_UGNI_LIBS
+CRAY_ALPS_LLI_CFLAGS
+CRAY_ALPS_LLI_LIBS
+CRAY_ALPS_LLI_STATIC_LIBS
+CRAY_ALPS_UTIL_CFLAGS
+CRAY_ALPS_UTIL_LIBS
+CRAY_UDREG_CFLAGS
+CRAY_UDREG_LIBS
+CRAY_XPMEM_CFLAGS
+CRAY_XPMEM_LIBS
+CRAY_PMI_CFLAGS
+CRAY_PMI_LIBS'
+
+
+# Initialize some variables set by options.
+ac_init_help=
+ac_init_version=false
+ac_unrecognized_opts=
+ac_unrecognized_sep=
+# The variables have the same names as the options, with
+# dashes changed to underlines.
+cache_file=/dev/null
+exec_prefix=NONE
+no_create=
+no_recursion=
+prefix=NONE
+program_prefix=NONE
+program_suffix=NONE
+program_transform_name=s,x,x,
+silent=
+site=
+srcdir=
+verbose=
+x_includes=NONE
+x_libraries=NONE
+
+# Installation directory options.
+# These are left unexpanded so users can "make install exec_prefix=/foo"
+# and all the variables that are supposed to be based on exec_prefix
+# by default will actually change.
+# Use braces instead of parens because sh, perl, etc. also accept them.
+# (The list follows the same order as the GNU Coding Standards.)
+bindir='${exec_prefix}/bin'
+sbindir='${exec_prefix}/sbin'
+libexecdir='${exec_prefix}/libexec'
+datarootdir='${prefix}/share'
+datadir='${datarootdir}'
+sysconfdir='${prefix}/etc'
+sharedstatedir='${prefix}/com'
+localstatedir='${prefix}/var'
+includedir='${prefix}/include'
+oldincludedir='/usr/include'
+docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
+infodir='${datarootdir}/info'
+htmldir='${docdir}'
+dvidir='${docdir}'
+pdfdir='${docdir}'
+psdir='${docdir}'
+libdir='${exec_prefix}/lib'
+localedir='${datarootdir}/locale'
+mandir='${datarootdir}/man'
+
+ac_prev=
+ac_dashdash=
+for ac_option
+do
+ # If the previous option needs an argument, assign it.
+ if test -n "$ac_prev"; then
+ eval $ac_prev=\$ac_option
+ ac_prev=
+ continue
+ fi
+
+ case $ac_option in
+ *=?*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;;
+ *=) ac_optarg= ;;
+ *) ac_optarg=yes ;;
+ esac
+
+ # Accept the important Cygnus configure options, so we can diagnose typos.
+
+ case $ac_dashdash$ac_option in
+ --)
+ ac_dashdash=yes ;;
+
+ -bindir | --bindir | --bindi | --bind | --bin | --bi)
+ ac_prev=bindir ;;
+ -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*)
+ bindir=$ac_optarg ;;
+
+ -build | --build | --buil | --bui | --bu)
+ ac_prev=build_alias ;;
+ -build=* | --build=* | --buil=* | --bui=* | --bu=*)
+ build_alias=$ac_optarg ;;
+
+ -cache-file | --cache-file | --cache-fil | --cache-fi \
+ | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+ ac_prev=cache_file ;;
+ -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+ | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*)
+ cache_file=$ac_optarg ;;
+
+ --config-cache | -C)
+ cache_file=config.cache ;;
+
+ -datadir | --datadir | --datadi | --datad)
+ ac_prev=datadir ;;
+ -datadir=* | --datadir=* | --datadi=* | --datad=*)
+ datadir=$ac_optarg ;;
+
+ -datarootdir | --datarootdir | --datarootdi | --datarootd | --dataroot \
+ | --dataroo | --dataro | --datar)
+ ac_prev=datarootdir ;;
+ -datarootdir=* | --datarootdir=* | --datarootdi=* | --datarootd=* \
+ | --dataroot=* | --dataroo=* | --dataro=* | --datar=*)
+ datarootdir=$ac_optarg ;;
+
+ -disable-* | --disable-*)
+ ac_useropt=`expr "x$ac_option" : 'x-*disable-\(.*\)'`
+ # Reject names that are not valid shell variable names.
+ expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null &&
+ as_fn_error $? "invalid feature name: $ac_useropt"
+ ac_useropt_orig=$ac_useropt
+ ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'`
+ case $ac_user_opts in
+ *"
+"enable_$ac_useropt"
+"*) ;;
+ *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--disable-$ac_useropt_orig"
+ ac_unrecognized_sep=', ';;
+ esac
+ eval enable_$ac_useropt=no ;;
+
+ -docdir | --docdir | --docdi | --doc | --do)
+ ac_prev=docdir ;;
+ -docdir=* | --docdir=* | --docdi=* | --doc=* | --do=*)
+ docdir=$ac_optarg ;;
+
+ -dvidir | --dvidir | --dvidi | --dvid | --dvi | --dv)
+ ac_prev=dvidir ;;
+ -dvidir=* | --dvidir=* | --dvidi=* | --dvid=* | --dvi=* | --dv=*)
+ dvidir=$ac_optarg ;;
+
+ -enable-* | --enable-*)
+ ac_useropt=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'`
+ # Reject names that are not valid shell variable names.
+ expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null &&
+ as_fn_error $? "invalid feature name: $ac_useropt"
+ ac_useropt_orig=$ac_useropt
+ ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'`
+ case $ac_user_opts in
+ *"
+"enable_$ac_useropt"
+"*) ;;
+ *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--enable-$ac_useropt_orig"
+ ac_unrecognized_sep=', ';;
+ esac
+ eval enable_$ac_useropt=\$ac_optarg ;;
+
+ -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \
+ | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \
+ | --exec | --exe | --ex)
+ ac_prev=exec_prefix ;;
+ -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \
+ | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \
+ | --exec=* | --exe=* | --ex=*)
+ exec_prefix=$ac_optarg ;;
+
+ -gas | --gas | --ga | --g)
+ # Obsolete; use --with-gas.
+ with_gas=yes ;;
+
+ -help | --help | --hel | --he | -h)
+ ac_init_help=long ;;
+ -help=r* | --help=r* | --hel=r* | --he=r* | -hr*)
+ ac_init_help=recursive ;;
+ -help=s* | --help=s* | --hel=s* | --he=s* | -hs*)
+ ac_init_help=short ;;
+
+ -host | --host | --hos | --ho)
+ ac_prev=host_alias ;;
+ -host=* | --host=* | --hos=* | --ho=*)
+ host_alias=$ac_optarg ;;
+
+ -htmldir | --htmldir | --htmldi | --htmld | --html | --htm | --ht)
+ ac_prev=htmldir ;;
+ -htmldir=* | --htmldir=* | --htmldi=* | --htmld=* | --html=* | --htm=* \
+ | --ht=*)
+ htmldir=$ac_optarg ;;
+
+ -includedir | --includedir | --includedi | --included | --include \
+ | --includ | --inclu | --incl | --inc)
+ ac_prev=includedir ;;
+ -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \
+ | --includ=* | --inclu=* | --incl=* | --inc=*)
+ includedir=$ac_optarg ;;
+
+ -infodir | --infodir | --infodi | --infod | --info | --inf)
+ ac_prev=infodir ;;
+ -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*)
+ infodir=$ac_optarg ;;
+
+ -libdir | --libdir | --libdi | --libd)
+ ac_prev=libdir ;;
+ -libdir=* | --libdir=* | --libdi=* | --libd=*)
+ libdir=$ac_optarg ;;
+
+ -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \
+ | --libexe | --libex | --libe)
+ ac_prev=libexecdir ;;
+ -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \
+ | --libexe=* | --libex=* | --libe=*)
+ libexecdir=$ac_optarg ;;
+
+ -localedir | --localedir | --localedi | --localed | --locale)
+ ac_prev=localedir ;;
+ -localedir=* | --localedir=* | --localedi=* | --localed=* | --locale=*)
+ localedir=$ac_optarg ;;
+
+ -localstatedir | --localstatedir | --localstatedi | --localstated \
+ | --localstate | --localstat | --localsta | --localst | --locals)
+ ac_prev=localstatedir ;;
+ -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \
+ | --localstate=* | --localstat=* | --localsta=* | --localst=* | --locals=*)
+ localstatedir=$ac_optarg ;;
+
+ -mandir | --mandir | --mandi | --mand | --man | --ma | --m)
+ ac_prev=mandir ;;
+ -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*)
+ mandir=$ac_optarg ;;
+
+ -nfp | --nfp | --nf)
+ # Obsolete; use --without-fp.
+ with_fp=no ;;
+
+ -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+ | --no-cr | --no-c | -n)
+ no_create=yes ;;
+
+ -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+ | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r)
+ no_recursion=yes ;;
+
+ -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \
+ | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \
+ | --oldin | --oldi | --old | --ol | --o)
+ ac_prev=oldincludedir ;;
+ -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \
+ | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \
+ | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*)
+ oldincludedir=$ac_optarg ;;
+
+ -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+ ac_prev=prefix ;;
+ -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+ prefix=$ac_optarg ;;
+
+ -program-prefix | --program-prefix | --program-prefi | --program-pref \
+ | --program-pre | --program-pr | --program-p)
+ ac_prev=program_prefix ;;
+ -program-prefix=* | --program-prefix=* | --program-prefi=* \
+ | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*)
+ program_prefix=$ac_optarg ;;
+
+ -program-suffix | --program-suffix | --program-suffi | --program-suff \
+ | --program-suf | --program-su | --program-s)
+ ac_prev=program_suffix ;;
+ -program-suffix=* | --program-suffix=* | --program-suffi=* \
+ | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*)
+ program_suffix=$ac_optarg ;;
+
+ -program-transform-name | --program-transform-name \
+ | --program-transform-nam | --program-transform-na \
+ | --program-transform-n | --program-transform- \
+ | --program-transform | --program-transfor \
+ | --program-transfo | --program-transf \
+ | --program-trans | --program-tran \
+ | --progr-tra | --program-tr | --program-t)
+ ac_prev=program_transform_name ;;
+ -program-transform-name=* | --program-transform-name=* \
+ | --program-transform-nam=* | --program-transform-na=* \
+ | --program-transform-n=* | --program-transform-=* \
+ | --program-transform=* | --program-transfor=* \
+ | --program-transfo=* | --program-transf=* \
+ | --program-trans=* | --program-tran=* \
+ | --progr-tra=* | --program-tr=* | --program-t=*)
+ program_transform_name=$ac_optarg ;;
+
+ -pdfdir | --pdfdir | --pdfdi | --pdfd | --pdf | --pd)
+ ac_prev=pdfdir ;;
+ -pdfdir=* | --pdfdir=* | --pdfdi=* | --pdfd=* | --pdf=* | --pd=*)
+ pdfdir=$ac_optarg ;;
+
+ -psdir | --psdir | --psdi | --psd | --ps)
+ ac_prev=psdir ;;
+ -psdir=* | --psdir=* | --psdi=* | --psd=* | --ps=*)
+ psdir=$ac_optarg ;;
+
+ -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+ | -silent | --silent | --silen | --sile | --sil)
+ silent=yes ;;
+
+ -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
+ ac_prev=sbindir ;;
+ -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
+ | --sbi=* | --sb=*)
+ sbindir=$ac_optarg ;;
+
+ -sharedstatedir | --sharedstatedir | --sharedstatedi \
+ | --sharedstated | --sharedstate | --sharedstat | --sharedsta \
+ | --sharedst | --shareds | --shared | --share | --shar \
+ | --sha | --sh)
+ ac_prev=sharedstatedir ;;
+ -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \
+ | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \
+ | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \
+ | --sha=* | --sh=*)
+ sharedstatedir=$ac_optarg ;;
+
+ -site | --site | --sit)
+ ac_prev=site ;;
+ -site=* | --site=* | --sit=*)
+ site=$ac_optarg ;;
+
+ -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+ ac_prev=srcdir ;;
+ -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+ srcdir=$ac_optarg ;;
+
+ -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \
+ | --syscon | --sysco | --sysc | --sys | --sy)
+ ac_prev=sysconfdir ;;
+ -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \
+ | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*)
+ sysconfdir=$ac_optarg ;;
+
+ -target | --target | --targe | --targ | --tar | --ta | --t)
+ ac_prev=target_alias ;;
+ -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*)
+ target_alias=$ac_optarg ;;
+
+ -v | -verbose | --verbose | --verbos | --verbo | --verb)
+ verbose=yes ;;
+
+ -version | --version | --versio | --versi | --vers | -V)
+ ac_init_version=: ;;
+
+ -with-* | --with-*)
+ ac_useropt=`expr "x$ac_option" : 'x-*with-\([^=]*\)'`
+ # Reject names that are not valid shell variable names.
+ expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null &&
+ as_fn_error $? "invalid package name: $ac_useropt"
+ ac_useropt_orig=$ac_useropt
+ ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'`
+ case $ac_user_opts in
+ *"
+"with_$ac_useropt"
+"*) ;;
+ *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--with-$ac_useropt_orig"
+ ac_unrecognized_sep=', ';;
+ esac
+ eval with_$ac_useropt=\$ac_optarg ;;
+
+ -without-* | --without-*)
+ ac_useropt=`expr "x$ac_option" : 'x-*without-\(.*\)'`
+ # Reject names that are not valid shell variable names.
+ expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null &&
+ as_fn_error $? "invalid package name: $ac_useropt"
+ ac_useropt_orig=$ac_useropt
+ ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'`
+ case $ac_user_opts in
+ *"
+"with_$ac_useropt"
+"*) ;;
+ *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--without-$ac_useropt_orig"
+ ac_unrecognized_sep=', ';;
+ esac
+ eval with_$ac_useropt=no ;;
+
+ --x)
+ # Obsolete; use --with-x.
+ with_x=yes ;;
+
+ -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \
+ | --x-incl | --x-inc | --x-in | --x-i)
+ ac_prev=x_includes ;;
+ -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \
+ | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*)
+ x_includes=$ac_optarg ;;
+
+ -x-libraries | --x-libraries | --x-librarie | --x-librari \
+ | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l)
+ ac_prev=x_libraries ;;
+ -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \
+ | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*)
+ x_libraries=$ac_optarg ;;
+
+ -*) as_fn_error $? "unrecognized option: \`$ac_option'
+Try \`$0 --help' for more information"
+ ;;
+
+ *=*)
+ ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='`
+ # Reject names that are not valid shell variable names.
+ case $ac_envvar in #(
+ '' | [0-9]* | *[!_$as_cr_alnum]* )
+ as_fn_error $? "invalid variable name: \`$ac_envvar'" ;;
+ esac
+ eval $ac_envvar=\$ac_optarg
+ export $ac_envvar ;;
+
+ *)
+ # FIXME: should be removed in autoconf 3.0.
+ $as_echo "$as_me: WARNING: you should use --build, --host, --target" >&2
+ expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null &&
+ $as_echo "$as_me: WARNING: invalid host type: $ac_option" >&2
+ : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}
+ ;;
+
+ esac
+done
+
+if test -n "$ac_prev"; then
+ ac_option=--`echo $ac_prev | sed 's/_/-/g'`
+ as_fn_error $? "missing argument to $ac_option"
+fi
+
+if test -n "$ac_unrecognized_opts"; then
+ case $enable_option_checking in
+ no) ;;
+ fatal) as_fn_error $? "unrecognized options: $ac_unrecognized_opts" ;;
+ *) $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2 ;;
+ esac
+fi
+
+# Check all directory arguments for consistency.
+for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
+ datadir sysconfdir sharedstatedir localstatedir includedir \
+ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
+ libdir localedir mandir
+do
+ eval ac_val=\$$ac_var
+ # Remove trailing slashes.
+ case $ac_val in
+ */ )
+ ac_val=`expr "X$ac_val" : 'X\(.*[^/]\)' \| "X$ac_val" : 'X\(.*\)'`
+ eval $ac_var=\$ac_val;;
+ esac
+ # Be sure to have absolute directory names.
+ case $ac_val in
+ [\\/$]* | ?:[\\/]* ) continue;;
+ NONE | '' ) case $ac_var in *prefix ) continue;; esac;;
+ esac
+ as_fn_error $? "expected an absolute directory name for --$ac_var: $ac_val"
+done
+
+# There might be people who depend on the old broken behavior: `$host'
+# used to hold the argument of --host etc.
+# FIXME: To remove some day.
+build=$build_alias
+host=$host_alias
+target=$target_alias
+
+# FIXME: To remove some day.
+if test "x$host_alias" != x; then
+ if test "x$build_alias" = x; then
+ cross_compiling=maybe
+ $as_echo "$as_me: WARNING: if you wanted to set the --build type, don't use --host.
+ If a cross compiler is detected then cross compile mode will be used" >&2
+ elif test "x$build_alias" != "x$host_alias"; then
+ cross_compiling=yes
+ fi
+fi
+
+ac_tool_prefix=
+test -n "$host_alias" && ac_tool_prefix=$host_alias-
+
+test "$silent" = yes && exec 6>/dev/null
+
+
+ac_pwd=`pwd` && test -n "$ac_pwd" &&
+ac_ls_di=`ls -di .` &&
+ac_pwd_ls_di=`cd "$ac_pwd" && ls -di .` ||
+ as_fn_error $? "working directory cannot be determined"
+test "X$ac_ls_di" = "X$ac_pwd_ls_di" ||
+ as_fn_error $? "pwd does not report name of working directory"
+
+
+# Find the source files, if location was not specified.
+if test -z "$srcdir"; then
+ ac_srcdir_defaulted=yes
+ # Try the directory containing this script, then the parent directory.
+ ac_confdir=`$as_dirname -- "$as_myself" ||
+$as_expr X"$as_myself" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+ X"$as_myself" : 'X\(//\)[^/]' \| \
+ X"$as_myself" : 'X\(//\)$' \| \
+ X"$as_myself" : 'X\(/\)' \| . 2>/dev/null ||
+$as_echo X"$as_myself" |
+ sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)[^/].*/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\).*/{
+ s//\1/
+ q
+ }
+ s/.*/./; q'`
+ srcdir=$ac_confdir
+ if test ! -r "$srcdir/$ac_unique_file"; then
+ srcdir=..
+ fi
+else
+ ac_srcdir_defaulted=no
+fi
+if test ! -r "$srcdir/$ac_unique_file"; then
+ test "$ac_srcdir_defaulted" = yes && srcdir="$ac_confdir or .."
+ as_fn_error $? "cannot find sources ($ac_unique_file) in $srcdir"
+fi
+ac_msg="sources are in $srcdir, but \`cd $srcdir' does not work"
+ac_abs_confdir=`(
+ cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error $? "$ac_msg"
+ pwd)`
+# When building in place, set srcdir=.
+if test "$ac_abs_confdir" = "$ac_pwd"; then
+ srcdir=.
+fi
+# Remove unnecessary trailing slashes from srcdir.
+# Double slashes in file names in object file debugging info
+# mess up M-x gdb in Emacs.
+case $srcdir in
+*/) srcdir=`expr "X$srcdir" : 'X\(.*[^/]\)' \| "X$srcdir" : 'X\(.*\)'`;;
+esac
+for ac_var in $ac_precious_vars; do
+ eval ac_env_${ac_var}_set=\${${ac_var}+set}
+ eval ac_env_${ac_var}_value=\$${ac_var}
+ eval ac_cv_env_${ac_var}_set=\${${ac_var}+set}
+ eval ac_cv_env_${ac_var}_value=\$${ac_var}
+done
+
+#
+# Report the --help message.
+#
+if test "$ac_init_help" = "long"; then
+ # Omit some internal or obsolete options to make the list less imposing.
+ # This message is too long to be a string in the A/UX 3.1 sh.
+ cat <<_ACEOF
+\`configure' configures libfabric 1.4.0rc2 to adapt to many kinds of systems.
+
+Usage: $0 [OPTION]... [VAR=VALUE]...
+
+To assign environment variables (e.g., CC, CFLAGS...), specify them as
+VAR=VALUE. See below for descriptions of some of the useful variables.
+
+Defaults for the options are specified in brackets.
+
+Configuration:
+ -h, --help display this help and exit
+ --help=short display options specific to this package
+ --help=recursive display the short help of all the included packages
+ -V, --version display version information and exit
+ -q, --quiet, --silent do not print \`checking ...' messages
+ --cache-file=FILE cache test results in FILE [disabled]
+ -C, --config-cache alias for \`--cache-file=config.cache'
+ -n, --no-create do not create output files
+ --srcdir=DIR find the sources in DIR [configure dir or \`..']
+
+Installation directories:
+ --prefix=PREFIX install architecture-independent files in PREFIX
+ [$ac_default_prefix]
+ --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX
+ [PREFIX]
+
+By default, \`make install' will install all the files in
+\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc. You can specify
+an installation prefix other than \`$ac_default_prefix' using \`--prefix',
+for instance \`--prefix=\$HOME'.
+
+For better control, use the options below.
+
+Fine tuning of the installation directories:
+ --bindir=DIR user executables [EPREFIX/bin]
+ --sbindir=DIR system admin executables [EPREFIX/sbin]
+ --libexecdir=DIR program executables [EPREFIX/libexec]
+ --sysconfdir=DIR read-only single-machine data [PREFIX/etc]
+ --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
+ --localstatedir=DIR modifiable single-machine data [PREFIX/var]
+ --libdir=DIR object code libraries [EPREFIX/lib]
+ --includedir=DIR C header files [PREFIX/include]
+ --oldincludedir=DIR C header files for non-gcc [/usr/include]
+ --datarootdir=DIR read-only arch.-independent data root [PREFIX/share]
+ --datadir=DIR read-only architecture-independent data [DATAROOTDIR]
+ --infodir=DIR info documentation [DATAROOTDIR/info]
+ --localedir=DIR locale-dependent data [DATAROOTDIR/locale]
+ --mandir=DIR man documentation [DATAROOTDIR/man]
+ --docdir=DIR documentation root [DATAROOTDIR/doc/libfabric]
+ --htmldir=DIR html documentation [DOCDIR]
+ --dvidir=DIR dvi documentation [DOCDIR]
+ --pdfdir=DIR pdf documentation [DOCDIR]
+ --psdir=DIR ps documentation [DOCDIR]
+_ACEOF
+
+ cat <<\_ACEOF
+
+Program names:
+ --program-prefix=PREFIX prepend PREFIX to installed program names
+ --program-suffix=SUFFIX append SUFFIX to installed program names
+ --program-transform-name=PROGRAM run sed PROGRAM on installed program names
+
+System types:
+ --build=BUILD configure for building on BUILD [guessed]
+ --host=HOST cross-compile to build programs to run on HOST [BUILD]
+_ACEOF
+fi
+
+if test -n "$ac_init_help"; then
+ case $ac_init_help in
+ short | recursive ) echo "Configuration of libfabric 1.4.0rc2:";;
+ esac
+ cat <<\_ACEOF
+
+Optional Features:
+ --disable-option-checking ignore unrecognized --enable/--with options
+ --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no)
+ --enable-FEATURE[=ARG] include FEATURE [ARG=yes]
+ --enable-silent-rules less verbose build output (undo: `make V=1')
+ --disable-silent-rules verbose build output (undo: `make V=0')
+ --enable-debug Enable debugging [default=no]
+
+ --enable-direct=[provider]
+ Enable direct calls to a fabric provider
+ [default=no]
+
+ --disable-dependency-tracking speeds up one-time build
+ --enable-dependency-tracking do not reject slow dependency extractors
+ --enable-shared[=PKGS] build shared libraries [default=yes]
+ --enable-static[=PKGS] build static libraries [default=yes]
+ --enable-fast-install[=PKGS]
+ optimize for fast installation [default=yes]
+ --disable-libtool-lock avoid locking (might break parallel builds)
+ --enable-embedded Enable embedded support (turns off symbol
+ versioning) [default=no]
+
+ --enable-psm[=yes|no|auto|dl|PATH]
+ Enable psm provider [default=auto] (yes: enable psm
+ provider; no: disable psm provider; auto: enable psm
+ provider if possible; dl: enable psm provider and
+ build as a loadable library; PATH: enable psm
+ provider and use psm installed under PATH)
+
+ --enable-psm2[=yes|no|auto|dl|PATH]
+ Enable psm2 provider [default=auto] (yes: enable
+ psm2 provider; no: disable psm2 provider; auto:
+ enable psm2 provider if possible; dl: enable psm2
+ provider and build as a loadable library; PATH:
+ enable psm2 provider and use psm2 installed under
+ PATH)
+
+ --enable-sockets[=yes|no|auto|dl|PATH]
+ Enable sockets provider [default=auto] (yes: enable
+ sockets provider; no: disable sockets provider;
+ auto: enable sockets provider if possible; dl:
+ enable sockets provider and build as a loadable
+ library; PATH: enable sockets provider and use
+ sockets installed under PATH)
+
+ --enable-verbs[=yes|no|auto|dl|PATH]
+ Enable verbs provider [default=auto] (yes: enable
+ verbs provider; no: disable verbs provider; auto:
+ enable verbs provider if possible; dl: enable verbs
+ provider and build as a loadable library; PATH:
+ enable verbs provider and use verbs installed under
+ PATH)
+
+ --enable-usnic[=yes|no|auto|dl|PATH]
+ Enable usnic provider [default=auto] (yes: enable
+ usnic provider; no: disable usnic provider; auto:
+ enable usnic provider if possible; dl: enable usnic
+ provider and build as a loadable library; PATH:
+ enable usnic provider and use usnic installed under
+ PATH)
+
+ --enable-mxm[=yes|no|auto|dl|PATH]
+ Enable mxm provider [default=auto] (yes: enable mxm
+ provider; no: disable mxm provider; auto: enable mxm
+ provider if possible; dl: enable mxm provider and
+ build as a loadable library; PATH: enable mxm
+ provider and use mxm installed under PATH)
+
+ --enable-gni[=yes|no|auto|dl|PATH]
+ Enable gni provider [default=auto] (yes: enable gni
+ provider; no: disable gni provider; auto: enable gni
+ provider if possible; dl: enable gni provider and
+ build as a loadable library; PATH: enable gni
+ provider and use gni installed under PATH)
+
+ --enable-xpmem Enable xpmem (gni provider) [default=yes]
+ --enable-udp[=yes|no|auto|dl|PATH]
+ Enable udp provider [default=auto] (yes: enable udp
+ provider; no: disable udp provider; auto: enable udp
+ provider if possible; dl: enable udp provider and
+ build as a loadable library; PATH: enable udp
+ provider and use udp installed under PATH)
+
+ --enable-rxm[=yes|no|auto|dl|PATH]
+ Enable rxm provider [default=auto] (yes: enable rxm
+ provider; no: disable rxm provider; auto: enable rxm
+ provider if possible; dl: enable rxm provider and
+ build as a loadable library; PATH: enable rxm
+ provider and use rxm installed under PATH)
+
+ --enable-rxd[=yes|no|auto|dl|PATH]
+ Enable rxd provider [default=auto] (yes: enable rxd
+ provider; no: disable rxd provider; auto: enable rxd
+ provider if possible; dl: enable rxd provider and
+ build as a loadable library; PATH: enable rxd
+ provider and use rxd installed under PATH)
+
+
+Optional Packages:
+ --with-PACKAGE[=ARG] use PACKAGE [ARG=yes]
+ --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no)
+ --with-valgrind Enable valgrind annotations [default=no]
+ --with-pic try to use only PIC/non-PIC objects [default=use
+ both]
+ --with-gnu-ld assume the C compiler uses GNU ld [default=no]
+ --with-dlopen dl-loadable provider support [default=yes]
+ --with-libnl(=DIR) Directory prefix for libnl (typically only necessary
+ if libnl is installed in a location that the
+ compiler/linker will not search by default)
+ --with-criterion Location for criterion unit testing framework
+ --with-kdreg Install directory for kdreg headers
+
+Some influential environment variables:
+ CC C compiler command
+ CFLAGS C compiler flags
+ LDFLAGS linker flags, e.g. -L<lib dir> if you have libraries in a
+ nonstandard directory <lib dir>
+ LIBS libraries to pass to the linker, e.g. -l<library>
+ CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I<include dir> if
+ you have headers in a nonstandard directory <include dir>
+ CPP C preprocessor
+ FI_PKG_CONFIG
+ path to pkg-config utility
+ FI_PKG_CONFIG_PATH
+ directories to add to pkg-config's search path
+ FI_PKG_CONFIG_LIBDIR
+ path overriding pkg-config's built-in search path
+ CRAY_GNI_HEADERS_CFLAGS
+ C compiler flags for CRAY_GNI_HEADERS, overriding pkg-config
+ CRAY_GNI_HEADERS_LIBS
+ linker flags for CRAY_GNI_HEADERS, overriding pkg-config
+ CRAY_UGNI_CFLAGS
+ C compiler flags for CRAY_UGNI, overriding pkg-config
+ CRAY_UGNI_LIBS
+ linker flags for CRAY_UGNI, overriding pkg-config
+ CRAY_ALPS_LLI_CFLAGS
+ C compiler flags for CRAY_ALPS_LLI, overriding pkg-config
+ CRAY_ALPS_LLI_LIBS
+ linker flags for CRAY_ALPS_LLI, overriding pkg-config
+ CRAY_ALPS_LLI_STATIC_LIBS
+ static linker flags for CRAY_ALPS_LLI, overriding pkg-config
+ CRAY_ALPS_UTIL_CFLAGS
+ C compiler flags for CRAY_ALPS_UTIL, overriding pkg-config
+ CRAY_ALPS_UTIL_LIBS
+ linker flags for CRAY_ALPS_UTIL, overriding pkg-config
+ CRAY_UDREG_CFLAGS
+ C compiler flags for CRAY_UDREG, overriding pkg-config
+ CRAY_UDREG_LIBS
+ linker flags for CRAY_UDREG, overriding pkg-config
+ CRAY_XPMEM_CFLAGS
+ C compiler flags for CRAY_XPMEM, overriding pkg-config
+ CRAY_XPMEM_LIBS
+ linker flags for CRAY_XPMEM, overriding pkg-config
+ CRAY_PMI_CFLAGS
+ C compiler flags for CRAY_PMI, overriding pkg-config
+ CRAY_PMI_LIBS
+ linker flags for CRAY_PMI, overriding pkg-config
+
+Use these variables to override the choices made by `configure' or to help
+it to find libraries and programs with nonstandard names/locations.
+
+Report bugs to <ofiwg at lists.openfabrics.org>.
+_ACEOF
+ac_status=$?
+fi
+
+if test "$ac_init_help" = "recursive"; then
+ # If there are subdirs, report their specific --help.
+ for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue
+ test -d "$ac_dir" ||
+ { cd "$srcdir" && ac_pwd=`pwd` && srcdir=. && test -d "$ac_dir"; } ||
+ continue
+ ac_builddir=.
+
+case "$ac_dir" in
+.) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;;
+*)
+ ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'`
+ # A ".." for each directory in $ac_dir_suffix.
+ ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'`
+ case $ac_top_builddir_sub in
+ "") ac_top_builddir_sub=. ac_top_build_prefix= ;;
+ *) ac_top_build_prefix=$ac_top_builddir_sub/ ;;
+ esac ;;
+esac
+ac_abs_top_builddir=$ac_pwd
+ac_abs_builddir=$ac_pwd$ac_dir_suffix
+# for backward compatibility:
+ac_top_builddir=$ac_top_build_prefix
+
+case $srcdir in
+ .) # We are building in place.
+ ac_srcdir=.
+ ac_top_srcdir=$ac_top_builddir_sub
+ ac_abs_top_srcdir=$ac_pwd ;;
+ [\\/]* | ?:[\\/]* ) # Absolute name.
+ ac_srcdir=$srcdir$ac_dir_suffix;
+ ac_top_srcdir=$srcdir
+ ac_abs_top_srcdir=$srcdir ;;
+ *) # Relative name.
+ ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix
+ ac_top_srcdir=$ac_top_build_prefix$srcdir
+ ac_abs_top_srcdir=$ac_pwd/$srcdir ;;
+esac
+ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix
+
+ cd "$ac_dir" || { ac_status=$?; continue; }
+ # Check for guested configure.
+ if test -f "$ac_srcdir/configure.gnu"; then
+ echo &&
+ $SHELL "$ac_srcdir/configure.gnu" --help=recursive
+ elif test -f "$ac_srcdir/configure"; then
+ echo &&
+ $SHELL "$ac_srcdir/configure" --help=recursive
+ else
+ $as_echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2
+ fi || ac_status=$?
+ cd "$ac_pwd" || { ac_status=$?; break; }
+ done
+fi
+
+test -n "$ac_init_help" && exit $ac_status
+if $ac_init_version; then
+ cat <<\_ACEOF
+libfabric configure 1.4.0rc2
+generated by GNU Autoconf 2.67
+
+Copyright (C) 2010 Free Software Foundation, Inc.
+This configure script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it.
+_ACEOF
+ exit
+fi
+
+## ------------------------ ##
+## Autoconf initialization. ##
+## ------------------------ ##
+
+# ac_fn_c_try_compile LINENO
+# --------------------------
+# Try to compile conftest.$ac_ext, and return whether this succeeded.
+ac_fn_c_try_compile ()
+{
+ as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+ rm -f conftest.$ac_objext
+ if { { ac_try="$ac_compile"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_compile") 2>conftest.err
+ ac_status=$?
+ if test -s conftest.err; then
+ grep -v '^ *+' conftest.err >conftest.er1
+ cat conftest.er1 >&5
+ mv -f conftest.er1 conftest.err
+ fi
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; } && {
+ test -z "$ac_c_werror_flag" ||
+ test ! -s conftest.err
+ } && test -s conftest.$ac_objext; then :
+ ac_retval=0
+else
+ $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ ac_retval=1
+fi
+ eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
+ as_fn_set_status $ac_retval
+
+} # ac_fn_c_try_compile
+
+# ac_fn_c_try_cpp LINENO
+# ----------------------
+# Try to preprocess conftest.$ac_ext, and return whether this succeeded.
+ac_fn_c_try_cpp ()
+{
+ as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+ if { { ac_try="$ac_cpp conftest.$ac_ext"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_cpp conftest.$ac_ext") 2>conftest.err
+ ac_status=$?
+ if test -s conftest.err; then
+ grep -v '^ *+' conftest.err >conftest.er1
+ cat conftest.er1 >&5
+ mv -f conftest.er1 conftest.err
+ fi
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; } > conftest.i && {
+ test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" ||
+ test ! -s conftest.err
+ }; then :
+ ac_retval=0
+else
+ $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ ac_retval=1
+fi
+ eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
+ as_fn_set_status $ac_retval
+
+} # ac_fn_c_try_cpp
+
+# ac_fn_c_try_run LINENO
+# ----------------------
+# Try to link conftest.$ac_ext, and return whether this succeeded. Assumes
+# that executables *can* be run.
+ac_fn_c_try_run ()
+{
+ as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+ if { { ac_try="$ac_link"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_link") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; } && { ac_try='./conftest$ac_exeext'
+ { { case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_try") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; }; then :
+ ac_retval=0
+else
+ $as_echo "$as_me: program exited with status $ac_status" >&5
+ $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ ac_retval=$ac_status
+fi
+ rm -rf conftest.dSYM conftest_ipa8_conftest.oo
+ eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
+ as_fn_set_status $ac_retval
+
+} # ac_fn_c_try_run
+
+# ac_fn_c_try_link LINENO
+# -----------------------
+# Try to link conftest.$ac_ext, and return whether this succeeded.
+ac_fn_c_try_link ()
+{
+ as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+ rm -f conftest.$ac_objext conftest$ac_exeext
+ if { { ac_try="$ac_link"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_link") 2>conftest.err
+ ac_status=$?
+ if test -s conftest.err; then
+ grep -v '^ *+' conftest.err >conftest.er1
+ cat conftest.er1 >&5
+ mv -f conftest.er1 conftest.err
+ fi
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; } && {
+ test -z "$ac_c_werror_flag" ||
+ test ! -s conftest.err
+ } && test -s conftest$ac_exeext && {
+ test "$cross_compiling" = yes ||
+ $as_test_x conftest$ac_exeext
+ }; then :
+ ac_retval=0
+else
+ $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ ac_retval=1
+fi
+ # Delete the IPA/IPO (Inter Procedural Analysis/Optimization) information
+ # created by the PGI compiler (conftest_ipa8_conftest.oo), as it would
+ # interfere with the next link command; also delete a directory that is
+ # left behind by Apple's compiler. We do this before executing the actions.
+ rm -rf conftest.dSYM conftest_ipa8_conftest.oo
+ eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
+ as_fn_set_status $ac_retval
+
+} # ac_fn_c_try_link
+
+# ac_fn_c_check_header_compile LINENO HEADER VAR INCLUDES
+# -------------------------------------------------------
+# Tests whether HEADER exists and can be compiled using the include files in
+# INCLUDES, setting the cache variable VAR accordingly.
+ac_fn_c_check_header_compile ()
+{
+ as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5
+$as_echo_n "checking for $2... " >&6; }
+if eval "test \"\${$3+set}\"" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+$4
+#include <$2>
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ eval "$3=yes"
+else
+ eval "$3=no"
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+eval ac_res=\$$3
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+$as_echo "$ac_res" >&6; }
+ eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
+
+} # ac_fn_c_check_header_compile
+
+# ac_fn_c_check_func LINENO FUNC VAR
+# ----------------------------------
+# Tests whether FUNC exists, setting the cache variable VAR accordingly
+ac_fn_c_check_func ()
+{
+ as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5
+$as_echo_n "checking for $2... " >&6; }
+if eval "test \"\${$3+set}\"" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+/* Define $2 to an innocuous variant, in case <limits.h> declares $2.
+ For example, HP-UX 11i <limits.h> declares gettimeofday. */
+#define $2 innocuous_$2
+
+/* System header to define __stub macros and hopefully few prototypes,
+ which can conflict with char $2 (); below.
+ Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+ <limits.h> exists even on freestanding compilers. */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $2
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char $2 ();
+/* The GNU C library defines this for functions which it implements
+ to always fail with ENOSYS. Some functions are actually named
+ something starting with __ and the normal name is an alias. */
+#if defined __stub_$2 || defined __stub___$2
+choke me
+#endif
+
+int
+main ()
+{
+return $2 ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ eval "$3=yes"
+else
+ eval "$3=no"
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+fi
+eval ac_res=\$$3
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+$as_echo "$ac_res" >&6; }
+ eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
+
+} # ac_fn_c_check_func
+
+# ac_fn_c_check_header_mongrel LINENO HEADER VAR INCLUDES
+# -------------------------------------------------------
+# Tests whether HEADER exists, giving a warning if it cannot be compiled using
+# the include files in INCLUDES and setting the cache variable VAR
+# accordingly.
+ac_fn_c_check_header_mongrel ()
+{
+ as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+ if eval "test \"\${$3+set}\"" = set; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5
+$as_echo_n "checking for $2... " >&6; }
+if eval "test \"\${$3+set}\"" = set; then :
+ $as_echo_n "(cached) " >&6
+fi
+eval ac_res=\$$3
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+$as_echo "$ac_res" >&6; }
+else
+ # Is the header compilable?
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 usability" >&5
+$as_echo_n "checking $2 usability... " >&6; }
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+$4
+#include <$2>
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ ac_header_compiler=yes
+else
+ ac_header_compiler=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_compiler" >&5
+$as_echo "$ac_header_compiler" >&6; }
+
+# Is the header present?
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 presence" >&5
+$as_echo_n "checking $2 presence... " >&6; }
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <$2>
+_ACEOF
+if ac_fn_c_try_cpp "$LINENO"; then :
+ ac_header_preproc=yes
+else
+ ac_header_preproc=no
+fi
+rm -f conftest.err conftest.i conftest.$ac_ext
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_preproc" >&5
+$as_echo "$ac_header_preproc" >&6; }
+
+# So? What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in #((
+ yes:no: )
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&5
+$as_echo "$as_me: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5
+$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;}
+ ;;
+ no:yes:* )
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: present but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: $2: present but cannot be compiled" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: check for missing prerequisite headers?" >&5
+$as_echo "$as_me: WARNING: $2: check for missing prerequisite headers?" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: see the Autoconf documentation" >&5
+$as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&5
+$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5
+$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;}
+( $as_echo "## ------------------------------------------ ##
+## Report this to ofiwg at lists.openfabrics.org ##
+## ------------------------------------------ ##"
+ ) | sed "s/^/$as_me: WARNING: /" >&2
+ ;;
+esac
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5
+$as_echo_n "checking for $2... " >&6; }
+if eval "test \"\${$3+set}\"" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ eval "$3=\$ac_header_compiler"
+fi
+eval ac_res=\$$3
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+$as_echo "$ac_res" >&6; }
+fi
+ eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
+
+} # ac_fn_c_check_header_mongrel
+
+# ac_fn_c_check_type LINENO TYPE VAR INCLUDES
+# -------------------------------------------
+# Tests whether TYPE exists after having included INCLUDES, setting cache
+# variable VAR accordingly.
+ac_fn_c_check_type ()
+{
+ as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5
+$as_echo_n "checking for $2... " >&6; }
+if eval "test \"\${$3+set}\"" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ eval "$3=no"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+$4
+int
+main ()
+{
+if (sizeof ($2))
+ return 0;
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+$4
+int
+main ()
+{
+if (sizeof (($2)))
+ return 0;
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+
+else
+ eval "$3=yes"
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+eval ac_res=\$$3
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+$as_echo "$ac_res" >&6; }
+ eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
+
+} # ac_fn_c_check_type
+
+# ac_fn_c_check_decl LINENO SYMBOL VAR INCLUDES
+# ---------------------------------------------
+# Tests whether SYMBOL is declared in INCLUDES, setting cache variable VAR
+# accordingly.
+ac_fn_c_check_decl ()
+{
+ as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+ as_decl_name=`echo $2|sed 's/ *(.*//'`
+ as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'`
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $as_decl_name is declared" >&5
+$as_echo_n "checking whether $as_decl_name is declared... " >&6; }
+if eval "test \"\${$3+set}\"" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+$4
+int
+main ()
+{
+#ifndef $as_decl_name
+#ifdef __cplusplus
+ (void) $as_decl_use;
+#else
+ (void) $as_decl_name;
+#endif
+#endif
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ eval "$3=yes"
+else
+ eval "$3=no"
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+eval ac_res=\$$3
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+$as_echo "$ac_res" >&6; }
+ eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;}
+
+} # ac_fn_c_check_decl
+cat >config.log <<_ACEOF
+This file contains any messages produced by compilers while
+running configure, to aid debugging if configure makes a mistake.
+
+It was created by libfabric $as_me 1.4.0rc2, which was
+generated by GNU Autoconf 2.67. Invocation command line was
+
+ $ $0 $@
+
+_ACEOF
+exec 5>>config.log
+{
+cat <<_ASUNAME
+## --------- ##
+## Platform. ##
+## --------- ##
+
+hostname = `(hostname || uname -n) 2>/dev/null | sed 1q`
+uname -m = `(uname -m) 2>/dev/null || echo unknown`
+uname -r = `(uname -r) 2>/dev/null || echo unknown`
+uname -s = `(uname -s) 2>/dev/null || echo unknown`
+uname -v = `(uname -v) 2>/dev/null || echo unknown`
+
+/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown`
+/bin/uname -X = `(/bin/uname -X) 2>/dev/null || echo unknown`
+
+/bin/arch = `(/bin/arch) 2>/dev/null || echo unknown`
+/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null || echo unknown`
+/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown`
+/usr/bin/hostinfo = `(/usr/bin/hostinfo) 2>/dev/null || echo unknown`
+/bin/machine = `(/bin/machine) 2>/dev/null || echo unknown`
+/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null || echo unknown`
+/bin/universe = `(/bin/universe) 2>/dev/null || echo unknown`
+
+_ASUNAME
+
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ $as_echo "PATH: $as_dir"
+ done
+IFS=$as_save_IFS
+
+} >&5
+
+cat >&5 <<_ACEOF
+
+
+## ----------- ##
+## Core tests. ##
+## ----------- ##
+
+_ACEOF
+
+
+# Keep a trace of the command line.
+# Strip out --no-create and --no-recursion so they do not pile up.
+# Strip out --silent because we don't want to record it for future runs.
+# Also quote any args containing shell meta-characters.
+# Make two passes to allow for proper duplicate-argument suppression.
+ac_configure_args=
+ac_configure_args0=
+ac_configure_args1=
+ac_must_keep_next=false
+for ac_pass in 1 2
+do
+ for ac_arg
+ do
+ case $ac_arg in
+ -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;;
+ -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+ | -silent | --silent | --silen | --sile | --sil)
+ continue ;;
+ *\'*)
+ ac_arg=`$as_echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;;
+ esac
+ case $ac_pass in
+ 1) as_fn_append ac_configure_args0 " '$ac_arg'" ;;
+ 2)
+ as_fn_append ac_configure_args1 " '$ac_arg'"
+ if test $ac_must_keep_next = true; then
+ ac_must_keep_next=false # Got value, back to normal.
+ else
+ case $ac_arg in
+ *=* | --config-cache | -C | -disable-* | --disable-* \
+ | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \
+ | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \
+ | -with-* | --with-* | -without-* | --without-* | --x)
+ case "$ac_configure_args0 " in
+ "$ac_configure_args1"*" '$ac_arg' "* ) continue ;;
+ esac
+ ;;
+ -* ) ac_must_keep_next=true ;;
+ esac
+ fi
+ as_fn_append ac_configure_args " '$ac_arg'"
+ ;;
+ esac
+ done
+done
+{ ac_configure_args0=; unset ac_configure_args0;}
+{ ac_configure_args1=; unset ac_configure_args1;}
+
+# When interrupted or exit'd, cleanup temporary files, and complete
+# config.log. We remove comments because anyway the quotes in there
+# would cause problems or look ugly.
+# WARNING: Use '\'' to represent an apostrophe within the trap.
+# WARNING: Do not start the trap code with a newline, due to a FreeBSD 4.0 bug.
+trap 'exit_status=$?
+ # Save into config.log some information that might help in debugging.
+ {
+ echo
+
+ $as_echo "## ---------------- ##
+## Cache variables. ##
+## ---------------- ##"
+ echo
+ # The following way of writing the cache mishandles newlines in values,
+(
+ for ac_var in `(set) 2>&1 | sed -n '\''s/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'\''`; do
+ eval ac_val=\$$ac_var
+ case $ac_val in #(
+ *${as_nl}*)
+ case $ac_var in #(
+ *_cv_*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5
+$as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;;
+ esac
+ case $ac_var in #(
+ _ | IFS | as_nl) ;; #(
+ BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #(
+ *) { eval $ac_var=; unset $ac_var;} ;;
+ esac ;;
+ esac
+ done
+ (set) 2>&1 |
+ case $as_nl`(ac_space='\'' '\''; set) 2>&1` in #(
+ *${as_nl}ac_space=\ *)
+ sed -n \
+ "s/'\''/'\''\\\\'\'''\''/g;
+ s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\''\\2'\''/p"
+ ;; #(
+ *)
+ sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p"
+ ;;
+ esac |
+ sort
+)
+ echo
+
+ $as_echo "## ----------------- ##
+## Output variables. ##
+## ----------------- ##"
+ echo
+ for ac_var in $ac_subst_vars
+ do
+ eval ac_val=\$$ac_var
+ case $ac_val in
+ *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;;
+ esac
+ $as_echo "$ac_var='\''$ac_val'\''"
+ done | sort
+ echo
+
+ if test -n "$ac_subst_files"; then
+ $as_echo "## ------------------- ##
+## File substitutions. ##
+## ------------------- ##"
+ echo
+ for ac_var in $ac_subst_files
+ do
+ eval ac_val=\$$ac_var
+ case $ac_val in
+ *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;;
+ esac
+ $as_echo "$ac_var='\''$ac_val'\''"
+ done | sort
+ echo
+ fi
+
+ if test -s confdefs.h; then
+ $as_echo "## ----------- ##
+## confdefs.h. ##
+## ----------- ##"
+ echo
+ cat confdefs.h
+ echo
+ fi
+ test "$ac_signal" != 0 &&
+ $as_echo "$as_me: caught signal $ac_signal"
+ $as_echo "$as_me: exit $exit_status"
+ } >&5
+ rm -f core *.core core.conftest.* &&
+ rm -f -r conftest* confdefs* conf$$* $ac_clean_files &&
+ exit $exit_status
+' 0
+for ac_signal in 1 2 13 15; do
+ trap 'ac_signal='$ac_signal'; as_fn_exit 1' $ac_signal
+done
+ac_signal=0
+
+# confdefs.h avoids OS command line length limits that DEFS can exceed.
+rm -f -r conftest* confdefs.h
+
+$as_echo "/* confdefs.h */" > confdefs.h
+
+# Predefined preprocessor variables.
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_NAME "$PACKAGE_NAME"
+_ACEOF
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_TARNAME "$PACKAGE_TARNAME"
+_ACEOF
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_VERSION "$PACKAGE_VERSION"
+_ACEOF
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_STRING "$PACKAGE_STRING"
+_ACEOF
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT"
+_ACEOF
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_URL "$PACKAGE_URL"
+_ACEOF
+
+
+# Let the site file select an alternate cache file if it wants to.
+# Prefer an explicitly selected file to automatically selected ones.
+ac_site_file1=NONE
+ac_site_file2=NONE
+if test -n "$CONFIG_SITE"; then
+ # We do not want a PATH search for config.site.
+ case $CONFIG_SITE in #((
+ -*) ac_site_file1=./$CONFIG_SITE;;
+ */*) ac_site_file1=$CONFIG_SITE;;
+ *) ac_site_file1=./$CONFIG_SITE;;
+ esac
+elif test "x$prefix" != xNONE; then
+ ac_site_file1=$prefix/share/config.site
+ ac_site_file2=$prefix/etc/config.site
+else
+ ac_site_file1=$ac_default_prefix/share/config.site
+ ac_site_file2=$ac_default_prefix/etc/config.site
+fi
+for ac_site_file in "$ac_site_file1" "$ac_site_file2"
+do
+ test "x$ac_site_file" = xNONE && continue
+ if test /dev/null != "$ac_site_file" && test -r "$ac_site_file"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: loading site script $ac_site_file" >&5
+$as_echo "$as_me: loading site script $ac_site_file" >&6;}
+ sed 's/^/| /' "$ac_site_file" >&5
+ . "$ac_site_file" \
+ || { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "failed to load site script $ac_site_file
+See \`config.log' for more details" "$LINENO" 5; }
+ fi
+done
+
+if test -r "$cache_file"; then
+ # Some versions of bash will fail to source /dev/null (special files
+ # actually), so we avoid doing that. DJGPP emulates it as a regular file.
+ if test /dev/null != "$cache_file" && test -f "$cache_file"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: loading cache $cache_file" >&5
+$as_echo "$as_me: loading cache $cache_file" >&6;}
+ case $cache_file in
+ [\\/]* | ?:[\\/]* ) . "$cache_file";;
+ *) . "./$cache_file";;
+ esac
+ fi
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: creating cache $cache_file" >&5
+$as_echo "$as_me: creating cache $cache_file" >&6;}
+ >$cache_file
+fi
+
+# Check that the precious variables saved in the cache have kept the same
+# value.
+ac_cache_corrupted=false
+for ac_var in $ac_precious_vars; do
+ eval ac_old_set=\$ac_cv_env_${ac_var}_set
+ eval ac_new_set=\$ac_env_${ac_var}_set
+ eval ac_old_val=\$ac_cv_env_${ac_var}_value
+ eval ac_new_val=\$ac_env_${ac_var}_value
+ case $ac_old_set,$ac_new_set in
+ set,)
+ { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5
+$as_echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;}
+ ac_cache_corrupted=: ;;
+ ,set)
+ { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was not set in the previous run" >&5
+$as_echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;}
+ ac_cache_corrupted=: ;;
+ ,);;
+ *)
+ if test "x$ac_old_val" != "x$ac_new_val"; then
+ # differences in whitespace do not lead to failure.
+ ac_old_val_w=`echo x $ac_old_val`
+ ac_new_val_w=`echo x $ac_new_val`
+ if test "$ac_old_val_w" != "$ac_new_val_w"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' has changed since the previous run:" >&5
+$as_echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;}
+ ac_cache_corrupted=:
+ else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&5
+$as_echo "$as_me: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&2;}
+ eval $ac_var=\$ac_old_val
+ fi
+ { $as_echo "$as_me:${as_lineno-$LINENO}: former value: \`$ac_old_val'" >&5
+$as_echo "$as_me: former value: \`$ac_old_val'" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: current value: \`$ac_new_val'" >&5
+$as_echo "$as_me: current value: \`$ac_new_val'" >&2;}
+ fi;;
+ esac
+ # Pass precious variables to config.status.
+ if test "$ac_new_set" = set; then
+ case $ac_new_val in
+ *\'*) ac_arg=$ac_var=`$as_echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;;
+ *) ac_arg=$ac_var=$ac_new_val ;;
+ esac
+ case " $ac_configure_args " in
+ *" '$ac_arg' "*) ;; # Avoid dups. Use of quotes ensures accuracy.
+ *) as_fn_append ac_configure_args " '$ac_arg'" ;;
+ esac
+ fi
+done
+if $ac_cache_corrupted; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5
+$as_echo "$as_me: error: changes in the environment can compromise the build" >&2;}
+ as_fn_error $? "run \`make distclean' and/or \`rm $cache_file' and start over" "$LINENO" 5
+fi
+## -------------------- ##
+## Main body of script. ##
+## -------------------- ##
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+
+ac_aux_dir=
+for ac_dir in config "$srcdir"/config; do
+ if test -f "$ac_dir/install-sh"; then
+ ac_aux_dir=$ac_dir
+ ac_install_sh="$ac_aux_dir/install-sh -c"
+ break
+ elif test -f "$ac_dir/install.sh"; then
+ ac_aux_dir=$ac_dir
+ ac_install_sh="$ac_aux_dir/install.sh -c"
+ break
+ elif test -f "$ac_dir/shtool"; then
+ ac_aux_dir=$ac_dir
+ ac_install_sh="$ac_aux_dir/shtool install -c"
+ break
+ fi
+done
+if test -z "$ac_aux_dir"; then
+ as_fn_error $? "cannot find install-sh, install.sh, or shtool in config \"$srcdir\"/config" "$LINENO" 5
+fi
+
+# These three variables are undocumented and unsupported,
+# and are intended to be withdrawn in a future Autoconf release.
+# They can cause serious problems if a builder's source tree is in a directory
+# whose full name contains unusual characters.
+ac_config_guess="$SHELL $ac_aux_dir/config.guess" # Please don't use this var.
+ac_config_sub="$SHELL $ac_aux_dir/config.sub" # Please don't use this var.
+ac_configure="$SHELL $ac_aux_dir/configure" # Please don't use this var.
+
+
+
+ac_config_headers="$ac_config_headers config.h"
+
+am__api_version='1.11'
+
+# Find a good install program. We prefer a C program (faster),
+# so one script is as good as another. But avoid the broken or
+# incompatible versions:
+# SysV /etc/install, /usr/sbin/install
+# SunOS /usr/etc/install
+# IRIX /sbin/install
+# AIX /bin/install
+# AmigaOS /C/install, which installs bootblocks on floppy discs
+# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag
+# AFS /usr/afsws/bin/install, which mishandles nonexistent args
+# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
+# OS/2's system install, which has a completely different semantic
+# ./install, which can be erroneously created by make from ./install.sh.
+# Reject install programs that cannot install multiple files.
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a BSD-compatible install" >&5
+$as_echo_n "checking for a BSD-compatible install... " >&6; }
+if test -z "$INSTALL"; then
+if test "${ac_cv_path_install+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ # Account for people who put trailing slashes in PATH elements.
+case $as_dir/ in #((
+ ./ | .// | /[cC]/* | \
+ /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \
+ ?:[\\/]os2[\\/]install[\\/]* | ?:[\\/]OS2[\\/]INSTALL[\\/]* | \
+ /usr/ucb/* ) ;;
+ *)
+ # OSF1 and SCO ODT 3.0 have their own names for install.
+ # Don't use installbsd from OSF since it installs stuff as root
+ # by default.
+ for ac_prog in ginstall scoinst install; do
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_prog$ac_exec_ext" && $as_test_x "$as_dir/$ac_prog$ac_exec_ext"; }; then
+ if test $ac_prog = install &&
+ grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+ # AIX install. It has an incompatible calling convention.
+ :
+ elif test $ac_prog = install &&
+ grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+ # program-specific install script used by HP pwplus--don't use.
+ :
+ else
+ rm -rf conftest.one conftest.two conftest.dir
+ echo one > conftest.one
+ echo two > conftest.two
+ mkdir conftest.dir
+ if "$as_dir/$ac_prog$ac_exec_ext" -c conftest.one conftest.two "`pwd`/conftest.dir" &&
+ test -s conftest.one && test -s conftest.two &&
+ test -s conftest.dir/conftest.one &&
+ test -s conftest.dir/conftest.two
+ then
+ ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c"
+ break 3
+ fi
+ fi
+ fi
+ done
+ done
+ ;;
+esac
+
+ done
+IFS=$as_save_IFS
+
+rm -rf conftest.one conftest.two conftest.dir
+
+fi
+ if test "${ac_cv_path_install+set}" = set; then
+ INSTALL=$ac_cv_path_install
+ else
+ # As a last resort, use the slow shell script. Don't cache a
+ # value for INSTALL within a source directory, because that will
+ # break other packages using the cache if that directory is
+ # removed, or if the value is a relative name.
+ INSTALL=$ac_install_sh
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $INSTALL" >&5
+$as_echo "$INSTALL" >&6; }
+
+# Use test -z because SunOS4 sh mishandles braces in ${var-val}.
+# It thinks the first close brace ends the variable substitution.
+test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}'
+
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}'
+
+test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether build environment is sane" >&5
+$as_echo_n "checking whether build environment is sane... " >&6; }
+# Just in case
+sleep 1
+echo timestamp > conftest.file
+# Reject unsafe characters in $srcdir or the absolute working directory
+# name. Accept space and tab only in the latter.
+am_lf='
+'
+case `pwd` in
+ *[\\\"\#\$\&\'\`$am_lf]*)
+ as_fn_error $? "unsafe absolute working directory name" "$LINENO" 5;;
+esac
+case $srcdir in
+ *[\\\"\#\$\&\'\`$am_lf\ \ ]*)
+ as_fn_error $? "unsafe srcdir value: \`$srcdir'" "$LINENO" 5;;
+esac
+
+# Do `set' in a subshell so we don't clobber the current shell's
+# arguments. Must try -L first in case configure is actually a
+# symlink; some systems play weird games with the mod time of symlinks
+# (eg FreeBSD returns the mod time of the symlink's containing
+# directory).
+if (
+ set X `ls -Lt "$srcdir/configure" conftest.file 2> /dev/null`
+ if test "$*" = "X"; then
+ # -L didn't work.
+ set X `ls -t "$srcdir/configure" conftest.file`
+ fi
+ rm -f conftest.file
+ if test "$*" != "X $srcdir/configure conftest.file" \
+ && test "$*" != "X conftest.file $srcdir/configure"; then
+
+ # If neither matched, then we have a broken ls. This can happen
+ # if, for instance, CONFIG_SHELL is bash and it inherits a
+ # broken ls alias from the environment. This has actually
+ # happened. Such a system could not be considered "sane".
+ as_fn_error $? "ls -t appears to fail. Make sure there is not a broken
+alias in your environment" "$LINENO" 5
+ fi
+
+ test "$2" = conftest.file
+ )
+then
+ # Ok.
+ :
+else
+ as_fn_error $? "newly created file is older than distributed files!
+Check your system clock" "$LINENO" 5
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+test "$program_prefix" != NONE &&
+ program_transform_name="s&^&$program_prefix&;$program_transform_name"
+# Use a double $ so make ignores it.
+test "$program_suffix" != NONE &&
+ program_transform_name="s&\$&$program_suffix&;$program_transform_name"
+# Double any \ or $.
+# By default was `s,x,x', remove it if useless.
+ac_script='s/[\\$]/&&/g;s/;s,x,x,$//'
+program_transform_name=`$as_echo "$program_transform_name" | sed "$ac_script"`
+
+# expand $ac_aux_dir to an absolute path
+am_aux_dir=`cd $ac_aux_dir && pwd`
+
+if test x"${MISSING+set}" != xset; then
+ case $am_aux_dir in
+ *\ * | *\ *)
+ MISSING="\${SHELL} \"$am_aux_dir/missing\"" ;;
+ *)
+ MISSING="\${SHELL} $am_aux_dir/missing" ;;
+ esac
+fi
+# Use eval to expand $SHELL
+if eval "$MISSING --run true"; then
+ am_missing_run="$MISSING --run "
+else
+ am_missing_run=
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: \`missing' script is too old or missing" >&5
+$as_echo "$as_me: WARNING: \`missing' script is too old or missing" >&2;}
+fi
+
+if test x"${install_sh}" != xset; then
+ case $am_aux_dir in
+ *\ * | *\ *)
+ install_sh="\${SHELL} '$am_aux_dir/install-sh'" ;;
+ *)
+ install_sh="\${SHELL} $am_aux_dir/install-sh"
+ esac
+fi
+
+# Installed binaries are usually stripped using `strip' when the user
+# run `make install-strip'. However `strip' might not be the right
+# tool to use in cross-compilation environments, therefore Automake
+# will honor the `STRIP' environment variable to overrule this program.
+if test "$cross_compiling" != no; then
+ if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}strip", so it can be a program name with args.
+set dummy ${ac_tool_prefix}strip; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_STRIP+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$STRIP"; then
+ ac_cv_prog_STRIP="$STRIP" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_STRIP="${ac_tool_prefix}strip"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+STRIP=$ac_cv_prog_STRIP
+if test -n "$STRIP"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $STRIP" >&5
+$as_echo "$STRIP" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_STRIP"; then
+ ac_ct_STRIP=$STRIP
+ # Extract the first word of "strip", so it can be a program name with args.
+set dummy strip; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ac_ct_STRIP+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_STRIP"; then
+ ac_cv_prog_ac_ct_STRIP="$ac_ct_STRIP" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_STRIP="strip"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_STRIP=$ac_cv_prog_ac_ct_STRIP
+if test -n "$ac_ct_STRIP"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_STRIP" >&5
+$as_echo "$ac_ct_STRIP" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+ if test "x$ac_ct_STRIP" = x; then
+ STRIP=":"
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ STRIP=$ac_ct_STRIP
+ fi
+else
+ STRIP="$ac_cv_prog_STRIP"
+fi
+
+fi
+INSTALL_STRIP_PROGRAM="\$(install_sh) -c -s"
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a thread-safe mkdir -p" >&5
+$as_echo_n "checking for a thread-safe mkdir -p... " >&6; }
+if test -z "$MKDIR_P"; then
+ if test "${ac_cv_path_mkdir+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH$PATH_SEPARATOR/opt/sfw/bin
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_prog in mkdir gmkdir; do
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ { test -f "$as_dir/$ac_prog$ac_exec_ext" && $as_test_x "$as_dir/$ac_prog$ac_exec_ext"; } || continue
+ case `"$as_dir/$ac_prog$ac_exec_ext" --version 2>&1` in #(
+ 'mkdir (GNU coreutils) '* | \
+ 'mkdir (coreutils) '* | \
+ 'mkdir (fileutils) '4.1*)
+ ac_cv_path_mkdir=$as_dir/$ac_prog$ac_exec_ext
+ break 3;;
+ esac
+ done
+ done
+ done
+IFS=$as_save_IFS
+
+fi
+
+ test -d ./--version && rmdir ./--version
+ if test "${ac_cv_path_mkdir+set}" = set; then
+ MKDIR_P="$ac_cv_path_mkdir -p"
+ else
+ # As a last resort, use the slow shell script. Don't cache a
+ # value for MKDIR_P within a source directory, because that will
+ # break other packages using the cache if that directory is
+ # removed, or if the value is a relative name.
+ MKDIR_P="$ac_install_sh -d"
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $MKDIR_P" >&5
+$as_echo "$MKDIR_P" >&6; }
+
+mkdir_p="$MKDIR_P"
+case $mkdir_p in
+ [\\/$]* | ?:[\\/]*) ;;
+ */*) mkdir_p="\$(top_builddir)/$mkdir_p" ;;
+esac
+
+for ac_prog in gawk mawk nawk awk
+do
+ # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_AWK+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$AWK"; then
+ ac_cv_prog_AWK="$AWK" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_AWK="$ac_prog"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+AWK=$ac_cv_prog_AWK
+if test -n "$AWK"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AWK" >&5
+$as_echo "$AWK" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ test -n "$AWK" && break
+done
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${MAKE-make} sets \$(MAKE)" >&5
+$as_echo_n "checking whether ${MAKE-make} sets \$(MAKE)... " >&6; }
+set x ${MAKE-make}
+ac_make=`$as_echo "$2" | sed 's/+/p/g; s/[^a-zA-Z0-9_]/_/g'`
+if eval "test \"\${ac_cv_prog_make_${ac_make}_set+set}\"" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat >conftest.make <<\_ACEOF
+SHELL = /bin/sh
+all:
+ @echo '@@@%%%=$(MAKE)=@@@%%%'
+_ACEOF
+# GNU make sometimes prints "make[1]: Entering ...", which would confuse us.
+case `${MAKE-make} -f conftest.make 2>/dev/null` in
+ *@@@%%%=?*=@@@%%%*)
+ eval ac_cv_prog_make_${ac_make}_set=yes;;
+ *)
+ eval ac_cv_prog_make_${ac_make}_set=no;;
+esac
+rm -f conftest.make
+fi
+if eval test \$ac_cv_prog_make_${ac_make}_set = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ SET_MAKE=
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ SET_MAKE="MAKE=${MAKE-make}"
+fi
+
+rm -rf .tst 2>/dev/null
+mkdir .tst 2>/dev/null
+if test -d .tst; then
+ am__leading_dot=.
+else
+ am__leading_dot=_
+fi
+rmdir .tst 2>/dev/null
+
+if test "`cd $srcdir && pwd`" != "`pwd`"; then
+ # Use -I$(srcdir) only when $(srcdir) != ., so that make's output
+ # is not polluted with repeated "-I."
+ am__isrc=' -I$(srcdir)'
+ # test to see if srcdir already configured
+ if test -f $srcdir/config.status; then
+ as_fn_error $? "source directory already configured; run \"make distclean\" there first" "$LINENO" 5
+ fi
+fi
+
+# test whether we have cygpath
+if test -z "$CYGPATH_W"; then
+ if (cygpath --version) >/dev/null 2>/dev/null; then
+ CYGPATH_W='cygpath -w'
+ else
+ CYGPATH_W=echo
+ fi
+fi
+
+
+# Define the identity of the package.
+ PACKAGE='libfabric'
+ VERSION='1.4.0rc2'
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE "$PACKAGE"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define VERSION "$VERSION"
+_ACEOF
+
+# Some tools Automake needs.
+
+ACLOCAL=${ACLOCAL-"${am_missing_run}aclocal-${am__api_version}"}
+
+
+AUTOCONF=${AUTOCONF-"${am_missing_run}autoconf"}
+
+
+AUTOMAKE=${AUTOMAKE-"${am_missing_run}automake-${am__api_version}"}
+
+
+AUTOHEADER=${AUTOHEADER-"${am_missing_run}autoheader"}
+
+
+MAKEINFO=${MAKEINFO-"${am_missing_run}makeinfo"}
+
+# We need awk for the "check" target. The system "awk" is bad on
+# some platforms.
+# Always define AMTAR for backward compatibility.
+
+AMTAR=${AMTAR-"${am_missing_run}tar"}
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to create a ustar tar archive" >&5
+$as_echo_n "checking how to create a ustar tar archive... " >&6; }
+# Loop over all known methods to create a tar archive until one works.
+_am_tools='gnutar plaintar pax cpio none'
+_am_tools=${am_cv_prog_tar_ustar-$_am_tools}
+# Do not fold the above two line into one, because Tru64 sh and
+# Solaris sh will not grok spaces in the rhs of `-'.
+for _am_tool in $_am_tools
+do
+ case $_am_tool in
+ gnutar)
+ for _am_tar in tar gnutar gtar;
+ do
+ { echo "$as_me:$LINENO: $_am_tar --version" >&5
+ ($_am_tar --version) >&5 2>&5
+ ac_status=$?
+ echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); } && break
+ done
+ am__tar="$_am_tar --format=ustar -chf - "'"$$tardir"'
+ am__tar_="$_am_tar --format=ustar -chf - "'"$tardir"'
+ am__untar="$_am_tar -xf -"
+ ;;
+ plaintar)
+ # Must skip GNU tar: if it does not support --format= it doesn't create
+ # ustar tarball either.
+ (tar --version) >/dev/null 2>&1 && continue
+ am__tar='tar chf - "$$tardir"'
+ am__tar_='tar chf - "$tardir"'
+ am__untar='tar xf -'
+ ;;
+ pax)
+ am__tar='pax -L -x ustar -w "$$tardir"'
+ am__tar_='pax -L -x ustar -w "$tardir"'
+ am__untar='pax -r'
+ ;;
+ cpio)
+ am__tar='find "$$tardir" -print | cpio -o -H ustar -L'
+ am__tar_='find "$tardir" -print | cpio -o -H ustar -L'
+ am__untar='cpio -i -H ustar -d'
+ ;;
+ none)
+ am__tar=false
+ am__tar_=false
+ am__untar=false
+ ;;
+ esac
+
+ # If the value was cached, stop now. We just wanted to have am__tar
+ # and am__untar set.
+ test -n "${am_cv_prog_tar_ustar}" && break
+
+ # tar/untar a dummy directory, and stop if the command works
+ rm -rf conftest.dir
+ mkdir conftest.dir
+ echo GrepMe > conftest.dir/file
+ { echo "$as_me:$LINENO: tardir=conftest.dir && eval $am__tar_ >conftest.tar" >&5
+ (tardir=conftest.dir && eval $am__tar_ >conftest.tar) >&5 2>&5
+ ac_status=$?
+ echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); }
+ rm -rf conftest.dir
+ if test -s conftest.tar; then
+ { echo "$as_me:$LINENO: $am__untar <conftest.tar" >&5
+ ($am__untar <conftest.tar) >&5 2>&5
+ ac_status=$?
+ echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); }
+ grep GrepMe conftest.dir/file >/dev/null 2>&1 && break
+ fi
+done
+rm -rf conftest.dir
+
+if test "${am_cv_prog_tar_ustar+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ am_cv_prog_tar_ustar=$_am_tool
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_prog_tar_ustar" >&5
+$as_echo "$am_cv_prog_tar_ustar" >&6; }
+
+
+
+
+
+# Check whether --enable-silent-rules was given.
+if test "${enable_silent_rules+set}" = set; then :
+ enableval=$enable_silent_rules;
+fi
+
+case $enable_silent_rules in
+yes) AM_DEFAULT_VERBOSITY=0;;
+no) AM_DEFAULT_VERBOSITY=1;;
+*) AM_DEFAULT_VERBOSITY=0;;
+esac
+AM_BACKSLASH='\'
+
+
+# Make sure we can run config.sub.
+$SHELL "$ac_aux_dir/config.sub" sun4 >/dev/null 2>&1 ||
+ as_fn_error $? "cannot run $SHELL $ac_aux_dir/config.sub" "$LINENO" 5
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking build system type" >&5
+$as_echo_n "checking build system type... " >&6; }
+if test "${ac_cv_build+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_build_alias=$build_alias
+test "x$ac_build_alias" = x &&
+ ac_build_alias=`$SHELL "$ac_aux_dir/config.guess"`
+test "x$ac_build_alias" = x &&
+ as_fn_error $? "cannot guess build type; you must specify one" "$LINENO" 5
+ac_cv_build=`$SHELL "$ac_aux_dir/config.sub" $ac_build_alias` ||
+ as_fn_error $? "$SHELL $ac_aux_dir/config.sub $ac_build_alias failed" "$LINENO" 5
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_build" >&5
+$as_echo "$ac_cv_build" >&6; }
+case $ac_cv_build in
+*-*-*) ;;
+*) as_fn_error $? "invalid value of canonical build" "$LINENO" 5;;
+esac
+build=$ac_cv_build
+ac_save_IFS=$IFS; IFS='-'
+set x $ac_cv_build
+shift
+build_cpu=$1
+build_vendor=$2
+shift; shift
+# Remember, the first character of IFS is used to create $*,
+# except with old shells:
+build_os=$*
+IFS=$ac_save_IFS
+case $build_os in *\ *) build_os=`echo "$build_os" | sed 's/ /-/g'`;; esac
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking host system type" >&5
+$as_echo_n "checking host system type... " >&6; }
+if test "${ac_cv_host+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test "x$host_alias" = x; then
+ ac_cv_host=$ac_cv_build
+else
+ ac_cv_host=`$SHELL "$ac_aux_dir/config.sub" $host_alias` ||
+ as_fn_error $? "$SHELL $ac_aux_dir/config.sub $host_alias failed" "$LINENO" 5
+fi
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_host" >&5
+$as_echo "$ac_cv_host" >&6; }
+case $ac_cv_host in
+*-*-*) ;;
+*) as_fn_error $? "invalid value of canonical host" "$LINENO" 5;;
+esac
+host=$ac_cv_host
+ac_save_IFS=$IFS; IFS='-'
+set x $ac_cv_host
+shift
+host_cpu=$1
+host_vendor=$2
+shift; shift
+# Remember, the first character of IFS is used to create $*,
+# except with old shells:
+host_os=$*
+IFS=$ac_save_IFS
+case $host_os in *\ *) host_os=`echo "$host_os" | sed 's/ /-/g'`;; esac
+
+
+
+macos=0
+linux=0
+freebsd=0
+
+case $host_os in
+*darwin*)
+ macos=1
+ ;;
+*linux*)
+ linux=1
+ ;;
+*freebsd*)
+ freebsd=1
+ ;;
+*)
+ as_fn_error $? "libfabric only builds on Linux, OS X, and FreeBSD" "$LINENO" 5
+ ;;
+esac
+
+ if test "x$macos" = "x1"; then
+ MACOS_TRUE=
+ MACOS_FALSE='#'
+else
+ MACOS_TRUE='#'
+ MACOS_FALSE=
+fi
+
+ if test "x$linux" = "x1"; then
+ LINUX_TRUE=
+ LINUX_FALSE='#'
+else
+ LINUX_TRUE='#'
+ LINUX_FALSE=
+fi
+
+ if test "x$freebsd" = "x1"; then
+ FREEBSD_TRUE=
+ FREEBSD_FALSE='#'
+else
+ FREEBSD_TRUE='#'
+ FREEBSD_FALSE=
+fi
+
+
+base_c_warn_flags="-Wall -Wundef -Wpointer-arith"
+
+# Check whether --enable-debug was given.
+if test "${enable_debug+set}" = set; then :
+ enableval=$enable_debug; CFLAGS="-g -O0 ${base_c_warn_flags} -Wextra -Wno-unused-parameter -Wno-sign-compare -Wno-missing-field-initializers $CFLAGS"
+ dbg=1
+else
+ enable_debug=no
+ dbg=0
+fi
+
+
+
+cat >>confdefs.h <<_ACEOF
+#define ENABLE_DEBUG $dbg
+_ACEOF
+
+
+# Fix autoconf's habit of setting CFLAGS="-g -O2" by default while still
+# allowing the user to explicitly set CFLAGS=""
+: ${CFLAGS="-fvisibility=hidden -O2 -DNDEBUG ${base_c_warn_flags}"}
+
+# AM PROG_AR did not exist pre AM 1.11.x (where x is somewhere >0 and
+# <3), but it is necessary in AM 1.12.x.
+
+
+
+# Check whether --with-valgrind was given.
+if test "${with_valgrind+set}" = set; then :
+ withval=$with_valgrind;
+fi
+
+
+if test "$with_valgrind" != "" && test "$with_valgrind" != "no"; then
+
+$as_echo "#define INCLUDE_VALGRIND 1" >>confdefs.h
+
+ if test -d $with_valgrind; then
+ CPPFLAGS="$CPPFLAGS -I$with_valgrind/include"
+ fi
+fi
+
+# Check whether --enable-direct was given.
+if test "${enable_direct+set}" = set; then :
+ enableval=$enable_direct;
+else
+ enable_direct=no
+fi
+
+
+DEPDIR="${am__leading_dot}deps"
+
+ac_config_commands="$ac_config_commands depfiles"
+
+
+am_make=${MAKE-make}
+cat > confinc << 'END'
+am__doit:
+ @echo this is the am__doit target
+.PHONY: am__doit
+END
+# If we don't find an include directive, just comment out the code.
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for style of include used by $am_make" >&5
+$as_echo_n "checking for style of include used by $am_make... " >&6; }
+am__include="#"
+am__quote=
+_am_result=none
+# First try GNU make style include.
+echo "include confinc" > confmf
+# Ignore all kinds of additional output from `make'.
+case `$am_make -s -f confmf 2> /dev/null` in #(
+*the\ am__doit\ target*)
+ am__include=include
+ am__quote=
+ _am_result=GNU
+ ;;
+esac
+# Now try BSD make style include.
+if test "$am__include" = "#"; then
+ echo '.include "confinc"' > confmf
+ case `$am_make -s -f confmf 2> /dev/null` in #(
+ *the\ am__doit\ target*)
+ am__include=.include
+ am__quote="\""
+ _am_result=BSD
+ ;;
+ esac
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $_am_result" >&5
+$as_echo "$_am_result" >&6; }
+rm -f confinc confmf
+
+# Check whether --enable-dependency-tracking was given.
+if test "${enable_dependency_tracking+set}" = set; then :
+ enableval=$enable_dependency_tracking;
+fi
+
+if test "x$enable_dependency_tracking" != xno; then
+ am_depcomp="$ac_aux_dir/depcomp"
+ AMDEPBACKSLASH='\'
+fi
+ if test "x$enable_dependency_tracking" != xno; then
+ AMDEP_TRUE=
+ AMDEP_FALSE='#'
+else
+ AMDEP_TRUE='#'
+ AMDEP_FALSE=
+fi
+
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}gcc; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_CC+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$CC"; then
+ ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_CC="${ac_tool_prefix}gcc"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
+$as_echo "$CC" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+ ac_ct_CC=$CC
+ # Extract the first word of "gcc", so it can be a program name with args.
+set dummy gcc; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_CC"; then
+ ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_CC="gcc"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5
+$as_echo "$ac_ct_CC" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+ if test "x$ac_ct_CC" = x; then
+ CC=""
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ CC=$ac_ct_CC
+ fi
+else
+ CC="$ac_cv_prog_CC"
+fi
+
+if test -z "$CC"; then
+ if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}cc; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_CC+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$CC"; then
+ ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_CC="${ac_tool_prefix}cc"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
+$as_echo "$CC" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ fi
+fi
+if test -z "$CC"; then
+ # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_CC+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$CC"; then
+ ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+ ac_prog_rejected=no
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then
+ ac_prog_rejected=yes
+ continue
+ fi
+ ac_cv_prog_CC="cc"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+if test $ac_prog_rejected = yes; then
+ # We found a bogon in the path, so make sure we never use it.
+ set dummy $ac_cv_prog_CC
+ shift
+ if test $# != 0; then
+ # We chose a different compiler from the bogus one.
+ # However, it has the same basename, so the bogon will be chosen
+ # first if we set CC to just the basename; use the full file name.
+ shift
+ ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@"
+ fi
+fi
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
+$as_echo "$CC" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$CC"; then
+ if test -n "$ac_tool_prefix"; then
+ for ac_prog in cl.exe
+ do
+ # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_CC+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$CC"; then
+ ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_CC="$ac_tool_prefix$ac_prog"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
+$as_echo "$CC" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ test -n "$CC" && break
+ done
+fi
+if test -z "$CC"; then
+ ac_ct_CC=$CC
+ for ac_prog in cl.exe
+do
+ # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_CC"; then
+ ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_CC="$ac_prog"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5
+$as_echo "$ac_ct_CC" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ test -n "$ac_ct_CC" && break
+done
+
+ if test "x$ac_ct_CC" = x; then
+ CC=""
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ CC=$ac_ct_CC
+ fi
+fi
+
+fi
+
+
+test -z "$CC" && { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "no acceptable C compiler found in \$PATH
+See \`config.log' for more details" "$LINENO" 5; }
+
+# Provide some information about the compiler.
+$as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5
+set X $ac_compile
+ac_compiler=$2
+for ac_option in --version -v -V -qversion; do
+ { { ac_try="$ac_compiler $ac_option >&5"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_compiler $ac_option >&5") 2>conftest.err
+ ac_status=$?
+ if test -s conftest.err; then
+ sed '10a\
+... rest of stderr output deleted ...
+ 10q' conftest.err >conftest.er1
+ cat conftest.er1 >&5
+ fi
+ rm -f conftest.er1 conftest.err
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }
+done
+
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files a.out a.out.dSYM a.exe b.out"
+# Try to create an executable without -o first, disregard a.out.
+# It will help us diagnose broken compilers, and finding out an intuition
+# of exeext.
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the C compiler works" >&5
+$as_echo_n "checking whether the C compiler works... " >&6; }
+ac_link_default=`$as_echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'`
+
+# The possible output files:
+ac_files="a.out conftest.exe conftest a.exe a_out.exe b.out conftest.*"
+
+ac_rmfiles=
+for ac_file in $ac_files
+do
+ case $ac_file in
+ *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;;
+ * ) ac_rmfiles="$ac_rmfiles $ac_file";;
+ esac
+done
+rm -f $ac_rmfiles
+
+if { { ac_try="$ac_link_default"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_link_default") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then :
+ # Autoconf-2.13 could set the ac_cv_exeext variable to `no'.
+# So ignore a value of `no', otherwise this would lead to `EXEEXT = no'
+# in a Makefile. We should not override ac_cv_exeext if it was cached,
+# so that the user can short-circuit this test for compilers unknown to
+# Autoconf.
+for ac_file in $ac_files ''
+do
+ test -f "$ac_file" || continue
+ case $ac_file in
+ *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj )
+ ;;
+ [ab].out )
+ # We found the default executable, but exeext='' is most
+ # certainly right.
+ break;;
+ *.* )
+ if test "${ac_cv_exeext+set}" = set && test "$ac_cv_exeext" != no;
+ then :; else
+ ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+ fi
+ # We set ac_cv_exeext here because the later test for it is not
+ # safe: cross compilers may not add the suffix if given an `-o'
+ # argument, so we may need to know it at that point already.
+ # Even if this section looks crufty: it has the advantage of
+ # actually working.
+ break;;
+ * )
+ break;;
+ esac
+done
+test "$ac_cv_exeext" = no && ac_cv_exeext=
+
+else
+ ac_file=''
+fi
+if test -z "$ac_file"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+$as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error 77 "C compiler cannot create executables
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler default output file name" >&5
+$as_echo_n "checking for C compiler default output file name... " >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_file" >&5
+$as_echo "$ac_file" >&6; }
+ac_exeext=$ac_cv_exeext
+
+rm -f -r a.out a.out.dSYM a.exe conftest$ac_cv_exeext b.out
+ac_clean_files=$ac_clean_files_save
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of executables" >&5
+$as_echo_n "checking for suffix of executables... " >&6; }
+if { { ac_try="$ac_link"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_link") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then :
+ # If both `conftest.exe' and `conftest' are `present' (well, observable)
+# catch `conftest.exe'. For instance with Cygwin, `ls conftest' will
+# work properly (i.e., refer to `conftest.exe'), while it won't with
+# `rm'.
+for ac_file in conftest.exe conftest conftest.*; do
+ test -f "$ac_file" || continue
+ case $ac_file in
+ *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;;
+ *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+ break;;
+ * ) break;;
+ esac
+done
+else
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details" "$LINENO" 5; }
+fi
+rm -f conftest conftest$ac_cv_exeext
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5
+$as_echo "$ac_cv_exeext" >&6; }
+
+rm -f conftest.$ac_ext
+EXEEXT=$ac_cv_exeext
+ac_exeext=$EXEEXT
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <stdio.h>
+int
+main ()
+{
+FILE *f = fopen ("conftest.out", "w");
+ return ferror (f) || fclose (f) != 0;
+
+ ;
+ return 0;
+}
+_ACEOF
+ac_clean_files="$ac_clean_files conftest.out"
+# Check that the compiler produces executables we can run. If not, either
+# the compiler is broken, or we cross compile.
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are cross compiling" >&5
+$as_echo_n "checking whether we are cross compiling... " >&6; }
+if test "$cross_compiling" != yes; then
+ { { ac_try="$ac_link"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_link") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }
+ if { ac_try='./conftest$ac_cv_exeext'
+ { { case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_try") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; }; then
+ cross_compiling=no
+ else
+ if test "$cross_compiling" = maybe; then
+ cross_compiling=yes
+ else
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "cannot run C compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details" "$LINENO" 5; }
+ fi
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $cross_compiling" >&5
+$as_echo "$cross_compiling" >&6; }
+
+rm -f conftest.$ac_ext conftest$ac_cv_exeext conftest.out
+ac_clean_files=$ac_clean_files_save
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5
+$as_echo_n "checking for suffix of object files... " >&6; }
+if test "${ac_cv_objext+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+rm -f conftest.o conftest.obj
+if { { ac_try="$ac_compile"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_compile") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then :
+ for ac_file in conftest.o conftest.obj conftest.*; do
+ test -f "$ac_file" || continue;
+ case $ac_file in
+ *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM ) ;;
+ *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'`
+ break;;
+ esac
+done
+else
+ $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "cannot compute suffix of object files: cannot compile
+See \`config.log' for more details" "$LINENO" 5; }
+fi
+rm -f conftest.$ac_cv_objext conftest.$ac_ext
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_objext" >&5
+$as_echo "$ac_cv_objext" >&6; }
+OBJEXT=$ac_cv_objext
+ac_objext=$OBJEXT
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C compiler" >&5
+$as_echo_n "checking whether we are using the GNU C compiler... " >&6; }
+if test "${ac_cv_c_compiler_gnu+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+#ifndef __GNUC__
+ choke me
+#endif
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ ac_compiler_gnu=yes
+else
+ ac_compiler_gnu=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_c_compiler_gnu=$ac_compiler_gnu
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5
+$as_echo "$ac_cv_c_compiler_gnu" >&6; }
+if test $ac_compiler_gnu = yes; then
+ GCC=yes
+else
+ GCC=
+fi
+ac_test_CFLAGS=${CFLAGS+set}
+ac_save_CFLAGS=$CFLAGS
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5
+$as_echo_n "checking whether $CC accepts -g... " >&6; }
+if test "${ac_cv_prog_cc_g+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_save_c_werror_flag=$ac_c_werror_flag
+ ac_c_werror_flag=yes
+ ac_cv_prog_cc_g=no
+ CFLAGS="-g"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ ac_cv_prog_cc_g=yes
+else
+ CFLAGS=""
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+
+else
+ ac_c_werror_flag=$ac_save_c_werror_flag
+ CFLAGS="-g"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ ac_cv_prog_cc_g=yes
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+ ac_c_werror_flag=$ac_save_c_werror_flag
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5
+$as_echo "$ac_cv_prog_cc_g" >&6; }
+if test "$ac_test_CFLAGS" = set; then
+ CFLAGS=$ac_save_CFLAGS
+elif test $ac_cv_prog_cc_g = yes; then
+ if test "$GCC" = yes; then
+ CFLAGS="-g -O2"
+ else
+ CFLAGS="-g"
+ fi
+else
+ if test "$GCC" = yes; then
+ CFLAGS="-O2"
+ else
+ CFLAGS=
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $CC option to accept ISO C89" >&5
+$as_echo_n "checking for $CC option to accept ISO C89... " >&6; }
+if test "${ac_cv_prog_cc_c89+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_cv_prog_cc_c89=no
+ac_save_CC=$CC
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <stdarg.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+/* Most of the following tests are stolen from RCS 5.7's src/conf.sh. */
+struct buf { int x; };
+FILE * (*rcsopen) (struct buf *, struct stat *, int);
+static char *e (p, i)
+ char **p;
+ int i;
+{
+ return p[i];
+}
+static char *f (char * (*g) (char **, int), char **p, ...)
+{
+ char *s;
+ va_list v;
+ va_start (v,p);
+ s = g (p, va_arg (v,int));
+ va_end (v);
+ return s;
+}
+
+/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has
+ function prototypes and stuff, but not '\xHH' hex character constants.
+ These don't provoke an error unfortunately, instead are silently treated
+ as 'x'. The following induces an error, until -std is added to get
+ proper ANSI mode. Curiously '\x00'!='x' always comes out true, for an
+ array size at least. It's necessary to write '\x00'==0 to get something
+ that's true only with -std. */
+int osf4_cc_array ['\x00' == 0 ? 1 : -1];
+
+/* IBM C 6 for AIX is almost-ANSI by default, but it replaces macro parameters
+ inside strings and character constants. */
+#define FOO(x) 'x'
+int xlc6_cc_array[FOO(a) == 'x' ? 1 : -1];
+
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int);
+int argc;
+char **argv;
+int
+main ()
+{
+return f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1];
+ ;
+ return 0;
+}
+_ACEOF
+for ac_arg in '' -qlanglvl=extc89 -qlanglvl=ansi -std \
+ -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__"
+do
+ CC="$ac_save_CC $ac_arg"
+ if ac_fn_c_try_compile "$LINENO"; then :
+ ac_cv_prog_cc_c89=$ac_arg
+fi
+rm -f core conftest.err conftest.$ac_objext
+ test "x$ac_cv_prog_cc_c89" != "xno" && break
+done
+rm -f conftest.$ac_ext
+CC=$ac_save_CC
+
+fi
+# AC_CACHE_VAL
+case "x$ac_cv_prog_cc_c89" in
+ x)
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: none needed" >&5
+$as_echo "none needed" >&6; } ;;
+ xno)
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5
+$as_echo "unsupported" >&6; } ;;
+ *)
+ CC="$CC $ac_cv_prog_cc_c89"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5
+$as_echo "$ac_cv_prog_cc_c89" >&6; } ;;
+esac
+if test "x$ac_cv_prog_cc_c89" != xno; then :
+
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+depcc="$CC" am_compiler_list=
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking dependency style of $depcc" >&5
+$as_echo_n "checking dependency style of $depcc... " >&6; }
+if test "${am_cv_CC_dependencies_compiler_type+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -z "$AMDEP_TRUE" && test -f "$am_depcomp"; then
+ # We make a subdir and do the tests there. Otherwise we can end up
+ # making bogus files that we don't know about and never remove. For
+ # instance it was reported that on HP-UX the gcc test will end up
+ # making a dummy file named `D' -- because `-MD' means `put the output
+ # in D'.
+ mkdir conftest.dir
+ # Copy depcomp to subdir because otherwise we won't find it if we're
+ # using a relative directory.
+ cp "$am_depcomp" conftest.dir
+ cd conftest.dir
+ # We will build objects and dependencies in a subdirectory because
+ # it helps to detect inapplicable dependency modes. For instance
+ # both Tru64's cc and ICC support -MD to output dependencies as a
+ # side effect of compilation, but ICC will put the dependencies in
+ # the current directory while Tru64 will put them in the object
+ # directory.
+ mkdir sub
+
+ am_cv_CC_dependencies_compiler_type=none
+ if test "$am_compiler_list" = ""; then
+ am_compiler_list=`sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < ./depcomp`
+ fi
+ am__universal=false
+ case " $depcc " in #(
+ *\ -arch\ *\ -arch\ *) am__universal=true ;;
+ esac
+
+ for depmode in $am_compiler_list; do
+ # Setup a source with many dependencies, because some compilers
+ # like to wrap large dependency lists on column 80 (with \), and
+ # we should not choose a depcomp mode which is confused by this.
+ #
+ # We need to recreate these files for each test, as the compiler may
+ # overwrite some of them when testing with obscure command lines.
+ # This happens at least with the AIX C compiler.
+ : > sub/conftest.c
+ for i in 1 2 3 4 5 6; do
+ echo '#include "conftst'$i'.h"' >> sub/conftest.c
+ # Using `: > sub/conftst$i.h' creates only sub/conftst1.h with
+ # Solaris 8's {/usr,}/bin/sh.
+ touch sub/conftst$i.h
+ done
+ echo "${am__include} ${am__quote}sub/conftest.Po${am__quote}" > confmf
+
+ # We check with `-c' and `-o' for the sake of the "dashmstdout"
+ # mode. It turns out that the SunPro C++ compiler does not properly
+ # handle `-M -o', and we need to detect this. Also, some Intel
+ # versions had trouble with output in subdirs
+ am__obj=sub/conftest.${OBJEXT-o}
+ am__minus_obj="-o $am__obj"
+ case $depmode in
+ gcc)
+ # This depmode causes a compiler race in universal mode.
+ test "$am__universal" = false || continue
+ ;;
+ nosideeffect)
+ # after this tag, mechanisms are not by side-effect, so they'll
+ # only be used when explicitly requested
+ if test "x$enable_dependency_tracking" = xyes; then
+ continue
+ else
+ break
+ fi
+ ;;
+ msvisualcpp | msvcmsys)
+ # This compiler won't grok `-c -o', but also, the minuso test has
+ # not run yet. These depmodes are late enough in the game, and
+ # so weak that their functioning should not be impacted.
+ am__obj=conftest.${OBJEXT-o}
+ am__minus_obj=
+ ;;
+ none) break ;;
+ esac
+ if depmode=$depmode \
+ source=sub/conftest.c object=$am__obj \
+ depfile=sub/conftest.Po tmpdepfile=sub/conftest.TPo \
+ $SHELL ./depcomp $depcc -c $am__minus_obj sub/conftest.c \
+ >/dev/null 2>conftest.err &&
+ grep sub/conftst1.h sub/conftest.Po > /dev/null 2>&1 &&
+ grep sub/conftst6.h sub/conftest.Po > /dev/null 2>&1 &&
+ grep $am__obj sub/conftest.Po > /dev/null 2>&1 &&
+ ${MAKE-make} -s -f confmf > /dev/null 2>&1; then
+ # icc doesn't choke on unknown options, it will just issue warnings
+ # or remarks (even with -Werror). So we grep stderr for any message
+ # that says an option was ignored or not supported.
+ # When given -MP, icc 7.0 and 7.1 complain thusly:
+ # icc: Command line warning: ignoring option '-M'; no argument required
+ # The diagnosis changed in icc 8.0:
+ # icc: Command line remark: option '-MP' not supported
+ if (grep 'ignoring option' conftest.err ||
+ grep 'not supported' conftest.err) >/dev/null 2>&1; then :; else
+ am_cv_CC_dependencies_compiler_type=$depmode
+ break
+ fi
+ fi
+ done
+
+ cd ..
+ rm -rf conftest.dir
+else
+ am_cv_CC_dependencies_compiler_type=none
+fi
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $am_cv_CC_dependencies_compiler_type" >&5
+$as_echo "$am_cv_CC_dependencies_compiler_type" >&6; }
+CCDEPMODE=depmode=$am_cv_CC_dependencies_compiler_type
+
+ if
+ test "x$enable_dependency_tracking" != xno \
+ && test "$am_cv_CC_dependencies_compiler_type" = gcc3; then
+ am__fastdepCC_TRUE=
+ am__fastdepCC_FALSE='#'
+else
+ am__fastdepCC_TRUE='#'
+ am__fastdepCC_FALSE=
+fi
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $CC option to accept ISO C99" >&5
+$as_echo_n "checking for $CC option to accept ISO C99... " >&6; }
+if test "${ac_cv_prog_cc_c99+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_cv_prog_cc_c99=no
+ac_save_CC=$CC
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <wchar.h>
+#include <stdio.h>
+
+// Check varargs macros. These examples are taken from C99 6.10.3.5.
+#define debug(...) fprintf (stderr, __VA_ARGS__)
+#define showlist(...) puts (#__VA_ARGS__)
+#define report(test,...) ((test) ? puts (#test) : printf (__VA_ARGS__))
+static void
+test_varargs_macros (void)
+{
+ int x = 1234;
+ int y = 5678;
+ debug ("Flag");
+ debug ("X = %d\n", x);
+ showlist (The first, second, and third items.);
+ report (x>y, "x is %d but y is %d", x, y);
+}
+
+// Check long long types.
+#define BIG64 18446744073709551615ull
+#define BIG32 4294967295ul
+#define BIG_OK (BIG64 / BIG32 == 4294967297ull && BIG64 % BIG32 == 0)
+#if !BIG_OK
+ your preprocessor is broken;
+#endif
+#if BIG_OK
+#else
+ your preprocessor is broken;
+#endif
+static long long int bignum = -9223372036854775807LL;
+static unsigned long long int ubignum = BIG64;
+
+struct incomplete_array
+{
+ int datasize;
+ double data[];
+};
+
+struct named_init {
+ int number;
+ const wchar_t *name;
+ double average;
+};
+
+typedef const char *ccp;
+
+static inline int
+test_restrict (ccp restrict text)
+{
+ // See if C++-style comments work.
+ // Iterate through items via the restricted pointer.
+ // Also check for declarations in for loops.
+ for (unsigned int i = 0; *(text+i) != '\0'; ++i)
+ continue;
+ return 0;
+}
+
+// Check varargs and va_copy.
+static void
+test_varargs (const char *format, ...)
+{
+ va_list args;
+ va_start (args, format);
+ va_list args_copy;
+ va_copy (args_copy, args);
+
+ const char *str;
+ int number;
+ float fnumber;
+
+ while (*format)
+ {
+ switch (*format++)
+ {
+ case 's': // string
+ str = va_arg (args_copy, const char *);
+ break;
+ case 'd': // int
+ number = va_arg (args_copy, int);
+ break;
+ case 'f': // float
+ fnumber = va_arg (args_copy, double);
+ break;
+ default:
+ break;
+ }
+ }
+ va_end (args_copy);
+ va_end (args);
+}
+
+int
+main ()
+{
+
+ // Check bool.
+ _Bool success = false;
+
+ // Check restrict.
+ if (test_restrict ("String literal") == 0)
+ success = true;
+ char *restrict newvar = "Another string";
+
+ // Check varargs.
+ test_varargs ("s, d' f .", "string", 65, 34.234);
+ test_varargs_macros ();
+
+ // Check flexible array members.
+ struct incomplete_array *ia =
+ malloc (sizeof (struct incomplete_array) + (sizeof (double) * 10));
+ ia->datasize = 10;
+ for (int i = 0; i < ia->datasize; ++i)
+ ia->data[i] = i * 1.234;
+
+ // Check named initializers.
+ struct named_init ni = {
+ .number = 34,
+ .name = L"Test wide string",
+ .average = 543.34343,
+ };
+
+ ni.number = 58;
+
+ int dynamic_array[ni.number];
+ dynamic_array[ni.number - 1] = 543;
+
+ // work around unused variable warnings
+ return (!success || bignum == 0LL || ubignum == 0uLL || newvar[0] == 'x'
+ || dynamic_array[ni.number - 1] != 543);
+
+ ;
+ return 0;
+}
+_ACEOF
+for ac_arg in '' -std=gnu99 -std=c99 -c99 -AC99 -xc99=all -qlanglvl=extc99
+do
+ CC="$ac_save_CC $ac_arg"
+ if ac_fn_c_try_compile "$LINENO"; then :
+ ac_cv_prog_cc_c99=$ac_arg
+fi
+rm -f core conftest.err conftest.$ac_objext
+ test "x$ac_cv_prog_cc_c99" != "xno" && break
+done
+rm -f conftest.$ac_ext
+CC=$ac_save_CC
+
+fi
+# AC_CACHE_VAL
+case "x$ac_cv_prog_cc_c99" in
+ x)
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: none needed" >&5
+$as_echo "none needed" >&6; } ;;
+ xno)
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5
+$as_echo "unsupported" >&6; } ;;
+ *)
+ CC="$CC $ac_cv_prog_cc_c99"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c99" >&5
+$as_echo "$ac_cv_prog_cc_c99" >&6; } ;;
+esac
+if test "x$ac_cv_prog_cc_c99" != xno; then :
+
+fi
+
+
+if test "x$CC" != xcc; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC and cc understand -c and -o together" >&5
+$as_echo_n "checking whether $CC and cc understand -c and -o together... " >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether cc understands -c and -o together" >&5
+$as_echo_n "checking whether cc understands -c and -o together... " >&6; }
+fi
+set dummy $CC; ac_cc=`$as_echo "$2" |
+ sed 's/[^a-zA-Z0-9_]/_/g;s/^[0-9]/_/'`
+if eval "test \"\${ac_cv_prog_cc_${ac_cc}_c_o+set}\"" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+# Make sure it works both with $CC and with simple cc.
+# We do the test twice because some compilers refuse to overwrite an
+# existing .o file with -o, though they will create one.
+ac_try='$CC -c conftest.$ac_ext -o conftest2.$ac_objext >&5'
+rm -f conftest2.*
+if { { case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_try") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; } &&
+ test -f conftest2.$ac_objext && { { case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_try") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; };
+then
+ eval ac_cv_prog_cc_${ac_cc}_c_o=yes
+ if test "x$CC" != xcc; then
+ # Test first that cc exists at all.
+ if { ac_try='cc -c conftest.$ac_ext >&5'
+ { { case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_try") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; }; then
+ ac_try='cc -c conftest.$ac_ext -o conftest2.$ac_objext >&5'
+ rm -f conftest2.*
+ if { { case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_try") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; } &&
+ test -f conftest2.$ac_objext && { { case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+$as_echo "$ac_try_echo"; } >&5
+ (eval "$ac_try") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; };
+ then
+ # cc works too.
+ :
+ else
+ # cc exists but doesn't like -o.
+ eval ac_cv_prog_cc_${ac_cc}_c_o=no
+ fi
+ fi
+ fi
+else
+ eval ac_cv_prog_cc_${ac_cc}_c_o=no
+fi
+rm -f core conftest*
+
+fi
+if eval test \$ac_cv_prog_cc_${ac_cc}_c_o = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+$as_echo "#define NO_MINUS_C_MINUS_O 1" >>confdefs.h
+
+fi
+
+# FIXME: we rely on the cache variable name because
+# there is no other way.
+set dummy $CC
+am_cc=`echo $2 | sed 's/[^a-zA-Z0-9_]/_/g;s/^[0-9]/_/'`
+eval am_t=\$ac_cv_prog_cc_${am_cc}_c_o
+if test "$am_t" != yes; then
+ # Losing compiler, so override with the script.
+ # FIXME: It is wrong to rewrite CC.
+ # But if we don't then we get into trouble of one sort or another.
+ # A longer-term fix would be to have automake use am__CC in this case,
+ # and then we could set am__CC="\$(top_srcdir)/compile \$(CC)"
+ CC="$am_aux_dir/compile $CC"
+fi
+
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to run the C preprocessor" >&5
+$as_echo_n "checking how to run the C preprocessor... " >&6; }
+# On Suns, sometimes $CPP names a directory.
+if test -n "$CPP" && test -d "$CPP"; then
+ CPP=
+fi
+if test -z "$CPP"; then
+ if test "${ac_cv_prog_CPP+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ # Double quotes because CPP needs to be expanded
+ for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp"
+ do
+ ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+ # Use a header file that comes with gcc, so configuring glibc
+ # with a fresh cross-compiler works.
+ # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+ # <limits.h> exists even on freestanding compilers.
+ # On the NeXT, cc -E runs the code through the compiler's parser,
+ # not just through cpp. "Syntax error" is here to catch this case.
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+ Syntax error
+_ACEOF
+if ac_fn_c_try_cpp "$LINENO"; then :
+
+else
+ # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.i conftest.$ac_ext
+
+ # OK, works on sane cases. Now check whether nonexistent headers
+ # can be detected and how.
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <ac_nonexistent.h>
+_ACEOF
+if ac_fn_c_try_cpp "$LINENO"; then :
+ # Broken: success on invalid input.
+continue
+else
+ # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.i conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.i conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then :
+ break
+fi
+
+ done
+ ac_cv_prog_CPP=$CPP
+
+fi
+ CPP=$ac_cv_prog_CPP
+else
+ ac_cv_prog_CPP=$CPP
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $CPP" >&5
+$as_echo "$CPP" >&6; }
+ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+ # Use a header file that comes with gcc, so configuring glibc
+ # with a fresh cross-compiler works.
+ # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+ # <limits.h> exists even on freestanding compilers.
+ # On the NeXT, cc -E runs the code through the compiler's parser,
+ # not just through cpp. "Syntax error" is here to catch this case.
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+ Syntax error
+_ACEOF
+if ac_fn_c_try_cpp "$LINENO"; then :
+
+else
+ # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.i conftest.$ac_ext
+
+ # OK, works on sane cases. Now check whether nonexistent headers
+ # can be detected and how.
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <ac_nonexistent.h>
+_ACEOF
+if ac_fn_c_try_cpp "$LINENO"; then :
+ # Broken: success on invalid input.
+continue
+else
+ # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.i conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.i conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then :
+
+else
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details" "$LINENO" 5; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for grep that handles long lines and -e" >&5
+$as_echo_n "checking for grep that handles long lines and -e... " >&6; }
+if test "${ac_cv_path_GREP+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -z "$GREP"; then
+ ac_path_GREP_found=false
+ # Loop through the user's path and test for each of PROGNAME-LIST
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_prog in grep ggrep; do
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ ac_path_GREP="$as_dir/$ac_prog$ac_exec_ext"
+ { test -f "$ac_path_GREP" && $as_test_x "$ac_path_GREP"; } || continue
+# Check for GNU ac_path_GREP and select it if it is found.
+ # Check for GNU $ac_path_GREP
+case `"$ac_path_GREP" --version 2>&1` in
+*GNU*)
+ ac_cv_path_GREP="$ac_path_GREP" ac_path_GREP_found=:;;
+*)
+ ac_count=0
+ $as_echo_n 0123456789 >"conftest.in"
+ while :
+ do
+ cat "conftest.in" "conftest.in" >"conftest.tmp"
+ mv "conftest.tmp" "conftest.in"
+ cp "conftest.in" "conftest.nl"
+ $as_echo 'GREP' >> "conftest.nl"
+ "$ac_path_GREP" -e 'GREP$' -e '-(cannot match)-' < "conftest.nl" >"conftest.out" 2>/dev/null || break
+ diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break
+ as_fn_arith $ac_count + 1 && ac_count=$as_val
+ if test $ac_count -gt ${ac_path_GREP_max-0}; then
+ # Best one so far, save it but keep looking for a better one
+ ac_cv_path_GREP="$ac_path_GREP"
+ ac_path_GREP_max=$ac_count
+ fi
+ # 10*(2^10) chars as input seems more than enough
+ test $ac_count -gt 10 && break
+ done
+ rm -f conftest.in conftest.tmp conftest.nl conftest.out;;
+esac
+
+ $ac_path_GREP_found && break 3
+ done
+ done
+ done
+IFS=$as_save_IFS
+ if test -z "$ac_cv_path_GREP"; then
+ as_fn_error $? "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5
+ fi
+else
+ ac_cv_path_GREP=$GREP
+fi
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_GREP" >&5
+$as_echo "$ac_cv_path_GREP" >&6; }
+ GREP="$ac_cv_path_GREP"
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for egrep" >&5
+$as_echo_n "checking for egrep... " >&6; }
+if test "${ac_cv_path_EGREP+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if echo a | $GREP -E '(a|b)' >/dev/null 2>&1
+ then ac_cv_path_EGREP="$GREP -E"
+ else
+ if test -z "$EGREP"; then
+ ac_path_EGREP_found=false
+ # Loop through the user's path and test for each of PROGNAME-LIST
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_prog in egrep; do
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ ac_path_EGREP="$as_dir/$ac_prog$ac_exec_ext"
+ { test -f "$ac_path_EGREP" && $as_test_x "$ac_path_EGREP"; } || continue
+# Check for GNU ac_path_EGREP and select it if it is found.
+ # Check for GNU $ac_path_EGREP
+case `"$ac_path_EGREP" --version 2>&1` in
+*GNU*)
+ ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_found=:;;
+*)
+ ac_count=0
+ $as_echo_n 0123456789 >"conftest.in"
+ while :
+ do
+ cat "conftest.in" "conftest.in" >"conftest.tmp"
+ mv "conftest.tmp" "conftest.in"
+ cp "conftest.in" "conftest.nl"
+ $as_echo 'EGREP' >> "conftest.nl"
+ "$ac_path_EGREP" 'EGREP$' < "conftest.nl" >"conftest.out" 2>/dev/null || break
+ diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break
+ as_fn_arith $ac_count + 1 && ac_count=$as_val
+ if test $ac_count -gt ${ac_path_EGREP_max-0}; then
+ # Best one so far, save it but keep looking for a better one
+ ac_cv_path_EGREP="$ac_path_EGREP"
+ ac_path_EGREP_max=$ac_count
+ fi
+ # 10*(2^10) chars as input seems more than enough
+ test $ac_count -gt 10 && break
+ done
+ rm -f conftest.in conftest.tmp conftest.nl conftest.out;;
+esac
+
+ $ac_path_EGREP_found && break 3
+ done
+ done
+ done
+IFS=$as_save_IFS
+ if test -z "$ac_cv_path_EGREP"; then
+ as_fn_error $? "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5
+ fi
+else
+ ac_cv_path_EGREP=$EGREP
+fi
+
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_EGREP" >&5
+$as_echo "$ac_cv_path_EGREP" >&6; }
+ EGREP="$ac_cv_path_EGREP"
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for ANSI C header files" >&5
+$as_echo_n "checking for ANSI C header files... " >&6; }
+if test "${ac_cv_header_stdc+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ ac_cv_header_stdc=yes
+else
+ ac_cv_header_stdc=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+ # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+ $EGREP "memchr" >/dev/null 2>&1; then :
+
+else
+ ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+ # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+ $EGREP "free" >/dev/null 2>&1; then :
+
+else
+ ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+ # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+ if test "$cross_compiling" = yes; then :
+ :
+else
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <ctype.h>
+#include <stdlib.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+ (('a' <= (c) && (c) <= 'i') \
+ || ('j' <= (c) && (c) <= 'r') \
+ || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+ int i;
+ for (i = 0; i < 256; i++)
+ if (XOR (islower (i), ISLOWER (i))
+ || toupper (i) != TOUPPER (i))
+ return 2;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_run "$LINENO"; then :
+
+else
+ ac_cv_header_stdc=no
+fi
+rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \
+ conftest.$ac_objext conftest.beam conftest.$ac_ext
+fi
+
+fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdc" >&5
+$as_echo "$ac_cv_header_stdc" >&6; }
+if test $ac_cv_header_stdc = yes; then
+
+$as_echo "#define STDC_HEADERS 1" >>confdefs.h
+
+fi
+
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for typeof syntax and keyword spelling" >&5
+$as_echo_n "checking for typeof syntax and keyword spelling... " >&6; }
+if test "${ac_cv_c_typeof+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_cv_c_typeof=no
+ for ac_kw in typeof __typeof__ no; do
+ test $ac_kw = no && break
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+ int value;
+ typedef struct {
+ char a [1
+ + ! (($ac_kw (value))
+ (($ac_kw (value)) 0 < ($ac_kw (value)) -1
+ ? ($ac_kw (value)) - 1
+ : ~ (~ ($ac_kw (value)) 0
+ << sizeof ($ac_kw (value)))))]; }
+ ac__typeof_type_;
+ return
+ (! ((void) ((ac__typeof_type_ *) 0), 0));
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ ac_cv_c_typeof=$ac_kw
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+ test $ac_cv_c_typeof != no && break
+ done
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_typeof" >&5
+$as_echo "$ac_cv_c_typeof" >&6; }
+ if test $ac_cv_c_typeof != no; then
+
+$as_echo "#define HAVE_TYPEOF 1" >>confdefs.h
+
+ if test $ac_cv_c_typeof != typeof; then
+
+cat >>confdefs.h <<_ACEOF
+#define typeof $ac_cv_c_typeof
+_ACEOF
+
+ fi
+ fi
+
+
+case `pwd` in
+ *\ * | *\ *)
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Libtool does not cope well with whitespace in \`pwd\`" >&5
+$as_echo "$as_me: WARNING: Libtool does not cope well with whitespace in \`pwd\`" >&2;} ;;
+esac
+
+
+
+macro_version='2.2.6b'
+macro_revision='1.3017'
+
+
+
+
+
+
+
+
+
+
+
+
+
+ltmain="$ac_aux_dir/ltmain.sh"
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a sed that does not truncate output" >&5
+$as_echo_n "checking for a sed that does not truncate output... " >&6; }
+if test "${ac_cv_path_SED+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_script=s/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/
+ for ac_i in 1 2 3 4 5 6 7; do
+ ac_script="$ac_script$as_nl$ac_script"
+ done
+ echo "$ac_script" 2>/dev/null | sed 99q >conftest.sed
+ { ac_script=; unset ac_script;}
+ if test -z "$SED"; then
+ ac_path_SED_found=false
+ # Loop through the user's path and test for each of PROGNAME-LIST
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_prog in sed gsed; do
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ ac_path_SED="$as_dir/$ac_prog$ac_exec_ext"
+ { test -f "$ac_path_SED" && $as_test_x "$ac_path_SED"; } || continue
+# Check for GNU ac_path_SED and select it if it is found.
+ # Check for GNU $ac_path_SED
+case `"$ac_path_SED" --version 2>&1` in
+*GNU*)
+ ac_cv_path_SED="$ac_path_SED" ac_path_SED_found=:;;
+*)
+ ac_count=0
+ $as_echo_n 0123456789 >"conftest.in"
+ while :
+ do
+ cat "conftest.in" "conftest.in" >"conftest.tmp"
+ mv "conftest.tmp" "conftest.in"
+ cp "conftest.in" "conftest.nl"
+ $as_echo '' >> "conftest.nl"
+ "$ac_path_SED" -f conftest.sed < "conftest.nl" >"conftest.out" 2>/dev/null || break
+ diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break
+ as_fn_arith $ac_count + 1 && ac_count=$as_val
+ if test $ac_count -gt ${ac_path_SED_max-0}; then
+ # Best one so far, save it but keep looking for a better one
+ ac_cv_path_SED="$ac_path_SED"
+ ac_path_SED_max=$ac_count
+ fi
+ # 10*(2^10) chars as input seems more than enough
+ test $ac_count -gt 10 && break
+ done
+ rm -f conftest.in conftest.tmp conftest.nl conftest.out;;
+esac
+
+ $ac_path_SED_found && break 3
+ done
+ done
+ done
+IFS=$as_save_IFS
+ if test -z "$ac_cv_path_SED"; then
+ as_fn_error $? "no acceptable sed could be found in \$PATH" "$LINENO" 5
+ fi
+else
+ ac_cv_path_SED=$SED
+fi
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_SED" >&5
+$as_echo "$ac_cv_path_SED" >&6; }
+ SED="$ac_cv_path_SED"
+ rm -f conftest.sed
+
+test -z "$SED" && SED=sed
+Xsed="$SED -e 1s/^X//"
+
+
+
+
+
+
+
+
+
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for fgrep" >&5
+$as_echo_n "checking for fgrep... " >&6; }
+if test "${ac_cv_path_FGREP+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if echo 'ab*c' | $GREP -F 'ab*c' >/dev/null 2>&1
+ then ac_cv_path_FGREP="$GREP -F"
+ else
+ if test -z "$FGREP"; then
+ ac_path_FGREP_found=false
+ # Loop through the user's path and test for each of PROGNAME-LIST
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_prog in fgrep; do
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ ac_path_FGREP="$as_dir/$ac_prog$ac_exec_ext"
+ { test -f "$ac_path_FGREP" && $as_test_x "$ac_path_FGREP"; } || continue
+# Check for GNU ac_path_FGREP and select it if it is found.
+ # Check for GNU $ac_path_FGREP
+case `"$ac_path_FGREP" --version 2>&1` in
+*GNU*)
+ ac_cv_path_FGREP="$ac_path_FGREP" ac_path_FGREP_found=:;;
+*)
+ ac_count=0
+ $as_echo_n 0123456789 >"conftest.in"
+ while :
+ do
+ cat "conftest.in" "conftest.in" >"conftest.tmp"
+ mv "conftest.tmp" "conftest.in"
+ cp "conftest.in" "conftest.nl"
+ $as_echo 'FGREP' >> "conftest.nl"
+ "$ac_path_FGREP" FGREP < "conftest.nl" >"conftest.out" 2>/dev/null || break
+ diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break
+ as_fn_arith $ac_count + 1 && ac_count=$as_val
+ if test $ac_count -gt ${ac_path_FGREP_max-0}; then
+ # Best one so far, save it but keep looking for a better one
+ ac_cv_path_FGREP="$ac_path_FGREP"
+ ac_path_FGREP_max=$ac_count
+ fi
+ # 10*(2^10) chars as input seems more than enough
+ test $ac_count -gt 10 && break
+ done
+ rm -f conftest.in conftest.tmp conftest.nl conftest.out;;
+esac
+
+ $ac_path_FGREP_found && break 3
+ done
+ done
+ done
+IFS=$as_save_IFS
+ if test -z "$ac_cv_path_FGREP"; then
+ as_fn_error $? "no acceptable fgrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5
+ fi
+else
+ ac_cv_path_FGREP=$FGREP
+fi
+
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_FGREP" >&5
+$as_echo "$ac_cv_path_FGREP" >&6; }
+ FGREP="$ac_cv_path_FGREP"
+
+
+test -z "$GREP" && GREP=grep
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# Check whether --with-gnu-ld was given.
+if test "${with_gnu_ld+set}" = set; then :
+ withval=$with_gnu_ld; test "$withval" = no || with_gnu_ld=yes
+else
+ with_gnu_ld=no
+fi
+
+ac_prog=ld
+if test "$GCC" = yes; then
+ # Check if gcc -print-prog-name=ld gives a path.
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ld used by $CC" >&5
+$as_echo_n "checking for ld used by $CC... " >&6; }
+ case $host in
+ *-*-mingw*)
+ # gcc leaves a trailing carriage return which upsets mingw
+ ac_prog=`($CC -print-prog-name=ld) 2>&5 | tr -d '\015'` ;;
+ *)
+ ac_prog=`($CC -print-prog-name=ld) 2>&5` ;;
+ esac
+ case $ac_prog in
+ # Accept absolute paths.
+ [\\/]* | ?:[\\/]*)
+ re_direlt='/[^/][^/]*/\.\./'
+ # Canonicalize the pathname of ld
+ ac_prog=`$ECHO "$ac_prog"| $SED 's%\\\\%/%g'`
+ while $ECHO "$ac_prog" | $GREP "$re_direlt" > /dev/null 2>&1; do
+ ac_prog=`$ECHO $ac_prog| $SED "s%$re_direlt%/%"`
+ done
+ test -z "$LD" && LD="$ac_prog"
+ ;;
+ "")
+ # If it fails, then pretend we aren't using GCC.
+ ac_prog=ld
+ ;;
+ *)
+ # If it is relative, then search for the first ld in PATH.
+ with_gnu_ld=unknown
+ ;;
+ esac
+elif test "$with_gnu_ld" = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for GNU ld" >&5
+$as_echo_n "checking for GNU ld... " >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for non-GNU ld" >&5
+$as_echo_n "checking for non-GNU ld... " >&6; }
+fi
+if test "${lt_cv_path_LD+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -z "$LD"; then
+ lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR
+ for ac_dir in $PATH; do
+ IFS="$lt_save_ifs"
+ test -z "$ac_dir" && ac_dir=.
+ if test -f "$ac_dir/$ac_prog" || test -f "$ac_dir/$ac_prog$ac_exeext"; then
+ lt_cv_path_LD="$ac_dir/$ac_prog"
+ # Check to see if the program is GNU ld. I'd rather use --version,
+ # but apparently some variants of GNU ld only accept -v.
+ # Break only if it was the GNU/non-GNU ld that we prefer.
+ case `"$lt_cv_path_LD" -v 2>&1 </dev/null` in
+ *GNU* | *'with BFD'*)
+ test "$with_gnu_ld" != no && break
+ ;;
+ *)
+ test "$with_gnu_ld" != yes && break
+ ;;
+ esac
+ fi
+ done
+ IFS="$lt_save_ifs"
+else
+ lt_cv_path_LD="$LD" # Let the user override the test with a path.
+fi
+fi
+
+LD="$lt_cv_path_LD"
+if test -n "$LD"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LD" >&5
+$as_echo "$LD" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+test -z "$LD" && as_fn_error $? "no acceptable ld found in \$PATH" "$LINENO" 5
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if the linker ($LD) is GNU ld" >&5
+$as_echo_n "checking if the linker ($LD) is GNU ld... " >&6; }
+if test "${lt_cv_prog_gnu_ld+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ # I'd rather use --version here, but apparently some GNU lds only accept -v.
+case `$LD -v 2>&1 </dev/null` in
+*GNU* | *'with BFD'*)
+ lt_cv_prog_gnu_ld=yes
+ ;;
+*)
+ lt_cv_prog_gnu_ld=no
+ ;;
+esac
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_gnu_ld" >&5
+$as_echo "$lt_cv_prog_gnu_ld" >&6; }
+with_gnu_ld=$lt_cv_prog_gnu_ld
+
+
+
+
+
+
+
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for BSD- or MS-compatible name lister (nm)" >&5
+$as_echo_n "checking for BSD- or MS-compatible name lister (nm)... " >&6; }
+if test "${lt_cv_path_NM+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$NM"; then
+ # Let the user override the test.
+ lt_cv_path_NM="$NM"
+else
+ lt_nm_to_check="${ac_tool_prefix}nm"
+ if test -n "$ac_tool_prefix" && test "$build" = "$host"; then
+ lt_nm_to_check="$lt_nm_to_check nm"
+ fi
+ for lt_tmp_nm in $lt_nm_to_check; do
+ lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR
+ for ac_dir in $PATH /usr/ccs/bin/elf /usr/ccs/bin /usr/ucb /bin; do
+ IFS="$lt_save_ifs"
+ test -z "$ac_dir" && ac_dir=.
+ tmp_nm="$ac_dir/$lt_tmp_nm"
+ if test -f "$tmp_nm" || test -f "$tmp_nm$ac_exeext" ; then
+ # Check to see if the nm accepts a BSD-compat flag.
+ # Adding the `sed 1q' prevents false positives on HP-UX, which says:
+ # nm: unknown option "B" ignored
+ # Tru64's nm complains that /dev/null is an invalid object file
+ case `"$tmp_nm" -B /dev/null 2>&1 | sed '1q'` in
+ */dev/null* | *'Invalid file or object type'*)
+ lt_cv_path_NM="$tmp_nm -B"
+ break
+ ;;
+ *)
+ case `"$tmp_nm" -p /dev/null 2>&1 | sed '1q'` in
+ */dev/null*)
+ lt_cv_path_NM="$tmp_nm -p"
+ break
+ ;;
+ *)
+ lt_cv_path_NM=${lt_cv_path_NM="$tmp_nm"} # keep the first match, but
+ continue # so that we can try to find one that supports BSD flags
+ ;;
+ esac
+ ;;
+ esac
+ fi
+ done
+ IFS="$lt_save_ifs"
+ done
+ : ${lt_cv_path_NM=no}
+fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_path_NM" >&5
+$as_echo "$lt_cv_path_NM" >&6; }
+if test "$lt_cv_path_NM" != "no"; then
+ NM="$lt_cv_path_NM"
+else
+ # Didn't find any BSD compatible name lister, look for dumpbin.
+ if test -n "$ac_tool_prefix"; then
+ for ac_prog in "dumpbin -symbols" "link -dump -symbols"
+ do
+ # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_DUMPBIN+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$DUMPBIN"; then
+ ac_cv_prog_DUMPBIN="$DUMPBIN" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_DUMPBIN="$ac_tool_prefix$ac_prog"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+DUMPBIN=$ac_cv_prog_DUMPBIN
+if test -n "$DUMPBIN"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $DUMPBIN" >&5
+$as_echo "$DUMPBIN" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ test -n "$DUMPBIN" && break
+ done
+fi
+if test -z "$DUMPBIN"; then
+ ac_ct_DUMPBIN=$DUMPBIN
+ for ac_prog in "dumpbin -symbols" "link -dump -symbols"
+do
+ # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ac_ct_DUMPBIN+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_DUMPBIN"; then
+ ac_cv_prog_ac_ct_DUMPBIN="$ac_ct_DUMPBIN" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_DUMPBIN="$ac_prog"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_DUMPBIN=$ac_cv_prog_ac_ct_DUMPBIN
+if test -n "$ac_ct_DUMPBIN"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_DUMPBIN" >&5
+$as_echo "$ac_ct_DUMPBIN" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ test -n "$ac_ct_DUMPBIN" && break
+done
+
+ if test "x$ac_ct_DUMPBIN" = x; then
+ DUMPBIN=":"
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ DUMPBIN=$ac_ct_DUMPBIN
+ fi
+fi
+
+
+ if test "$DUMPBIN" != ":"; then
+ NM="$DUMPBIN"
+ fi
+fi
+test -z "$NM" && NM=nm
+
+
+
+
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking the name lister ($NM) interface" >&5
+$as_echo_n "checking the name lister ($NM) interface... " >&6; }
+if test "${lt_cv_nm_interface+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ lt_cv_nm_interface="BSD nm"
+ echo "int some_variable = 0;" > conftest.$ac_ext
+ (eval echo "\"\$as_me:5657: $ac_compile\"" >&5)
+ (eval "$ac_compile" 2>conftest.err)
+ cat conftest.err >&5
+ (eval echo "\"\$as_me:5660: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+ (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
+ cat conftest.err >&5
+ (eval echo "\"\$as_me:5663: output\"" >&5)
+ cat conftest.out >&5
+ if $GREP 'External.*some_variable' conftest.out > /dev/null; then
+ lt_cv_nm_interface="MS dumpbin"
+ fi
+ rm -f conftest*
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_nm_interface" >&5
+$as_echo "$lt_cv_nm_interface" >&6; }
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ln -s works" >&5
+$as_echo_n "checking whether ln -s works... " >&6; }
+LN_S=$as_ln_s
+if test "$LN_S" = "ln -s"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no, using $LN_S" >&5
+$as_echo "no, using $LN_S" >&6; }
+fi
+
+# find the maximum length of command line arguments
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking the maximum length of command line arguments" >&5
+$as_echo_n "checking the maximum length of command line arguments... " >&6; }
+if test "${lt_cv_sys_max_cmd_len+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ i=0
+ teststring="ABCD"
+
+ case $build_os in
+ msdosdjgpp*)
+ # On DJGPP, this test can blow up pretty badly due to problems in libc
+ # (any single argument exceeding 2000 bytes causes a buffer overrun
+ # during glob expansion). Even if it were fixed, the result of this
+ # check would be larger than it should be.
+ lt_cv_sys_max_cmd_len=12288; # 12K is about right
+ ;;
+
+ gnu*)
+ # Under GNU Hurd, this test is not required because there is
+ # no limit to the length of command line arguments.
+ # Libtool will interpret -1 as no limit whatsoever
+ lt_cv_sys_max_cmd_len=-1;
+ ;;
+
+ cygwin* | mingw* | cegcc*)
+ # On Win9x/ME, this test blows up -- it succeeds, but takes
+ # about 5 minutes as the teststring grows exponentially.
+ # Worse, since 9x/ME are not pre-emptively multitasking,
+ # you end up with a "frozen" computer, even though with patience
+ # the test eventually succeeds (with a max line length of 256k).
+ # Instead, let's just punt: use the minimum linelength reported by
+ # all of the supported platforms: 8192 (on NT/2K/XP).
+ lt_cv_sys_max_cmd_len=8192;
+ ;;
+
+ amigaos*)
+ # On AmigaOS with pdksh, this test takes hours, literally.
+ # So we just punt and use a minimum line length of 8192.
+ lt_cv_sys_max_cmd_len=8192;
+ ;;
+
+ netbsd* | freebsd* | openbsd* | darwin* | dragonfly*)
+ # This has been around since 386BSD, at least. Likely further.
+ if test -x /sbin/sysctl; then
+ lt_cv_sys_max_cmd_len=`/sbin/sysctl -n kern.argmax`
+ elif test -x /usr/sbin/sysctl; then
+ lt_cv_sys_max_cmd_len=`/usr/sbin/sysctl -n kern.argmax`
+ else
+ lt_cv_sys_max_cmd_len=65536 # usable default for all BSDs
+ fi
+ # And add a safety zone
+ lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 4`
+ lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \* 3`
+ ;;
+
+ interix*)
+ # We know the value 262144 and hardcode it with a safety zone (like BSD)
+ lt_cv_sys_max_cmd_len=196608
+ ;;
+
+ osf*)
+ # Dr. Hans Ekkehard Plesser reports seeing a kernel panic running configure
+ # due to this test when exec_disable_arg_limit is 1 on Tru64. It is not
+ # nice to cause kernel panics so lets avoid the loop below.
+ # First set a reasonable default.
+ lt_cv_sys_max_cmd_len=16384
+ #
+ if test -x /sbin/sysconfig; then
+ case `/sbin/sysconfig -q proc exec_disable_arg_limit` in
+ *1*) lt_cv_sys_max_cmd_len=-1 ;;
+ esac
+ fi
+ ;;
+ sco3.2v5*)
+ lt_cv_sys_max_cmd_len=102400
+ ;;
+ sysv5* | sco5v6* | sysv4.2uw2*)
+ kargmax=`grep ARG_MAX /etc/conf/cf.d/stune 2>/dev/null`
+ if test -n "$kargmax"; then
+ lt_cv_sys_max_cmd_len=`echo $kargmax | sed 's/.*[ ]//'`
+ else
+ lt_cv_sys_max_cmd_len=32768
+ fi
+ ;;
+ *)
+ lt_cv_sys_max_cmd_len=`(getconf ARG_MAX) 2> /dev/null`
+ if test -n "$lt_cv_sys_max_cmd_len"; then
+ lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 4`
+ lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \* 3`
+ else
+ # Make teststring a little bigger before we do anything with it.
+ # a 1K string should be a reasonable start.
+ for i in 1 2 3 4 5 6 7 8 ; do
+ teststring=$teststring$teststring
+ done
+ SHELL=${SHELL-${CONFIG_SHELL-/bin/sh}}
+ # If test is not a shell built-in, we'll probably end up computing a
+ # maximum length that is only half of the actual maximum length, but
+ # we can't tell.
+ while { test "X"`$SHELL $0 --fallback-echo "X$teststring$teststring" 2>/dev/null` \
+ = "XX$teststring$teststring"; } >/dev/null 2>&1 &&
+ test $i != 17 # 1/2 MB should be enough
+ do
+ i=`expr $i + 1`
+ teststring=$teststring$teststring
+ done
+ # Only check the string length outside the loop.
+ lt_cv_sys_max_cmd_len=`expr "X$teststring" : ".*" 2>&1`
+ teststring=
+ # Add a significant safety factor because C++ compilers can tack on
+ # massive amounts of additional arguments before passing them to the
+ # linker. It appears as though 1/2 is a usable value.
+ lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 2`
+ fi
+ ;;
+ esac
+
+fi
+
+if test -n $lt_cv_sys_max_cmd_len ; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_sys_max_cmd_len" >&5
+$as_echo "$lt_cv_sys_max_cmd_len" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: none" >&5
+$as_echo "none" >&6; }
+fi
+max_cmd_len=$lt_cv_sys_max_cmd_len
+
+
+
+
+
+
+: ${CP="cp -f"}
+: ${MV="mv -f"}
+: ${RM="rm -f"}
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the shell understands some XSI constructs" >&5
+$as_echo_n "checking whether the shell understands some XSI constructs... " >&6; }
+# Try some XSI features
+xsi_shell=no
+( _lt_dummy="a/b/c"
+ test "${_lt_dummy##*/},${_lt_dummy%/*},"${_lt_dummy%"$_lt_dummy"}, \
+ = c,a/b,, \
+ && eval 'test $(( 1 + 1 )) -eq 2 \
+ && test "${#_lt_dummy}" -eq 5' ) >/dev/null 2>&1 \
+ && xsi_shell=yes
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $xsi_shell" >&5
+$as_echo "$xsi_shell" >&6; }
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the shell understands \"+=\"" >&5
+$as_echo_n "checking whether the shell understands \"+=\"... " >&6; }
+lt_shell_append=no
+( foo=bar; set foo baz; eval "$1+=\$2" && test "$foo" = barbaz ) \
+ >/dev/null 2>&1 \
+ && lt_shell_append=yes
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_shell_append" >&5
+$as_echo "$lt_shell_append" >&6; }
+
+
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+ lt_unset=unset
+else
+ lt_unset=false
+fi
+
+
+
+
+
+# test EBCDIC or ASCII
+case `echo X|tr X '\101'` in
+ A) # ASCII based system
+ # \n is not interpreted correctly by Solaris 8 /usr/ucb/tr
+ lt_SP2NL='tr \040 \012'
+ lt_NL2SP='tr \015\012 \040\040'
+ ;;
+ *) # EBCDIC based system
+ lt_SP2NL='tr \100 \n'
+ lt_NL2SP='tr \r\n \100\100'
+ ;;
+esac
+
+
+
+
+
+
+
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $LD option to reload object files" >&5
+$as_echo_n "checking for $LD option to reload object files... " >&6; }
+if test "${lt_cv_ld_reload_flag+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ lt_cv_ld_reload_flag='-r'
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_ld_reload_flag" >&5
+$as_echo "$lt_cv_ld_reload_flag" >&6; }
+reload_flag=$lt_cv_ld_reload_flag
+case $reload_flag in
+"" | " "*) ;;
+*) reload_flag=" $reload_flag" ;;
+esac
+reload_cmds='$LD$reload_flag -o $output$reload_objs'
+case $host_os in
+ darwin*)
+ if test "$GCC" = yes; then
+ reload_cmds='$LTCC $LTCFLAGS -nostdlib ${wl}-r -o $output$reload_objs'
+ else
+ reload_cmds='$LD$reload_flag -o $output$reload_objs'
+ fi
+ ;;
+esac
+
+
+
+
+
+
+
+
+
+if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}objdump", so it can be a program name with args.
+set dummy ${ac_tool_prefix}objdump; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_OBJDUMP+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$OBJDUMP"; then
+ ac_cv_prog_OBJDUMP="$OBJDUMP" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_OBJDUMP="${ac_tool_prefix}objdump"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+OBJDUMP=$ac_cv_prog_OBJDUMP
+if test -n "$OBJDUMP"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $OBJDUMP" >&5
+$as_echo "$OBJDUMP" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_OBJDUMP"; then
+ ac_ct_OBJDUMP=$OBJDUMP
+ # Extract the first word of "objdump", so it can be a program name with args.
+set dummy objdump; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ac_ct_OBJDUMP+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_OBJDUMP"; then
+ ac_cv_prog_ac_ct_OBJDUMP="$ac_ct_OBJDUMP" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_OBJDUMP="objdump"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_OBJDUMP=$ac_cv_prog_ac_ct_OBJDUMP
+if test -n "$ac_ct_OBJDUMP"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_OBJDUMP" >&5
+$as_echo "$ac_ct_OBJDUMP" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+ if test "x$ac_ct_OBJDUMP" = x; then
+ OBJDUMP="false"
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ OBJDUMP=$ac_ct_OBJDUMP
+ fi
+else
+ OBJDUMP="$ac_cv_prog_OBJDUMP"
+fi
+
+test -z "$OBJDUMP" && OBJDUMP=objdump
+
+
+
+
+
+
+
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to recognize dependent libraries" >&5
+$as_echo_n "checking how to recognize dependent libraries... " >&6; }
+if test "${lt_cv_deplibs_check_method+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ lt_cv_file_magic_cmd='$MAGIC_CMD'
+lt_cv_file_magic_test_file=
+lt_cv_deplibs_check_method='unknown'
+# Need to set the preceding variable on all platforms that support
+# interlibrary dependencies.
+# 'none' -- dependencies not supported.
+# `unknown' -- same as none, but documents that we really don't know.
+# 'pass_all' -- all dependencies passed with no checks.
+# 'test_compile' -- check by making test program.
+# 'file_magic [[regex]]' -- check by looking for files in library path
+# which responds to the $file_magic_cmd with a given extended regex.
+# If you have `file' or equivalent on your system and you're not sure
+# whether `pass_all' will *always* work, you probably want this one.
+
+case $host_os in
+aix[4-9]*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+beos*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+bsdi[45]*)
+ lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [ML]SB (shared object|dynamic lib)'
+ lt_cv_file_magic_cmd='/usr/bin/file -L'
+ lt_cv_file_magic_test_file=/shlib/libc.so
+ ;;
+
+cygwin*)
+ # func_win32_libid is a shell function defined in ltmain.sh
+ lt_cv_deplibs_check_method='file_magic ^x86 archive import|^x86 DLL'
+ lt_cv_file_magic_cmd='func_win32_libid'
+ ;;
+
+mingw* | pw32*)
+ # Base MSYS/MinGW do not provide the 'file' command needed by
+ # func_win32_libid shell function, so use a weaker test based on 'objdump',
+ # unless we find 'file', for example because we are cross-compiling.
+ if ( file / ) >/dev/null 2>&1; then
+ lt_cv_deplibs_check_method='file_magic ^x86 archive import|^x86 DLL'
+ lt_cv_file_magic_cmd='func_win32_libid'
+ else
+ lt_cv_deplibs_check_method='file_magic file format pei*-i386(.*architecture: i386)?'
+ lt_cv_file_magic_cmd='$OBJDUMP -f'
+ fi
+ ;;
+
+cegcc)
+ # use the weaker test based on 'objdump'. See mingw*.
+ lt_cv_deplibs_check_method='file_magic file format pe-arm-.*little(.*architecture: arm)?'
+ lt_cv_file_magic_cmd='$OBJDUMP -f'
+ ;;
+
+darwin* | rhapsody*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+freebsd* | dragonfly*)
+ if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then
+ case $host_cpu in
+ i*86 )
+ # Not sure whether the presence of OpenBSD here was a mistake.
+ # Let's accept both of them until this is cleared up.
+ lt_cv_deplibs_check_method='file_magic (FreeBSD|OpenBSD|DragonFly)/i[3-9]86 (compact )?demand paged shared library'
+ lt_cv_file_magic_cmd=/usr/bin/file
+ lt_cv_file_magic_test_file=`echo /usr/lib/libc.so.*`
+ ;;
+ esac
+ else
+ lt_cv_deplibs_check_method=pass_all
+ fi
+ ;;
+
+gnu*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+hpux10.20* | hpux11*)
+ lt_cv_file_magic_cmd=/usr/bin/file
+ case $host_cpu in
+ ia64*)
+ lt_cv_deplibs_check_method='file_magic (s[0-9][0-9][0-9]|ELF-[0-9][0-9]) shared object file - IA64'
+ lt_cv_file_magic_test_file=/usr/lib/hpux32/libc.so
+ ;;
+ hppa*64*)
+ lt_cv_deplibs_check_method='file_magic (s[0-9][0-9][0-9]|ELF-[0-9][0-9]) shared object file - PA-RISC [0-9].[0-9]'
+ lt_cv_file_magic_test_file=/usr/lib/pa20_64/libc.sl
+ ;;
+ *)
+ lt_cv_deplibs_check_method='file_magic (s[0-9][0-9][0-9]|PA-RISC[0-9].[0-9]) shared library'
+ lt_cv_file_magic_test_file=/usr/lib/libc.sl
+ ;;
+ esac
+ ;;
+
+interix[3-9]*)
+ # PIC code is broken on Interix 3.x, that's why |\.a not |_pic\.a here
+ lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so|\.a)$'
+ ;;
+
+irix5* | irix6* | nonstopux*)
+ case $LD in
+ *-32|*"-32 ") libmagic=32-bit;;
+ *-n32|*"-n32 ") libmagic=N32;;
+ *-64|*"-64 ") libmagic=64-bit;;
+ *) libmagic=never-match;;
+ esac
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+# This must be Linux ELF.
+linux* | k*bsd*-gnu)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+netbsd*)
+ if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then
+ lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so\.[0-9]+\.[0-9]+|_pic\.a)$'
+ else
+ lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so|_pic\.a)$'
+ fi
+ ;;
+
+newos6*)
+ lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [ML]SB (executable|dynamic lib)'
+ lt_cv_file_magic_cmd=/usr/bin/file
+ lt_cv_file_magic_test_file=/usr/lib/libnls.so
+ ;;
+
+*nto* | *qnx*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+openbsd*)
+ if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then
+ lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so\.[0-9]+\.[0-9]+|\.so|_pic\.a)$'
+ else
+ lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so\.[0-9]+\.[0-9]+|_pic\.a)$'
+ fi
+ ;;
+
+osf3* | osf4* | osf5*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+rdos*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+solaris*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX* | sysv4*uw2*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+
+sysv4 | sysv4.3*)
+ case $host_vendor in
+ motorola)
+ lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [ML]SB (shared object|dynamic lib) M[0-9][0-9]* Version [0-9]'
+ lt_cv_file_magic_test_file=`echo /usr/lib/libc.so*`
+ ;;
+ ncr)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+ sequent)
+ lt_cv_file_magic_cmd='/bin/file'
+ lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [LM]SB (shared object|dynamic lib )'
+ ;;
+ sni)
+ lt_cv_file_magic_cmd='/bin/file'
+ lt_cv_deplibs_check_method="file_magic ELF [0-9][0-9]*-bit [LM]SB dynamic lib"
+ lt_cv_file_magic_test_file=/lib/libc.so
+ ;;
+ siemens)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+ pc)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+ esac
+ ;;
+
+tpf*)
+ lt_cv_deplibs_check_method=pass_all
+ ;;
+esac
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_deplibs_check_method" >&5
+$as_echo "$lt_cv_deplibs_check_method" >&6; }
+file_magic_cmd=$lt_cv_file_magic_cmd
+deplibs_check_method=$lt_cv_deplibs_check_method
+test -z "$deplibs_check_method" && deplibs_check_method=unknown
+
+
+
+
+
+
+
+
+
+
+
+
+if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}ar", so it can be a program name with args.
+set dummy ${ac_tool_prefix}ar; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_AR+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$AR"; then
+ ac_cv_prog_AR="$AR" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_AR="${ac_tool_prefix}ar"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+AR=$ac_cv_prog_AR
+if test -n "$AR"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AR" >&5
+$as_echo "$AR" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_AR"; then
+ ac_ct_AR=$AR
+ # Extract the first word of "ar", so it can be a program name with args.
+set dummy ar; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ac_ct_AR+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_AR"; then
+ ac_cv_prog_ac_ct_AR="$ac_ct_AR" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_AR="ar"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_AR=$ac_cv_prog_ac_ct_AR
+if test -n "$ac_ct_AR"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_AR" >&5
+$as_echo "$ac_ct_AR" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+ if test "x$ac_ct_AR" = x; then
+ AR="false"
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ AR=$ac_ct_AR
+ fi
+else
+ AR="$ac_cv_prog_AR"
+fi
+
+test -z "$AR" && AR=ar
+test -z "$AR_FLAGS" && AR_FLAGS=cru
+
+
+
+
+
+
+
+
+
+
+
+if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}strip", so it can be a program name with args.
+set dummy ${ac_tool_prefix}strip; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_STRIP+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$STRIP"; then
+ ac_cv_prog_STRIP="$STRIP" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_STRIP="${ac_tool_prefix}strip"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+STRIP=$ac_cv_prog_STRIP
+if test -n "$STRIP"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $STRIP" >&5
+$as_echo "$STRIP" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_STRIP"; then
+ ac_ct_STRIP=$STRIP
+ # Extract the first word of "strip", so it can be a program name with args.
+set dummy strip; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ac_ct_STRIP+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_STRIP"; then
+ ac_cv_prog_ac_ct_STRIP="$ac_ct_STRIP" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_STRIP="strip"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_STRIP=$ac_cv_prog_ac_ct_STRIP
+if test -n "$ac_ct_STRIP"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_STRIP" >&5
+$as_echo "$ac_ct_STRIP" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+ if test "x$ac_ct_STRIP" = x; then
+ STRIP=":"
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ STRIP=$ac_ct_STRIP
+ fi
+else
+ STRIP="$ac_cv_prog_STRIP"
+fi
+
+test -z "$STRIP" && STRIP=:
+
+
+
+
+
+
+if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}ranlib", so it can be a program name with args.
+set dummy ${ac_tool_prefix}ranlib; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_RANLIB+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$RANLIB"; then
+ ac_cv_prog_RANLIB="$RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_RANLIB="${ac_tool_prefix}ranlib"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+RANLIB=$ac_cv_prog_RANLIB
+if test -n "$RANLIB"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $RANLIB" >&5
+$as_echo "$RANLIB" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_RANLIB"; then
+ ac_ct_RANLIB=$RANLIB
+ # Extract the first word of "ranlib", so it can be a program name with args.
+set dummy ranlib; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ac_ct_RANLIB+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_RANLIB"; then
+ ac_cv_prog_ac_ct_RANLIB="$ac_ct_RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_RANLIB="ranlib"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_RANLIB=$ac_cv_prog_ac_ct_RANLIB
+if test -n "$ac_ct_RANLIB"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_RANLIB" >&5
+$as_echo "$ac_ct_RANLIB" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+ if test "x$ac_ct_RANLIB" = x; then
+ RANLIB=":"
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ RANLIB=$ac_ct_RANLIB
+ fi
+else
+ RANLIB="$ac_cv_prog_RANLIB"
+fi
+
+test -z "$RANLIB" && RANLIB=:
+
+
+
+
+
+
+# Determine commands to create old-style static archives.
+old_archive_cmds='$AR $AR_FLAGS $oldlib$oldobjs'
+old_postinstall_cmds='chmod 644 $oldlib'
+old_postuninstall_cmds=
+
+if test -n "$RANLIB"; then
+ case $host_os in
+ openbsd*)
+ old_postinstall_cmds="$old_postinstall_cmds~\$RANLIB -t \$oldlib"
+ ;;
+ *)
+ old_postinstall_cmds="$old_postinstall_cmds~\$RANLIB \$oldlib"
+ ;;
+ esac
+ old_archive_cmds="$old_archive_cmds~\$RANLIB \$oldlib"
+fi
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# If no C compiler was specified, use CC.
+LTCC=${LTCC-"$CC"}
+
+# If no C compiler flags were specified, use CFLAGS.
+LTCFLAGS=${LTCFLAGS-"$CFLAGS"}
+
+# Allow CC to be a program name with arguments.
+compiler=$CC
+
+
+# Check for command to grab the raw symbol name followed by C symbol from nm.
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking command to parse $NM output from $compiler object" >&5
+$as_echo_n "checking command to parse $NM output from $compiler object... " >&6; }
+if test "${lt_cv_sys_global_symbol_pipe+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+
+# These are sane defaults that work on at least a few old systems.
+# [They come from Ultrix. What could be older than Ultrix?!! ;)]
+
+# Character class describing NM global symbol codes.
+symcode='[BCDEGRST]'
+
+# Regexp to match symbols that can be accessed directly from C.
+sympat='\([_A-Za-z][_A-Za-z0-9]*\)'
+
+# Define system-specific variables.
+case $host_os in
+aix*)
+ symcode='[BCDT]'
+ ;;
+cygwin* | mingw* | pw32* | cegcc*)
+ symcode='[ABCDGISTW]'
+ ;;
+hpux*)
+ if test "$host_cpu" = ia64; then
+ symcode='[ABCDEGRST]'
+ fi
+ ;;
+irix* | nonstopux*)
+ symcode='[BCDEGRST]'
+ ;;
+osf*)
+ symcode='[BCDEGQRST]'
+ ;;
+solaris*)
+ symcode='[BDRT]'
+ ;;
+sco3.2v5*)
+ symcode='[DT]'
+ ;;
+sysv4.2uw2*)
+ symcode='[DT]'
+ ;;
+sysv5* | sco5v6* | unixware* | OpenUNIX*)
+ symcode='[ABDT]'
+ ;;
+sysv4)
+ symcode='[DFNSTU]'
+ ;;
+esac
+
+# If we're using GNU nm, then use its standard symbol codes.
+case `$NM -V 2>&1` in
+*GNU* | *'with BFD'*)
+ symcode='[ABCDGIRSTW]' ;;
+esac
+
+# Transform an extracted symbol line into a proper C declaration.
+# Some systems (esp. on ia64) link data and code symbols differently,
+# so use this general approach.
+lt_cv_sys_global_symbol_to_cdecl="sed -n -e 's/^T .* \(.*\)$/extern int \1();/p' -e 's/^$symcode* .* \(.*\)$/extern char \1;/p'"
+
+# Transform an extracted symbol line into symbol name and symbol address
+lt_cv_sys_global_symbol_to_c_name_address="sed -n -e 's/^: \([^ ]*\) $/ {\\\"\1\\\", (void *) 0},/p' -e 's/^$symcode* \([^ ]*\) \([^ ]*\)$/ {\"\2\", (void *) \&\2},/p'"
+lt_cv_sys_global_symbol_to_c_name_address_lib_prefix="sed -n -e 's/^: \([^ ]*\) $/ {\\\"\1\\\", (void *) 0},/p' -e 's/^$symcode* \([^ ]*\) \(lib[^ ]*\)$/ {\"\2\", (void *) \&\2},/p' -e 's/^$symcode* \([^ ]*\) \([^ ]*\)$/ {\"lib\2\", (void *) \&\2},/p'"
+
+# Handle CRLF in mingw tool chain
+opt_cr=
+case $build_os in
+mingw*)
+ opt_cr=`$ECHO 'x\{0,1\}' | tr x '\015'` # option cr in regexp
+ ;;
+esac
+
+# Try without a prefix underscore, then with it.
+for ac_symprfx in "" "_"; do
+
+ # Transform symcode, sympat, and symprfx into a raw symbol and a C symbol.
+ symxfrm="\\1 $ac_symprfx\\2 \\2"
+
+ # Write the raw and C identifiers.
+ if test "$lt_cv_nm_interface" = "MS dumpbin"; then
+ # Fake it for dumpbin and say T for any non-static function
+ # and D for any global variable.
+ # Also find C++ and __fastcall symbols from MSVC++,
+ # which start with @ or ?.
+ lt_cv_sys_global_symbol_pipe="$AWK '"\
+" {last_section=section; section=\$ 3};"\
+" /Section length .*#relocs.*(pick any)/{hide[last_section]=1};"\
+" \$ 0!~/External *\|/{next};"\
+" / 0+ UNDEF /{next}; / UNDEF \([^|]\)*()/{next};"\
+" {if(hide[section]) next};"\
+" {f=0}; \$ 0~/\(\).*\|/{f=1}; {printf f ? \"T \" : \"D \"};"\
+" {split(\$ 0, a, /\||\r/); split(a[2], s)};"\
+" s[1]~/^[@?]/{print s[1], s[1]; next};"\
+" s[1]~prfx {split(s[1],t,\"@\"); print t[1], substr(t[1],length(prfx))}"\
+" ' prfx=^$ac_symprfx"
+ else
+ lt_cv_sys_global_symbol_pipe="sed -n -e 's/^.*[ ]\($symcode$symcode*\)[ ][ ]*$ac_symprfx$sympat$opt_cr$/$symxfrm/p'"
+ fi
+
+ # Check to see that the pipe works correctly.
+ pipe_works=no
+
+ rm -f conftest*
+ cat > conftest.$ac_ext <<_LT_EOF
+#ifdef __cplusplus
+extern "C" {
+#endif
+char nm_test_var;
+void nm_test_func(void);
+void nm_test_func(void){}
+#ifdef __cplusplus
+}
+#endif
+int main(){nm_test_var='a';nm_test_func();return(0);}
+_LT_EOF
+
+ if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
+ (eval $ac_compile) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ # Now try to grab the symbols.
+ nlist=conftest.nm
+ if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$NM conftest.$ac_objext \| $lt_cv_sys_global_symbol_pipe \> $nlist\""; } >&5
+ (eval $NM conftest.$ac_objext \| $lt_cv_sys_global_symbol_pipe \> $nlist) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; } && test -s "$nlist"; then
+ # Try sorting and uniquifying the output.
+ if sort "$nlist" | uniq > "$nlist"T; then
+ mv -f "$nlist"T "$nlist"
+ else
+ rm -f "$nlist"T
+ fi
+
+ # Make sure that we snagged all the symbols we need.
+ if $GREP ' nm_test_var$' "$nlist" >/dev/null; then
+ if $GREP ' nm_test_func$' "$nlist" >/dev/null; then
+ cat <<_LT_EOF > conftest.$ac_ext
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+_LT_EOF
+ # Now generate the symbol file.
+ eval "$lt_cv_sys_global_symbol_to_cdecl"' < "$nlist" | $GREP -v main >> conftest.$ac_ext'
+
+ cat <<_LT_EOF >> conftest.$ac_ext
+
+/* The mapping between symbol names and symbols. */
+const struct {
+ const char *name;
+ void *address;
+}
+lt__PROGRAM__LTX_preloaded_symbols[] =
+{
+ { "@PROGRAM@", (void *) 0 },
+_LT_EOF
+ $SED "s/^$symcode$symcode* \(.*\) \(.*\)$/ {\"\2\", (void *) \&\2},/" < "$nlist" | $GREP -v main >> conftest.$ac_ext
+ cat <<\_LT_EOF >> conftest.$ac_ext
+ {0, (void *) 0}
+};
+
+/* This works around a problem in FreeBSD linker */
+#ifdef FREEBSD_WORKAROUND
+static const void *lt_preloaded_setup() {
+ return lt__PROGRAM__LTX_preloaded_symbols;
+}
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+_LT_EOF
+ # Now try linking the two files.
+ mv conftest.$ac_objext conftstm.$ac_objext
+ lt_save_LIBS="$LIBS"
+ lt_save_CFLAGS="$CFLAGS"
+ LIBS="conftstm.$ac_objext"
+ CFLAGS="$CFLAGS$lt_prog_compiler_no_builtin_flag"
+ if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_link\""; } >&5
+ (eval $ac_link) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; } && test -s conftest${ac_exeext}; then
+ pipe_works=yes
+ fi
+ LIBS="$lt_save_LIBS"
+ CFLAGS="$lt_save_CFLAGS"
+ else
+ echo "cannot find nm_test_func in $nlist" >&5
+ fi
+ else
+ echo "cannot find nm_test_var in $nlist" >&5
+ fi
+ else
+ echo "cannot run $lt_cv_sys_global_symbol_pipe" >&5
+ fi
+ else
+ echo "$progname: failed program was:" >&5
+ cat conftest.$ac_ext >&5
+ fi
+ rm -rf conftest* conftst*
+
+ # Do not use the global_symbol_pipe unless it works.
+ if test "$pipe_works" = yes; then
+ break
+ else
+ lt_cv_sys_global_symbol_pipe=
+ fi
+done
+
+fi
+
+if test -z "$lt_cv_sys_global_symbol_pipe"; then
+ lt_cv_sys_global_symbol_to_cdecl=
+fi
+if test -z "$lt_cv_sys_global_symbol_pipe$lt_cv_sys_global_symbol_to_cdecl"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: failed" >&5
+$as_echo "failed" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: ok" >&5
+$as_echo "ok" >&6; }
+fi
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# Check whether --enable-libtool-lock was given.
+if test "${enable_libtool_lock+set}" = set; then :
+ enableval=$enable_libtool_lock;
+fi
+
+test "x$enable_libtool_lock" != xno && enable_libtool_lock=yes
+
+# Some flags need to be propagated to the compiler or linker for good
+# libtool support.
+case $host in
+ia64-*-hpux*)
+ # Find out which ABI we are using.
+ echo 'int i;' > conftest.$ac_ext
+ if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
+ (eval $ac_compile) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ case `/usr/bin/file conftest.$ac_objext` in
+ *ELF-32*)
+ HPUX_IA64_MODE="32"
+ ;;
+ *ELF-64*)
+ HPUX_IA64_MODE="64"
+ ;;
+ esac
+ fi
+ rm -rf conftest*
+ ;;
+*-*-irix6*)
+ # Find out which ABI we are using.
+ echo '#line 6868 "configure"' > conftest.$ac_ext
+ if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
+ (eval $ac_compile) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ if test "$lt_cv_prog_gnu_ld" = yes; then
+ case `/usr/bin/file conftest.$ac_objext` in
+ *32-bit*)
+ LD="${LD-ld} -melf32bsmip"
+ ;;
+ *N32*)
+ LD="${LD-ld} -melf32bmipn32"
+ ;;
+ *64-bit*)
+ LD="${LD-ld} -melf64bmip"
+ ;;
+ esac
+ else
+ case `/usr/bin/file conftest.$ac_objext` in
+ *32-bit*)
+ LD="${LD-ld} -32"
+ ;;
+ *N32*)
+ LD="${LD-ld} -n32"
+ ;;
+ *64-bit*)
+ LD="${LD-ld} -64"
+ ;;
+ esac
+ fi
+ fi
+ rm -rf conftest*
+ ;;
+
+x86_64-*kfreebsd*-gnu|x86_64-*linux*|ppc*-*linux*|powerpc*-*linux*| \
+s390*-*linux*|s390*-*tpf*|sparc*-*linux*)
+ # Find out which ABI we are using.
+ echo 'int i;' > conftest.$ac_ext
+ if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
+ (eval $ac_compile) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ case `/usr/bin/file conftest.o` in
+ *32-bit*)
+ case $host in
+ x86_64-*kfreebsd*-gnu)
+ LD="${LD-ld} -m elf_i386_fbsd"
+ ;;
+ x86_64-*linux*)
+ LD="${LD-ld} -m elf_i386"
+ ;;
+ ppc64-*linux*|powerpc64-*linux*)
+ LD="${LD-ld} -m elf32ppclinux"
+ ;;
+ s390x-*linux*)
+ LD="${LD-ld} -m elf_s390"
+ ;;
+ sparc64-*linux*)
+ LD="${LD-ld} -m elf32_sparc"
+ ;;
+ esac
+ ;;
+ *64-bit*)
+ case $host in
+ x86_64-*kfreebsd*-gnu)
+ LD="${LD-ld} -m elf_x86_64_fbsd"
+ ;;
+ x86_64-*linux*)
+ LD="${LD-ld} -m elf_x86_64"
+ ;;
+ ppc*-*linux*|powerpc*-*linux*)
+ LD="${LD-ld} -m elf64ppc"
+ ;;
+ s390*-*linux*|s390*-*tpf*)
+ LD="${LD-ld} -m elf64_s390"
+ ;;
+ sparc*-*linux*)
+ LD="${LD-ld} -m elf64_sparc"
+ ;;
+ esac
+ ;;
+ esac
+ fi
+ rm -rf conftest*
+ ;;
+
+*-*-sco3.2v5*)
+ # On SCO OpenServer 5, we need -belf to get full-featured binaries.
+ SAVE_CFLAGS="$CFLAGS"
+ CFLAGS="$CFLAGS -belf"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the C compiler needs -belf" >&5
+$as_echo_n "checking whether the C compiler needs -belf... " >&6; }
+if test "${lt_cv_cc_needs_belf+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ lt_cv_cc_needs_belf=yes
+else
+ lt_cv_cc_needs_belf=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+ ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_cc_needs_belf" >&5
+$as_echo "$lt_cv_cc_needs_belf" >&6; }
+ if test x"$lt_cv_cc_needs_belf" != x"yes"; then
+ # this is probably gcc 2.8.0, egcs 1.0 or newer; no need for -belf
+ CFLAGS="$SAVE_CFLAGS"
+ fi
+ ;;
+sparc*-*solaris*)
+ # Find out which ABI we are using.
+ echo 'int i;' > conftest.$ac_ext
+ if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
+ (eval $ac_compile) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ case `/usr/bin/file conftest.o` in
+ *64-bit*)
+ case $lt_cv_prog_gnu_ld in
+ yes*) LD="${LD-ld} -m elf64_sparc" ;;
+ *)
+ if ${LD-ld} -64 -r -o conftest2.o conftest.o >/dev/null 2>&1; then
+ LD="${LD-ld} -64"
+ fi
+ ;;
+ esac
+ ;;
+ esac
+ fi
+ rm -rf conftest*
+ ;;
+esac
+
+need_locks="$enable_libtool_lock"
+
+
+ case $host_os in
+ rhapsody* | darwin*)
+ if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}dsymutil", so it can be a program name with args.
+set dummy ${ac_tool_prefix}dsymutil; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_DSYMUTIL+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$DSYMUTIL"; then
+ ac_cv_prog_DSYMUTIL="$DSYMUTIL" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_DSYMUTIL="${ac_tool_prefix}dsymutil"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+DSYMUTIL=$ac_cv_prog_DSYMUTIL
+if test -n "$DSYMUTIL"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $DSYMUTIL" >&5
+$as_echo "$DSYMUTIL" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_DSYMUTIL"; then
+ ac_ct_DSYMUTIL=$DSYMUTIL
+ # Extract the first word of "dsymutil", so it can be a program name with args.
+set dummy dsymutil; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ac_ct_DSYMUTIL+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_DSYMUTIL"; then
+ ac_cv_prog_ac_ct_DSYMUTIL="$ac_ct_DSYMUTIL" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_DSYMUTIL="dsymutil"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_DSYMUTIL=$ac_cv_prog_ac_ct_DSYMUTIL
+if test -n "$ac_ct_DSYMUTIL"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_DSYMUTIL" >&5
+$as_echo "$ac_ct_DSYMUTIL" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+ if test "x$ac_ct_DSYMUTIL" = x; then
+ DSYMUTIL=":"
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ DSYMUTIL=$ac_ct_DSYMUTIL
+ fi
+else
+ DSYMUTIL="$ac_cv_prog_DSYMUTIL"
+fi
+
+ if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}nmedit", so it can be a program name with args.
+set dummy ${ac_tool_prefix}nmedit; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_NMEDIT+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$NMEDIT"; then
+ ac_cv_prog_NMEDIT="$NMEDIT" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_NMEDIT="${ac_tool_prefix}nmedit"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+NMEDIT=$ac_cv_prog_NMEDIT
+if test -n "$NMEDIT"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $NMEDIT" >&5
+$as_echo "$NMEDIT" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_NMEDIT"; then
+ ac_ct_NMEDIT=$NMEDIT
+ # Extract the first word of "nmedit", so it can be a program name with args.
+set dummy nmedit; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ac_ct_NMEDIT+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_NMEDIT"; then
+ ac_cv_prog_ac_ct_NMEDIT="$ac_ct_NMEDIT" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_NMEDIT="nmedit"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_NMEDIT=$ac_cv_prog_ac_ct_NMEDIT
+if test -n "$ac_ct_NMEDIT"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_NMEDIT" >&5
+$as_echo "$ac_ct_NMEDIT" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+ if test "x$ac_ct_NMEDIT" = x; then
+ NMEDIT=":"
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ NMEDIT=$ac_ct_NMEDIT
+ fi
+else
+ NMEDIT="$ac_cv_prog_NMEDIT"
+fi
+
+ if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}lipo", so it can be a program name with args.
+set dummy ${ac_tool_prefix}lipo; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_LIPO+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$LIPO"; then
+ ac_cv_prog_LIPO="$LIPO" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_LIPO="${ac_tool_prefix}lipo"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+LIPO=$ac_cv_prog_LIPO
+if test -n "$LIPO"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LIPO" >&5
+$as_echo "$LIPO" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_LIPO"; then
+ ac_ct_LIPO=$LIPO
+ # Extract the first word of "lipo", so it can be a program name with args.
+set dummy lipo; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ac_ct_LIPO+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_LIPO"; then
+ ac_cv_prog_ac_ct_LIPO="$ac_ct_LIPO" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_LIPO="lipo"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_LIPO=$ac_cv_prog_ac_ct_LIPO
+if test -n "$ac_ct_LIPO"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_LIPO" >&5
+$as_echo "$ac_ct_LIPO" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+ if test "x$ac_ct_LIPO" = x; then
+ LIPO=":"
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ LIPO=$ac_ct_LIPO
+ fi
+else
+ LIPO="$ac_cv_prog_LIPO"
+fi
+
+ if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}otool", so it can be a program name with args.
+set dummy ${ac_tool_prefix}otool; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_OTOOL+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$OTOOL"; then
+ ac_cv_prog_OTOOL="$OTOOL" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_OTOOL="${ac_tool_prefix}otool"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+OTOOL=$ac_cv_prog_OTOOL
+if test -n "$OTOOL"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $OTOOL" >&5
+$as_echo "$OTOOL" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_OTOOL"; then
+ ac_ct_OTOOL=$OTOOL
+ # Extract the first word of "otool", so it can be a program name with args.
+set dummy otool; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ac_ct_OTOOL+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_OTOOL"; then
+ ac_cv_prog_ac_ct_OTOOL="$ac_ct_OTOOL" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_OTOOL="otool"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_OTOOL=$ac_cv_prog_ac_ct_OTOOL
+if test -n "$ac_ct_OTOOL"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_OTOOL" >&5
+$as_echo "$ac_ct_OTOOL" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+ if test "x$ac_ct_OTOOL" = x; then
+ OTOOL=":"
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ OTOOL=$ac_ct_OTOOL
+ fi
+else
+ OTOOL="$ac_cv_prog_OTOOL"
+fi
+
+ if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}otool64", so it can be a program name with args.
+set dummy ${ac_tool_prefix}otool64; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_OTOOL64+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$OTOOL64"; then
+ ac_cv_prog_OTOOL64="$OTOOL64" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_OTOOL64="${ac_tool_prefix}otool64"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+OTOOL64=$ac_cv_prog_OTOOL64
+if test -n "$OTOOL64"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $OTOOL64" >&5
+$as_echo "$OTOOL64" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_OTOOL64"; then
+ ac_ct_OTOOL64=$OTOOL64
+ # Extract the first word of "otool64", so it can be a program name with args.
+set dummy otool64; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ac_ct_OTOOL64+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ac_ct_OTOOL64"; then
+ ac_cv_prog_ac_ct_OTOOL64="$ac_ct_OTOOL64" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ac_ct_OTOOL64="otool64"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_OTOOL64=$ac_cv_prog_ac_ct_OTOOL64
+if test -n "$ac_ct_OTOOL64"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_OTOOL64" >&5
+$as_echo "$ac_ct_OTOOL64" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+ if test "x$ac_ct_OTOOL64" = x; then
+ OTOOL64=":"
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ OTOOL64=$ac_ct_OTOOL64
+ fi
+else
+ OTOOL64="$ac_cv_prog_OTOOL64"
+fi
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -single_module linker flag" >&5
+$as_echo_n "checking for -single_module linker flag... " >&6; }
+if test "${lt_cv_apple_cc_single_mod+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ lt_cv_apple_cc_single_mod=no
+ if test -z "${LT_MULTI_MODULE}"; then
+ # By default we will add the -single_module flag. You can override
+ # by either setting the environment variable LT_MULTI_MODULE
+ # non-empty at configure time, or by adding -multi_module to the
+ # link flags.
+ rm -rf libconftest.dylib*
+ echo "int foo(void){return 1;}" > conftest.c
+ echo "$LTCC $LTCFLAGS $LDFLAGS -o libconftest.dylib \
+-dynamiclib -Wl,-single_module conftest.c" >&5
+ $LTCC $LTCFLAGS $LDFLAGS -o libconftest.dylib \
+ -dynamiclib -Wl,-single_module conftest.c 2>conftest.err
+ _lt_result=$?
+ if test -f libconftest.dylib && test ! -s conftest.err && test $_lt_result = 0; then
+ lt_cv_apple_cc_single_mod=yes
+ else
+ cat conftest.err >&5
+ fi
+ rm -rf libconftest.dylib*
+ rm -f conftest.*
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_apple_cc_single_mod" >&5
+$as_echo "$lt_cv_apple_cc_single_mod" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -exported_symbols_list linker flag" >&5
+$as_echo_n "checking for -exported_symbols_list linker flag... " >&6; }
+if test "${lt_cv_ld_exported_symbols_list+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ lt_cv_ld_exported_symbols_list=no
+ save_LDFLAGS=$LDFLAGS
+ echo "_main" > conftest.sym
+ LDFLAGS="$LDFLAGS -Wl,-exported_symbols_list,conftest.sym"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ lt_cv_ld_exported_symbols_list=yes
+else
+ lt_cv_ld_exported_symbols_list=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+ LDFLAGS="$save_LDFLAGS"
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_ld_exported_symbols_list" >&5
+$as_echo "$lt_cv_ld_exported_symbols_list" >&6; }
+ case $host_os in
+ rhapsody* | darwin1.[012])
+ _lt_dar_allow_undefined='${wl}-undefined ${wl}suppress' ;;
+ darwin1.*)
+ _lt_dar_allow_undefined='${wl}-flat_namespace ${wl}-undefined ${wl}suppress' ;;
+ darwin*) # darwin 5.x on
+ # if running on 10.5 or later, the deployment target defaults
+ # to the OS version, if on x86, and 10.4, the deployment
+ # target defaults to 10.4. Don't you love it?
+ case ${MACOSX_DEPLOYMENT_TARGET-10.0},$host in
+ 10.0,*86*-darwin8*|10.0,*-darwin[91]*)
+ _lt_dar_allow_undefined='${wl}-undefined ${wl}dynamic_lookup' ;;
+ 10.[012]*)
+ _lt_dar_allow_undefined='${wl}-flat_namespace ${wl}-undefined ${wl}suppress' ;;
+ 10.*)
+ _lt_dar_allow_undefined='${wl}-undefined ${wl}dynamic_lookup' ;;
+ esac
+ ;;
+ esac
+ if test "$lt_cv_apple_cc_single_mod" = "yes"; then
+ _lt_dar_single_mod='$single_module'
+ fi
+ if test "$lt_cv_ld_exported_symbols_list" = "yes"; then
+ _lt_dar_export_syms=' ${wl}-exported_symbols_list,$output_objdir/${libname}-symbols.expsym'
+ else
+ _lt_dar_export_syms='~$NMEDIT -s $output_objdir/${libname}-symbols.expsym ${lib}'
+ fi
+ if test "$DSYMUTIL" != ":"; then
+ _lt_dsymutil='~$DSYMUTIL $lib || :'
+ else
+ _lt_dsymutil=
+ fi
+ ;;
+ esac
+
+# On IRIX 5.3, sys/types and inttypes.h are conflicting.
+for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \
+ inttypes.h stdint.h unistd.h
+do :
+ as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
+ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default
+"
+if eval test \"x\$"$as_ac_Header"\" = x"yes"; then :
+ cat >>confdefs.h <<_ACEOF
+#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+
+for ac_header in dlfcn.h
+do :
+ ac_fn_c_check_header_compile "$LINENO" "dlfcn.h" "ac_cv_header_dlfcn_h" "$ac_includes_default
+"
+if test "x$ac_cv_header_dlfcn_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_DLFCN_H 1
+_ACEOF
+
+fi
+
+done
+
+
+
+# Set options
+
+
+
+ enable_dlopen=no
+
+
+ enable_win32_dll=no
+
+
+ # Check whether --enable-shared was given.
+if test "${enable_shared+set}" = set; then :
+ enableval=$enable_shared; p=${PACKAGE-default}
+ case $enableval in
+ yes) enable_shared=yes ;;
+ no) enable_shared=no ;;
+ *)
+ enable_shared=no
+ # Look at the argument we got. We use all the common list separators.
+ lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR,"
+ for pkg in $enableval; do
+ IFS="$lt_save_ifs"
+ if test "X$pkg" = "X$p"; then
+ enable_shared=yes
+ fi
+ done
+ IFS="$lt_save_ifs"
+ ;;
+ esac
+else
+ enable_shared=yes
+fi
+
+
+
+
+
+
+
+
+
+ # Check whether --enable-static was given.
+if test "${enable_static+set}" = set; then :
+ enableval=$enable_static; p=${PACKAGE-default}
+ case $enableval in
+ yes) enable_static=yes ;;
+ no) enable_static=no ;;
+ *)
+ enable_static=no
+ # Look at the argument we got. We use all the common list separators.
+ lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR,"
+ for pkg in $enableval; do
+ IFS="$lt_save_ifs"
+ if test "X$pkg" = "X$p"; then
+ enable_static=yes
+ fi
+ done
+ IFS="$lt_save_ifs"
+ ;;
+ esac
+else
+ enable_static=yes
+fi
+
+
+
+
+
+
+
+
+
+
+# Check whether --with-pic was given.
+if test "${with_pic+set}" = set; then :
+ withval=$with_pic; pic_mode="$withval"
+else
+ pic_mode=default
+fi
+
+
+test -z "$pic_mode" && pic_mode=default
+
+
+
+
+
+
+
+ # Check whether --enable-fast-install was given.
+if test "${enable_fast_install+set}" = set; then :
+ enableval=$enable_fast_install; p=${PACKAGE-default}
+ case $enableval in
+ yes) enable_fast_install=yes ;;
+ no) enable_fast_install=no ;;
+ *)
+ enable_fast_install=no
+ # Look at the argument we got. We use all the common list separators.
+ lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR,"
+ for pkg in $enableval; do
+ IFS="$lt_save_ifs"
+ if test "X$pkg" = "X$p"; then
+ enable_fast_install=yes
+ fi
+ done
+ IFS="$lt_save_ifs"
+ ;;
+ esac
+else
+ enable_fast_install=yes
+fi
+
+
+
+
+
+
+
+
+
+
+
+# This can be used to rebuild libtool when needed
+LIBTOOL_DEPS="$ltmain"
+
+# Always use our own libtool.
+LIBTOOL='$(SHELL) $(top_builddir)/libtool'
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+test -z "$LN_S" && LN_S="ln -s"
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+if test -n "${ZSH_VERSION+set}" ; then
+ setopt NO_GLOB_SUBST
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for objdir" >&5
+$as_echo_n "checking for objdir... " >&6; }
+if test "${lt_cv_objdir+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ rm -f .libs 2>/dev/null
+mkdir .libs 2>/dev/null
+if test -d .libs; then
+ lt_cv_objdir=.libs
+else
+ # MS-DOS does not allow filenames that begin with a dot.
+ lt_cv_objdir=_libs
+fi
+rmdir .libs 2>/dev/null
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_objdir" >&5
+$as_echo "$lt_cv_objdir" >&6; }
+objdir=$lt_cv_objdir
+
+
+
+
+
+cat >>confdefs.h <<_ACEOF
+#define LT_OBJDIR "$lt_cv_objdir/"
+_ACEOF
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+case $host_os in
+aix3*)
+ # AIX sometimes has problems with the GCC collect2 program. For some
+ # reason, if we set the COLLECT_NAMES environment variable, the problems
+ # vanish in a puff of smoke.
+ if test "X${COLLECT_NAMES+set}" != Xset; then
+ COLLECT_NAMES=
+ export COLLECT_NAMES
+ fi
+ ;;
+esac
+
+# Sed substitution that helps us do robust quoting. It backslashifies
+# metacharacters that are still active within double-quoted strings.
+sed_quote_subst='s/\(["`$\\]\)/\\\1/g'
+
+# Same as above, but do not quote variable references.
+double_quote_subst='s/\(["`\\]\)/\\\1/g'
+
+# Sed substitution to delay expansion of an escaped shell variable in a
+# double_quote_subst'ed string.
+delay_variable_subst='s/\\\\\\\\\\\$/\\\\\\$/g'
+
+# Sed substitution to delay expansion of an escaped single quote.
+delay_single_quote_subst='s/'\''/'\'\\\\\\\'\''/g'
+
+# Sed substitution to avoid accidental globbing in evaled expressions
+no_glob_subst='s/\*/\\\*/g'
+
+# Global variables:
+ofile=libtool
+can_build_shared=yes
+
+# All known linkers require a `.a' archive for static linking (except MSVC,
+# which needs '.lib').
+libext=a
+
+with_gnu_ld="$lt_cv_prog_gnu_ld"
+
+old_CC="$CC"
+old_CFLAGS="$CFLAGS"
+
+# Set sane defaults for various variables
+test -z "$CC" && CC=cc
+test -z "$LTCC" && LTCC=$CC
+test -z "$LTCFLAGS" && LTCFLAGS=$CFLAGS
+test -z "$LD" && LD=ld
+test -z "$ac_objext" && ac_objext=o
+
+for cc_temp in $compiler""; do
+ case $cc_temp in
+ compile | *[\\/]compile | ccache | *[\\/]ccache ) ;;
+ distcc | *[\\/]distcc | purify | *[\\/]purify ) ;;
+ \-*) ;;
+ *) break;;
+ esac
+done
+cc_basename=`$ECHO "X$cc_temp" | $Xsed -e 's%.*/%%' -e "s%^$host_alias-%%"`
+
+
+# Only perform the check for file, if the check method requires it
+test -z "$MAGIC_CMD" && MAGIC_CMD=file
+case $deplibs_check_method in
+file_magic*)
+ if test "$file_magic_cmd" = '$MAGIC_CMD'; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ${ac_tool_prefix}file" >&5
+$as_echo_n "checking for ${ac_tool_prefix}file... " >&6; }
+if test "${lt_cv_path_MAGIC_CMD+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ case $MAGIC_CMD in
+[\\/*] | ?:[\\/]*)
+ lt_cv_path_MAGIC_CMD="$MAGIC_CMD" # Let the user override the test with a path.
+ ;;
+*)
+ lt_save_MAGIC_CMD="$MAGIC_CMD"
+ lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR
+ ac_dummy="/usr/bin$PATH_SEPARATOR$PATH"
+ for ac_dir in $ac_dummy; do
+ IFS="$lt_save_ifs"
+ test -z "$ac_dir" && ac_dir=.
+ if test -f $ac_dir/${ac_tool_prefix}file; then
+ lt_cv_path_MAGIC_CMD="$ac_dir/${ac_tool_prefix}file"
+ if test -n "$file_magic_test_file"; then
+ case $deplibs_check_method in
+ "file_magic "*)
+ file_magic_regex=`expr "$deplibs_check_method" : "file_magic \(.*\)"`
+ MAGIC_CMD="$lt_cv_path_MAGIC_CMD"
+ if eval $file_magic_cmd \$file_magic_test_file 2> /dev/null |
+ $EGREP "$file_magic_regex" > /dev/null; then
+ :
+ else
+ cat <<_LT_EOF 1>&2
+
+*** Warning: the command libtool uses to detect shared libraries,
+*** $file_magic_cmd, produces output that libtool cannot recognize.
+*** The result is that libtool may fail to recognize shared libraries
+*** as such. This will affect the creation of libtool libraries that
+*** depend on shared libraries, but programs linked with such libtool
+*** libraries will work regardless of this problem. Nevertheless, you
+*** may want to report the problem to your system manager and/or to
+*** bug-libtool at gnu.org
+
+_LT_EOF
+ fi ;;
+ esac
+ fi
+ break
+ fi
+ done
+ IFS="$lt_save_ifs"
+ MAGIC_CMD="$lt_save_MAGIC_CMD"
+ ;;
+esac
+fi
+
+MAGIC_CMD="$lt_cv_path_MAGIC_CMD"
+if test -n "$MAGIC_CMD"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $MAGIC_CMD" >&5
+$as_echo "$MAGIC_CMD" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+
+
+
+if test -z "$lt_cv_path_MAGIC_CMD"; then
+ if test -n "$ac_tool_prefix"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for file" >&5
+$as_echo_n "checking for file... " >&6; }
+if test "${lt_cv_path_MAGIC_CMD+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ case $MAGIC_CMD in
+[\\/*] | ?:[\\/]*)
+ lt_cv_path_MAGIC_CMD="$MAGIC_CMD" # Let the user override the test with a path.
+ ;;
+*)
+ lt_save_MAGIC_CMD="$MAGIC_CMD"
+ lt_save_ifs="$IFS"; IFS=$PATH_SEPARATOR
+ ac_dummy="/usr/bin$PATH_SEPARATOR$PATH"
+ for ac_dir in $ac_dummy; do
+ IFS="$lt_save_ifs"
+ test -z "$ac_dir" && ac_dir=.
+ if test -f $ac_dir/file; then
+ lt_cv_path_MAGIC_CMD="$ac_dir/file"
+ if test -n "$file_magic_test_file"; then
+ case $deplibs_check_method in
+ "file_magic "*)
+ file_magic_regex=`expr "$deplibs_check_method" : "file_magic \(.*\)"`
+ MAGIC_CMD="$lt_cv_path_MAGIC_CMD"
+ if eval $file_magic_cmd \$file_magic_test_file 2> /dev/null |
+ $EGREP "$file_magic_regex" > /dev/null; then
+ :
+ else
+ cat <<_LT_EOF 1>&2
+
+*** Warning: the command libtool uses to detect shared libraries,
+*** $file_magic_cmd, produces output that libtool cannot recognize.
+*** The result is that libtool may fail to recognize shared libraries
+*** as such. This will affect the creation of libtool libraries that
+*** depend on shared libraries, but programs linked with such libtool
+*** libraries will work regardless of this problem. Nevertheless, you
+*** may want to report the problem to your system manager and/or to
+*** bug-libtool at gnu.org
+
+_LT_EOF
+ fi ;;
+ esac
+ fi
+ break
+ fi
+ done
+ IFS="$lt_save_ifs"
+ MAGIC_CMD="$lt_save_MAGIC_CMD"
+ ;;
+esac
+fi
+
+MAGIC_CMD="$lt_cv_path_MAGIC_CMD"
+if test -n "$MAGIC_CMD"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $MAGIC_CMD" >&5
+$as_echo "$MAGIC_CMD" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ else
+ MAGIC_CMD=:
+ fi
+fi
+
+ fi
+ ;;
+esac
+
+# Use C for the default configuration in the libtool script
+
+lt_save_CC="$CC"
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+# Source file extension for C test sources.
+ac_ext=c
+
+# Object file extension for compiled C test sources.
+objext=o
+objext=$objext
+
+# Code to be used in simple compile tests
+lt_simple_compile_test_code="int some_variable = 0;"
+
+# Code to be used in simple link tests
+lt_simple_link_test_code='int main(){return(0);}'
+
+
+
+
+
+
+
+# If no C compiler was specified, use CC.
+LTCC=${LTCC-"$CC"}
+
+# If no C compiler flags were specified, use CFLAGS.
+LTCFLAGS=${LTCFLAGS-"$CFLAGS"}
+
+# Allow CC to be a program name with arguments.
+compiler=$CC
+
+# Save the default compiler, since it gets overwritten when the other
+# tags are being tested, and _LT_TAGVAR(compiler, []) is a NOP.
+compiler_DEFAULT=$CC
+
+# save warnings/boilerplate of simple test code
+ac_outfile=conftest.$ac_objext
+echo "$lt_simple_compile_test_code" >conftest.$ac_ext
+eval "$ac_compile" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err
+_lt_compiler_boilerplate=`cat conftest.err`
+$RM conftest*
+
+ac_outfile=conftest.$ac_objext
+echo "$lt_simple_link_test_code" >conftest.$ac_ext
+eval "$ac_link" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err
+_lt_linker_boilerplate=`cat conftest.err`
+$RM -r conftest*
+
+
+## CAVEAT EMPTOR:
+## There is no encapsulation within the following macros, do not change
+## the running order or otherwise move them around unless you know exactly
+## what you are doing...
+if test -n "$compiler"; then
+
+lt_prog_compiler_no_builtin_flag=
+
+if test "$GCC" = yes; then
+ lt_prog_compiler_no_builtin_flag=' -fno-builtin'
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler supports -fno-rtti -fno-exceptions" >&5
+$as_echo_n "checking if $compiler supports -fno-rtti -fno-exceptions... " >&6; }
+if test "${lt_cv_prog_compiler_rtti_exceptions+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ lt_cv_prog_compiler_rtti_exceptions=no
+ ac_outfile=conftest.$ac_objext
+ echo "$lt_simple_compile_test_code" > conftest.$ac_ext
+ lt_compiler_flag="-fno-rtti -fno-exceptions"
+ # Insert the option either (1) after the last *FLAGS variable, or
+ # (2) before a word containing "conftest.", or (3) at the end.
+ # Note that $ac_compile itself does not contain backslashes and begins
+ # with a dollar sign (not a hyphen), so the echo should work correctly.
+ # The option is referenced via a variable to avoid confusing sed.
+ lt_compile=`echo "$ac_compile" | $SED \
+ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
+ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
+ -e 's:$: $lt_compiler_flag:'`
+ (eval echo "\"\$as_me:8147: $lt_compile\"" >&5)
+ (eval "$lt_compile" 2>conftest.err)
+ ac_status=$?
+ cat conftest.err >&5
+ echo "$as_me:8151: \$? = $ac_status" >&5
+ if (exit $ac_status) && test -s "$ac_outfile"; then
+ # The compiler can only warn and ignore the option if not recognized
+ # So say no if there are warnings other than the usual output.
+ $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp
+ $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2
+ if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then
+ lt_cv_prog_compiler_rtti_exceptions=yes
+ fi
+ fi
+ $RM conftest*
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_rtti_exceptions" >&5
+$as_echo "$lt_cv_prog_compiler_rtti_exceptions" >&6; }
+
+if test x"$lt_cv_prog_compiler_rtti_exceptions" = xyes; then
+ lt_prog_compiler_no_builtin_flag="$lt_prog_compiler_no_builtin_flag -fno-rtti -fno-exceptions"
+else
+ :
+fi
+
+fi
+
+
+
+
+
+
+ lt_prog_compiler_wl=
+lt_prog_compiler_pic=
+lt_prog_compiler_static=
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $compiler option to produce PIC" >&5
+$as_echo_n "checking for $compiler option to produce PIC... " >&6; }
+
+ if test "$GCC" = yes; then
+ lt_prog_compiler_wl='-Wl,'
+ lt_prog_compiler_static='-static'
+
+ case $host_os in
+ aix*)
+ # All AIX code is PIC.
+ if test "$host_cpu" = ia64; then
+ # AIX 5 now supports IA64 processor
+ lt_prog_compiler_static='-Bstatic'
+ fi
+ ;;
+
+ amigaos*)
+ case $host_cpu in
+ powerpc)
+ # see comment about AmigaOS4 .so support
+ lt_prog_compiler_pic='-fPIC'
+ ;;
+ m68k)
+ # FIXME: we need at least 68020 code to build shared libraries, but
+ # adding the `-m68020' flag to GCC prevents building anything better,
+ # like `-m68040'.
+ lt_prog_compiler_pic='-m68020 -resident32 -malways-restore-a4'
+ ;;
+ esac
+ ;;
+
+ beos* | irix5* | irix6* | nonstopux* | osf3* | osf4* | osf5*)
+ # PIC is the default for these OSes.
+ ;;
+
+ mingw* | cygwin* | pw32* | os2* | cegcc*)
+ # This hack is so that the source file can tell whether it is being
+ # built for inclusion in a dll (and should export symbols for example).
+ # Although the cygwin gcc ignores -fPIC, still need this for old-style
+ # (--disable-auto-import) libraries
+ lt_prog_compiler_pic='-DDLL_EXPORT'
+ ;;
+
+ darwin* | rhapsody*)
+ # PIC is the default on this platform
+ # Common symbols not allowed in MH_DYLIB files
+ lt_prog_compiler_pic='-fno-common'
+ ;;
+
+ hpux*)
+ # PIC is the default for 64-bit PA HP-UX, but not for 32-bit
+ # PA HP-UX. On IA64 HP-UX, PIC is the default but the pic flag
+ # sets the default TLS model and affects inlining.
+ case $host_cpu in
+ hppa*64*)
+ # +Z the default
+ ;;
+ *)
+ lt_prog_compiler_pic='-fPIC'
+ ;;
+ esac
+ ;;
+
+ interix[3-9]*)
+ # Interix 3.x gcc -fpic/-fPIC options generate broken code.
+ # Instead, we relocate shared libraries at runtime.
+ ;;
+
+ msdosdjgpp*)
+ # Just because we use GCC doesn't mean we suddenly get shared libraries
+ # on systems that don't support them.
+ lt_prog_compiler_can_build_shared=no
+ enable_shared=no
+ ;;
+
+ *nto* | *qnx*)
+ # QNX uses GNU C++, but need to define -shared option too, otherwise
+ # it will coredump.
+ lt_prog_compiler_pic='-fPIC -shared'
+ ;;
+
+ sysv4*MP*)
+ if test -d /usr/nec; then
+ lt_prog_compiler_pic=-Kconform_pic
+ fi
+ ;;
+
+ *)
+ lt_prog_compiler_pic='-fPIC'
+ ;;
+ esac
+ else
+ # PORTME Check for flag to pass linker flags through the system compiler.
+ case $host_os in
+ aix*)
+ lt_prog_compiler_wl='-Wl,'
+ if test "$host_cpu" = ia64; then
+ # AIX 5 now supports IA64 processor
+ lt_prog_compiler_static='-Bstatic'
+ else
+ lt_prog_compiler_static='-bnso -bI:/lib/syscalls.exp'
+ fi
+ ;;
+
+ mingw* | cygwin* | pw32* | os2* | cegcc*)
+ # This hack is so that the source file can tell whether it is being
+ # built for inclusion in a dll (and should export symbols for example).
+ lt_prog_compiler_pic='-DDLL_EXPORT'
+ ;;
+
+ hpux9* | hpux10* | hpux11*)
+ lt_prog_compiler_wl='-Wl,'
+ # PIC is the default for IA64 HP-UX and 64-bit HP-UX, but
+ # not for PA HP-UX.
+ case $host_cpu in
+ hppa*64*|ia64*)
+ # +Z the default
+ ;;
+ *)
+ lt_prog_compiler_pic='+Z'
+ ;;
+ esac
+ # Is there a better lt_prog_compiler_static that works with the bundled CC?
+ lt_prog_compiler_static='${wl}-a ${wl}archive'
+ ;;
+
+ irix5* | irix6* | nonstopux*)
+ lt_prog_compiler_wl='-Wl,'
+ # PIC (with -KPIC) is the default.
+ lt_prog_compiler_static='-non_shared'
+ ;;
+
+ linux* | k*bsd*-gnu)
+ case $cc_basename in
+ # old Intel for x86_64 which still supported -KPIC.
+ ecc*)
+ lt_prog_compiler_wl='-Wl,'
+ lt_prog_compiler_pic='-KPIC'
+ lt_prog_compiler_static='-static'
+ ;;
+ # icc used to be incompatible with GCC.
+ # ICC 10 doesn't accept -KPIC any more.
+ icc* | ifort*)
+ lt_prog_compiler_wl='-Wl,'
+ lt_prog_compiler_pic='-fPIC'
+ lt_prog_compiler_static='-static'
+ ;;
+ # Lahey Fortran 8.1.
+ lf95*)
+ lt_prog_compiler_wl='-Wl,'
+ lt_prog_compiler_pic='--shared'
+ lt_prog_compiler_static='--static'
+ ;;
+ pgcc* | pgf77* | pgf90* | pgf95*)
+ # Portland Group compilers (*not* the Pentium gcc compiler,
+ # which looks to be a dead project)
+ lt_prog_compiler_wl='-Wl,'
+ lt_prog_compiler_pic='-fpic'
+ lt_prog_compiler_static='-Bstatic'
+ ;;
+ ccc*)
+ lt_prog_compiler_wl='-Wl,'
+ # All Alpha code is PIC.
+ lt_prog_compiler_static='-non_shared'
+ ;;
+ xl*)
+ # IBM XL C 8.0/Fortran 10.1 on PPC
+ lt_prog_compiler_wl='-Wl,'
+ lt_prog_compiler_pic='-qpic'
+ lt_prog_compiler_static='-qstaticlink'
+ ;;
+ *)
+ case `$CC -V 2>&1 | sed 5q` in
+ *Sun\ C*)
+ # Sun C 5.9
+ lt_prog_compiler_pic='-KPIC'
+ lt_prog_compiler_static='-Bstatic'
+ lt_prog_compiler_wl='-Wl,'
+ ;;
+ *Sun\ F*)
+ # Sun Fortran 8.3 passes all unrecognized flags to the linker
+ lt_prog_compiler_pic='-KPIC'
+ lt_prog_compiler_static='-Bstatic'
+ lt_prog_compiler_wl=''
+ ;;
+ esac
+ ;;
+ esac
+ ;;
+
+ newsos6)
+ lt_prog_compiler_pic='-KPIC'
+ lt_prog_compiler_static='-Bstatic'
+ ;;
+
+ *nto* | *qnx*)
+ # QNX uses GNU C++, but need to define -shared option too, otherwise
+ # it will coredump.
+ lt_prog_compiler_pic='-fPIC -shared'
+ ;;
+
+ osf3* | osf4* | osf5*)
+ lt_prog_compiler_wl='-Wl,'
+ # All OSF/1 code is PIC.
+ lt_prog_compiler_static='-non_shared'
+ ;;
+
+ rdos*)
+ lt_prog_compiler_static='-non_shared'
+ ;;
+
+ solaris*)
+ lt_prog_compiler_pic='-KPIC'
+ lt_prog_compiler_static='-Bstatic'
+ case $cc_basename in
+ f77* | f90* | f95*)
+ lt_prog_compiler_wl='-Qoption ld ';;
+ *)
+ lt_prog_compiler_wl='-Wl,';;
+ esac
+ ;;
+
+ sunos4*)
+ lt_prog_compiler_wl='-Qoption ld '
+ lt_prog_compiler_pic='-PIC'
+ lt_prog_compiler_static='-Bstatic'
+ ;;
+
+ sysv4 | sysv4.2uw2* | sysv4.3*)
+ lt_prog_compiler_wl='-Wl,'
+ lt_prog_compiler_pic='-KPIC'
+ lt_prog_compiler_static='-Bstatic'
+ ;;
+
+ sysv4*MP*)
+ if test -d /usr/nec ;then
+ lt_prog_compiler_pic='-Kconform_pic'
+ lt_prog_compiler_static='-Bstatic'
+ fi
+ ;;
+
+ sysv5* | unixware* | sco3.2v5* | sco5v6* | OpenUNIX*)
+ lt_prog_compiler_wl='-Wl,'
+ lt_prog_compiler_pic='-KPIC'
+ lt_prog_compiler_static='-Bstatic'
+ ;;
+
+ unicos*)
+ lt_prog_compiler_wl='-Wl,'
+ lt_prog_compiler_can_build_shared=no
+ ;;
+
+ uts4*)
+ lt_prog_compiler_pic='-pic'
+ lt_prog_compiler_static='-Bstatic'
+ ;;
+
+ *)
+ lt_prog_compiler_can_build_shared=no
+ ;;
+ esac
+ fi
+
+case $host_os in
+ # For platforms which do not support PIC, -DPIC is meaningless:
+ *djgpp*)
+ lt_prog_compiler_pic=
+ ;;
+ *)
+ lt_prog_compiler_pic="$lt_prog_compiler_pic -DPIC"
+ ;;
+esac
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_prog_compiler_pic" >&5
+$as_echo "$lt_prog_compiler_pic" >&6; }
+
+
+
+
+
+
+#
+# Check to make sure the PIC flag actually works.
+#
+if test -n "$lt_prog_compiler_pic"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler PIC flag $lt_prog_compiler_pic works" >&5
+$as_echo_n "checking if $compiler PIC flag $lt_prog_compiler_pic works... " >&6; }
+if test "${lt_cv_prog_compiler_pic_works+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ lt_cv_prog_compiler_pic_works=no
+ ac_outfile=conftest.$ac_objext
+ echo "$lt_simple_compile_test_code" > conftest.$ac_ext
+ lt_compiler_flag="$lt_prog_compiler_pic -DPIC"
+ # Insert the option either (1) after the last *FLAGS variable, or
+ # (2) before a word containing "conftest.", or (3) at the end.
+ # Note that $ac_compile itself does not contain backslashes and begins
+ # with a dollar sign (not a hyphen), so the echo should work correctly.
+ # The option is referenced via a variable to avoid confusing sed.
+ lt_compile=`echo "$ac_compile" | $SED \
+ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
+ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
+ -e 's:$: $lt_compiler_flag:'`
+ (eval echo "\"\$as_me:8486: $lt_compile\"" >&5)
+ (eval "$lt_compile" 2>conftest.err)
+ ac_status=$?
+ cat conftest.err >&5
+ echo "$as_me:8490: \$? = $ac_status" >&5
+ if (exit $ac_status) && test -s "$ac_outfile"; then
+ # The compiler can only warn and ignore the option if not recognized
+ # So say no if there are warnings other than the usual output.
+ $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp
+ $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2
+ if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then
+ lt_cv_prog_compiler_pic_works=yes
+ fi
+ fi
+ $RM conftest*
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_pic_works" >&5
+$as_echo "$lt_cv_prog_compiler_pic_works" >&6; }
+
+if test x"$lt_cv_prog_compiler_pic_works" = xyes; then
+ case $lt_prog_compiler_pic in
+ "" | " "*) ;;
+ *) lt_prog_compiler_pic=" $lt_prog_compiler_pic" ;;
+ esac
+else
+ lt_prog_compiler_pic=
+ lt_prog_compiler_can_build_shared=no
+fi
+
+fi
+
+
+
+
+
+
+#
+# Check to make sure the static flag actually works.
+#
+wl=$lt_prog_compiler_wl eval lt_tmp_static_flag=\"$lt_prog_compiler_static\"
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler static flag $lt_tmp_static_flag works" >&5
+$as_echo_n "checking if $compiler static flag $lt_tmp_static_flag works... " >&6; }
+if test "${lt_cv_prog_compiler_static_works+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ lt_cv_prog_compiler_static_works=no
+ save_LDFLAGS="$LDFLAGS"
+ LDFLAGS="$LDFLAGS $lt_tmp_static_flag"
+ echo "$lt_simple_link_test_code" > conftest.$ac_ext
+ if (eval $ac_link 2>conftest.err) && test -s conftest$ac_exeext; then
+ # The linker can only warn and ignore the option if not recognized
+ # So say no if there are warnings
+ if test -s conftest.err; then
+ # Append any errors to the config.log.
+ cat conftest.err 1>&5
+ $ECHO "X$_lt_linker_boilerplate" | $Xsed -e '/^$/d' > conftest.exp
+ $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2
+ if diff conftest.exp conftest.er2 >/dev/null; then
+ lt_cv_prog_compiler_static_works=yes
+ fi
+ else
+ lt_cv_prog_compiler_static_works=yes
+ fi
+ fi
+ $RM -r conftest*
+ LDFLAGS="$save_LDFLAGS"
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_static_works" >&5
+$as_echo "$lt_cv_prog_compiler_static_works" >&6; }
+
+if test x"$lt_cv_prog_compiler_static_works" = xyes; then
+ :
+else
+ lt_prog_compiler_static=
+fi
+
+
+
+
+
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler supports -c -o file.$ac_objext" >&5
+$as_echo_n "checking if $compiler supports -c -o file.$ac_objext... " >&6; }
+if test "${lt_cv_prog_compiler_c_o+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ lt_cv_prog_compiler_c_o=no
+ $RM -r conftest 2>/dev/null
+ mkdir conftest
+ cd conftest
+ mkdir out
+ echo "$lt_simple_compile_test_code" > conftest.$ac_ext
+
+ lt_compiler_flag="-o out/conftest2.$ac_objext"
+ # Insert the option either (1) after the last *FLAGS variable, or
+ # (2) before a word containing "conftest.", or (3) at the end.
+ # Note that $ac_compile itself does not contain backslashes and begins
+ # with a dollar sign (not a hyphen), so the echo should work correctly.
+ lt_compile=`echo "$ac_compile" | $SED \
+ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
+ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
+ -e 's:$: $lt_compiler_flag:'`
+ (eval echo "\"\$as_me:8591: $lt_compile\"" >&5)
+ (eval "$lt_compile" 2>out/conftest.err)
+ ac_status=$?
+ cat out/conftest.err >&5
+ echo "$as_me:8595: \$? = $ac_status" >&5
+ if (exit $ac_status) && test -s out/conftest2.$ac_objext
+ then
+ # The compiler can only warn and ignore the option if not recognized
+ # So say no if there are warnings
+ $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp
+ $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2
+ if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then
+ lt_cv_prog_compiler_c_o=yes
+ fi
+ fi
+ chmod u+w . 2>&5
+ $RM conftest*
+ # SGI C++ compiler will create directory out/ii_files/ for
+ # template instantiation
+ test -d out/ii_files && $RM out/ii_files/* && rmdir out/ii_files
+ $RM out/* && rmdir out
+ cd ..
+ $RM -r conftest
+ $RM conftest*
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_c_o" >&5
+$as_echo "$lt_cv_prog_compiler_c_o" >&6; }
+
+
+
+
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler supports -c -o file.$ac_objext" >&5
+$as_echo_n "checking if $compiler supports -c -o file.$ac_objext... " >&6; }
+if test "${lt_cv_prog_compiler_c_o+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ lt_cv_prog_compiler_c_o=no
+ $RM -r conftest 2>/dev/null
+ mkdir conftest
+ cd conftest
+ mkdir out
+ echo "$lt_simple_compile_test_code" > conftest.$ac_ext
+
+ lt_compiler_flag="-o out/conftest2.$ac_objext"
+ # Insert the option either (1) after the last *FLAGS variable, or
+ # (2) before a word containing "conftest.", or (3) at the end.
+ # Note that $ac_compile itself does not contain backslashes and begins
+ # with a dollar sign (not a hyphen), so the echo should work correctly.
+ lt_compile=`echo "$ac_compile" | $SED \
+ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
+ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
+ -e 's:$: $lt_compiler_flag:'`
+ (eval echo "\"\$as_me:8646: $lt_compile\"" >&5)
+ (eval "$lt_compile" 2>out/conftest.err)
+ ac_status=$?
+ cat out/conftest.err >&5
+ echo "$as_me:8650: \$? = $ac_status" >&5
+ if (exit $ac_status) && test -s out/conftest2.$ac_objext
+ then
+ # The compiler can only warn and ignore the option if not recognized
+ # So say no if there are warnings
+ $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp
+ $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2
+ if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then
+ lt_cv_prog_compiler_c_o=yes
+ fi
+ fi
+ chmod u+w . 2>&5
+ $RM conftest*
+ # SGI C++ compiler will create directory out/ii_files/ for
+ # template instantiation
+ test -d out/ii_files && $RM out/ii_files/* && rmdir out/ii_files
+ $RM out/* && rmdir out
+ cd ..
+ $RM -r conftest
+ $RM conftest*
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_c_o" >&5
+$as_echo "$lt_cv_prog_compiler_c_o" >&6; }
+
+
+
+
+hard_links="nottested"
+if test "$lt_cv_prog_compiler_c_o" = no && test "$need_locks" != no; then
+ # do not overwrite the value of need_locks provided by the user
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking if we can lock with hard links" >&5
+$as_echo_n "checking if we can lock with hard links... " >&6; }
+ hard_links=yes
+ $RM conftest*
+ ln conftest.a conftest.b 2>/dev/null && hard_links=no
+ touch conftest.a
+ ln conftest.a conftest.b 2>&5 || hard_links=no
+ ln conftest.a conftest.b 2>/dev/null && hard_links=no
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $hard_links" >&5
+$as_echo "$hard_links" >&6; }
+ if test "$hard_links" = no; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: \`$CC' does not support \`-c -o', so \`make -j' may be unsafe" >&5
+$as_echo "$as_me: WARNING: \`$CC' does not support \`-c -o', so \`make -j' may be unsafe" >&2;}
+ need_locks=warn
+ fi
+else
+ need_locks=no
+fi
+
+
+
+
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the $compiler linker ($LD) supports shared libraries" >&5
+$as_echo_n "checking whether the $compiler linker ($LD) supports shared libraries... " >&6; }
+
+ runpath_var=
+ allow_undefined_flag=
+ always_export_symbols=no
+ archive_cmds=
+ archive_expsym_cmds=
+ compiler_needs_object=no
+ enable_shared_with_static_runtimes=no
+ export_dynamic_flag_spec=
+ export_symbols_cmds='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols'
+ hardcode_automatic=no
+ hardcode_direct=no
+ hardcode_direct_absolute=no
+ hardcode_libdir_flag_spec=
+ hardcode_libdir_flag_spec_ld=
+ hardcode_libdir_separator=
+ hardcode_minus_L=no
+ hardcode_shlibpath_var=unsupported
+ inherit_rpath=no
+ link_all_deplibs=unknown
+ module_cmds=
+ module_expsym_cmds=
+ old_archive_from_new_cmds=
+ old_archive_from_expsyms_cmds=
+ thread_safe_flag_spec=
+ whole_archive_flag_spec=
+ # include_expsyms should be a list of space-separated symbols to be *always*
+ # included in the symbol list
+ include_expsyms=
+ # exclude_expsyms can be an extended regexp of symbols to exclude
+ # it will be wrapped by ` (' and `)$', so one must not match beginning or
+ # end of line. Example: `a|bc|.*d.*' will exclude the symbols `a' and `bc',
+ # as well as any symbol that contains `d'.
+ exclude_expsyms='_GLOBAL_OFFSET_TABLE_|_GLOBAL__F[ID]_.*'
+ # Although _GLOBAL_OFFSET_TABLE_ is a valid symbol C name, most a.out
+ # platforms (ab)use it in PIC code, but their linkers get confused if
+ # the symbol is explicitly referenced. Since portable code cannot
+ # rely on this symbol name, it's probably fine to never include it in
+ # preloaded symbol tables.
+ # Exclude shared library initialization/finalization symbols.
+ extract_expsyms_cmds=
+
+ case $host_os in
+ cygwin* | mingw* | pw32* | cegcc*)
+ # FIXME: the MSVC++ port hasn't been tested in a loooong time
+ # When not using gcc, we currently assume that we are using
+ # Microsoft Visual C++.
+ if test "$GCC" != yes; then
+ with_gnu_ld=no
+ fi
+ ;;
+ interix*)
+ # we just hope/assume this is gcc and not c89 (= MSVC++)
+ with_gnu_ld=yes
+ ;;
+ openbsd*)
+ with_gnu_ld=no
+ ;;
+ esac
+
+ ld_shlibs=yes
+ if test "$with_gnu_ld" = yes; then
+ # If archive_cmds runs LD, not CC, wlarc should be empty
+ wlarc='${wl}'
+
+ # Set some defaults for GNU ld with shared library support. These
+ # are reset later if shared libraries are not supported. Putting them
+ # here allows them to be overridden if necessary.
+ runpath_var=LD_RUN_PATH
+ hardcode_libdir_flag_spec='${wl}-rpath ${wl}$libdir'
+ export_dynamic_flag_spec='${wl}--export-dynamic'
+ # ancient GNU ld didn't support --whole-archive et. al.
+ if $LD --help 2>&1 | $GREP 'no-whole-archive' > /dev/null; then
+ whole_archive_flag_spec="$wlarc"'--whole-archive$convenience '"$wlarc"'--no-whole-archive'
+ else
+ whole_archive_flag_spec=
+ fi
+ supports_anon_versioning=no
+ case `$LD -v 2>&1` in
+ *\ [01].* | *\ 2.[0-9].* | *\ 2.10.*) ;; # catch versions < 2.11
+ *\ 2.11.93.0.2\ *) supports_anon_versioning=yes ;; # RH7.3 ...
+ *\ 2.11.92.0.12\ *) supports_anon_versioning=yes ;; # Mandrake 8.2 ...
+ *\ 2.11.*) ;; # other 2.11 versions
+ *) supports_anon_versioning=yes ;;
+ esac
+
+ # See if GNU ld supports shared libraries.
+ case $host_os in
+ aix[3-9]*)
+ # On AIX/PPC, the GNU linker is very broken
+ if test "$host_cpu" != ia64; then
+ ld_shlibs=no
+ cat <<_LT_EOF 1>&2
+
+*** Warning: the GNU linker, at least up to release 2.9.1, is reported
+*** to be unable to reliably create shared libraries on AIX.
+*** Therefore, libtool is disabling shared libraries support. If you
+*** really care for shared libraries, you may want to modify your PATH
+*** so that a non-GNU linker is found, and then restart.
+
+_LT_EOF
+ fi
+ ;;
+
+ amigaos*)
+ case $host_cpu in
+ powerpc)
+ # see comment about AmigaOS4 .so support
+ archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ archive_expsym_cmds=''
+ ;;
+ m68k)
+ archive_cmds='$RM $output_objdir/a2ixlibrary.data~$ECHO "#define NAME $libname" > $output_objdir/a2ixlibrary.data~$ECHO "#define LIBRARY_ID 1" >> $output_objdir/a2ixlibrary.data~$ECHO "#define VERSION $major" >> $output_objdir/a2ixlibrary.data~$ECHO "#define REVISION $revision" >> $output_objdir/a2ixlibrary.data~$AR $AR_FLAGS $lib $libobjs~$RANLIB $lib~(cd $output_objdir && a2ixlibrary -32)'
+ hardcode_libdir_flag_spec='-L$libdir'
+ hardcode_minus_L=yes
+ ;;
+ esac
+ ;;
+
+ beos*)
+ if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then
+ allow_undefined_flag=unsupported
+ # Joseph Beckenbach <jrb3 at best.com> says some releases of gcc
+ # support --undefined. This deserves some investigation. FIXME
+ archive_cmds='$CC -nostart $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ else
+ ld_shlibs=no
+ fi
+ ;;
+
+ cygwin* | mingw* | pw32* | cegcc*)
+ # _LT_TAGVAR(hardcode_libdir_flag_spec, ) is actually meaningless,
+ # as there is no search path for DLLs.
+ hardcode_libdir_flag_spec='-L$libdir'
+ allow_undefined_flag=unsupported
+ always_export_symbols=no
+ enable_shared_with_static_runtimes=yes
+ export_symbols_cmds='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[BCDGRS][ ]/s/.*[ ]\([^ ]*\)/\1 DATA/'\'' | $SED -e '\''/^[AITW][ ]/s/.*[ ]//'\'' | sort | uniq > $export_symbols'
+
+ if $LD --help 2>&1 | $GREP 'auto-import' > /dev/null; then
+ archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib'
+ # If the export-symbols file already is a .def file (1st line
+ # is EXPORTS), use it as is; otherwise, prepend...
+ archive_expsym_cmds='if test "x`$SED 1q $export_symbols`" = xEXPORTS; then
+ cp $export_symbols $output_objdir/$soname.def;
+ else
+ echo EXPORTS > $output_objdir/$soname.def;
+ cat $export_symbols >> $output_objdir/$soname.def;
+ fi~
+ $CC -shared $output_objdir/$soname.def $libobjs $deplibs $compiler_flags -o $output_objdir/$soname ${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib'
+ else
+ ld_shlibs=no
+ fi
+ ;;
+
+ interix[3-9]*)
+ hardcode_direct=no
+ hardcode_shlibpath_var=no
+ hardcode_libdir_flag_spec='${wl}-rpath,$libdir'
+ export_dynamic_flag_spec='${wl}-E'
+ # Hack: On Interix 3.x, we cannot compile PIC because of a broken gcc.
+ # Instead, shared libraries are loaded at an image base (0x10000000 by
+ # default) and relocated if they conflict, which is a slow very memory
+ # consuming and fragmenting process. To avoid this, we pick a random,
+ # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link
+ # time. Moving up from 0x10000000 also allows more sbrk(2) space.
+ archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib'
+ archive_expsym_cmds='sed "s,^,_," $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags ${wl}-h,$soname ${wl}--retain-symbols-file,$output_objdir/$soname.expsym ${wl}--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib'
+ ;;
+
+ gnu* | linux* | tpf* | k*bsd*-gnu)
+ tmp_diet=no
+ if test "$host_os" = linux-dietlibc; then
+ case $cc_basename in
+ diet\ *) tmp_diet=yes;; # linux-dietlibc with static linking (!diet-dyn)
+ esac
+ fi
+ if $LD --help 2>&1 | $EGREP ': supported targets:.* elf' > /dev/null \
+ && test "$tmp_diet" = no
+ then
+ tmp_addflag=
+ tmp_sharedflag='-shared'
+ case $cc_basename,$host_cpu in
+ pgcc*) # Portland Group C compiler
+ whole_archive_flag_spec='${wl}--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; $ECHO \"$new_convenience\"` ${wl}--no-whole-archive'
+ tmp_addflag=' $pic_flag'
+ ;;
+ pgf77* | pgf90* | pgf95*) # Portland Group f77 and f90 compilers
+ whole_archive_flag_spec='${wl}--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; $ECHO \"$new_convenience\"` ${wl}--no-whole-archive'
+ tmp_addflag=' $pic_flag -Mnomain' ;;
+ ecc*,ia64* | icc*,ia64*) # Intel C compiler on ia64
+ tmp_addflag=' -i_dynamic' ;;
+ efc*,ia64* | ifort*,ia64*) # Intel Fortran compiler on ia64
+ tmp_addflag=' -i_dynamic -nofor_main' ;;
+ ifc* | ifort*) # Intel Fortran compiler
+ tmp_addflag=' -nofor_main' ;;
+ lf95*) # Lahey Fortran 8.1
+ whole_archive_flag_spec=
+ tmp_sharedflag='--shared' ;;
+ xl[cC]*) # IBM XL C 8.0 on PPC (deal with xlf below)
+ tmp_sharedflag='-qmkshrobj'
+ tmp_addflag= ;;
+ esac
+ case `$CC -V 2>&1 | sed 5q` in
+ *Sun\ C*) # Sun C 5.9
+ whole_archive_flag_spec='${wl}--whole-archive`new_convenience=; for conv in $convenience\"\"; do test -z \"$conv\" || new_convenience=\"$new_convenience,$conv\"; done; $ECHO \"$new_convenience\"` ${wl}--no-whole-archive'
+ compiler_needs_object=yes
+ tmp_sharedflag='-G' ;;
+ *Sun\ F*) # Sun Fortran 8.3
+ tmp_sharedflag='-G' ;;
+ esac
+ archive_cmds='$CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+
+ if test "x$supports_anon_versioning" = xyes; then
+ archive_expsym_cmds='echo "{ global:" > $output_objdir/$libname.ver~
+ cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~
+ echo "local: *; };" >> $output_objdir/$libname.ver~
+ $CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-version-script ${wl}$output_objdir/$libname.ver -o $lib'
+ fi
+
+ case $cc_basename in
+ xlf*)
+ # IBM XL Fortran 10.1 on PPC cannot create shared libs itself
+ whole_archive_flag_spec='--whole-archive$convenience --no-whole-archive'
+ hardcode_libdir_flag_spec=
+ hardcode_libdir_flag_spec_ld='-rpath $libdir'
+ archive_cmds='$LD -shared $libobjs $deplibs $compiler_flags -soname $soname -o $lib'
+ if test "x$supports_anon_versioning" = xyes; then
+ archive_expsym_cmds='echo "{ global:" > $output_objdir/$libname.ver~
+ cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~
+ echo "local: *; };" >> $output_objdir/$libname.ver~
+ $LD -shared $libobjs $deplibs $compiler_flags -soname $soname -version-script $output_objdir/$libname.ver -o $lib'
+ fi
+ ;;
+ esac
+ else
+ ld_shlibs=no
+ fi
+ ;;
+
+ netbsd*)
+ if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then
+ archive_cmds='$LD -Bshareable $libobjs $deplibs $linker_flags -o $lib'
+ wlarc=
+ else
+ archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ archive_expsym_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib'
+ fi
+ ;;
+
+ solaris*)
+ if $LD -v 2>&1 | $GREP 'BFD 2\.8' > /dev/null; then
+ ld_shlibs=no
+ cat <<_LT_EOF 1>&2
+
+*** Warning: The releases 2.8.* of the GNU linker cannot reliably
+*** create shared libraries on Solaris systems. Therefore, libtool
+*** is disabling shared libraries support. We urge you to upgrade GNU
+*** binutils to release 2.9.1 or newer. Another option is to modify
+*** your PATH or compiler configuration so that the native linker is
+*** used, and then restart.
+
+_LT_EOF
+ elif $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then
+ archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ archive_expsym_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib'
+ else
+ ld_shlibs=no
+ fi
+ ;;
+
+ sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX*)
+ case `$LD -v 2>&1` in
+ *\ [01].* | *\ 2.[0-9].* | *\ 2.1[0-5].*)
+ ld_shlibs=no
+ cat <<_LT_EOF 1>&2
+
+*** Warning: Releases of the GNU linker prior to 2.16.91.0.3 can not
+*** reliably create shared libraries on SCO systems. Therefore, libtool
+*** is disabling shared libraries support. We urge you to upgrade GNU
+*** binutils to release 2.16.91.0.3 or newer. Another option is to modify
+*** your PATH or compiler configuration so that the native linker is
+*** used, and then restart.
+
+_LT_EOF
+ ;;
+ *)
+ # For security reasons, it is highly recommended that you always
+ # use absolute paths for naming shared libraries, and exclude the
+ # DT_RUNPATH tag from executables and libraries. But doing so
+ # requires that you compile everything twice, which is a pain.
+ if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then
+ hardcode_libdir_flag_spec='${wl}-rpath ${wl}$libdir'
+ archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ archive_expsym_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib'
+ else
+ ld_shlibs=no
+ fi
+ ;;
+ esac
+ ;;
+
+ sunos4*)
+ archive_cmds='$LD -assert pure-text -Bshareable -o $lib $libobjs $deplibs $linker_flags'
+ wlarc=
+ hardcode_direct=yes
+ hardcode_shlibpath_var=no
+ ;;
+
+ *)
+ if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then
+ archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ archive_expsym_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname ${wl}-retain-symbols-file $wl$export_symbols -o $lib'
+ else
+ ld_shlibs=no
+ fi
+ ;;
+ esac
+
+ if test "$ld_shlibs" = no; then
+ runpath_var=
+ hardcode_libdir_flag_spec=
+ export_dynamic_flag_spec=
+ whole_archive_flag_spec=
+ fi
+ else
+ # PORTME fill in a description of your system's linker (not GNU ld)
+ case $host_os in
+ aix3*)
+ allow_undefined_flag=unsupported
+ always_export_symbols=yes
+ archive_expsym_cmds='$LD -o $output_objdir/$soname $libobjs $deplibs $linker_flags -bE:$export_symbols -T512 -H512 -bM:SRE~$AR $AR_FLAGS $lib $output_objdir/$soname'
+ # Note: this linker hardcodes the directories in LIBPATH if there
+ # are no directories specified by -L.
+ hardcode_minus_L=yes
+ if test "$GCC" = yes && test -z "$lt_prog_compiler_static"; then
+ # Neither direct hardcoding nor static linking is supported with a
+ # broken collect2.
+ hardcode_direct=unsupported
+ fi
+ ;;
+
+ aix[4-9]*)
+ if test "$host_cpu" = ia64; then
+ # On IA64, the linker does run time linking by default, so we don't
+ # have to do anything special.
+ aix_use_runtimelinking=no
+ exp_sym_flag='-Bexport'
+ no_entry_flag=""
+ else
+ # If we're using GNU nm, then we don't want the "-C" option.
+ # -C means demangle to AIX nm, but means don't demangle with GNU nm
+ if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then
+ export_symbols_cmds='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B")) && (substr(\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols'
+ else
+ export_symbols_cmds='$NM -BCpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B")) && (substr(\$ 3,1,1) != ".")) { print \$ 3 } }'\'' | sort -u > $export_symbols'
+ fi
+ aix_use_runtimelinking=no
+
+ # Test if we are trying to use run time linking or normal
+ # AIX style linking. If -brtl is somewhere in LDFLAGS, we
+ # need to do runtime linking.
+ case $host_os in aix4.[23]|aix4.[23].*|aix[5-9]*)
+ for ld_flag in $LDFLAGS; do
+ if (test $ld_flag = "-brtl" || test $ld_flag = "-Wl,-brtl"); then
+ aix_use_runtimelinking=yes
+ break
+ fi
+ done
+ ;;
+ esac
+
+ exp_sym_flag='-bexport'
+ no_entry_flag='-bnoentry'
+ fi
+
+ # When large executables or shared objects are built, AIX ld can
+ # have problems creating the table of contents. If linking a library
+ # or program results in "error TOC overflow" add -mminimal-toc to
+ # CXXFLAGS/CFLAGS for g++/gcc. In the cases where that is not
+ # enough to fix the problem, add -Wl,-bbigtoc to LDFLAGS.
+
+ archive_cmds=''
+ hardcode_direct=yes
+ hardcode_direct_absolute=yes
+ hardcode_libdir_separator=':'
+ link_all_deplibs=yes
+ file_list_spec='${wl}-f,'
+
+ if test "$GCC" = yes; then
+ case $host_os in aix4.[012]|aix4.[012].*)
+ # We only want to do this on AIX 4.2 and lower, the check
+ # below for broken collect2 doesn't work under 4.3+
+ collect2name=`${CC} -print-prog-name=collect2`
+ if test -f "$collect2name" &&
+ strings "$collect2name" | $GREP resolve_lib_name >/dev/null
+ then
+ # We have reworked collect2
+ :
+ else
+ # We have old collect2
+ hardcode_direct=unsupported
+ # It fails to find uninstalled libraries when the uninstalled
+ # path is not listed in the libpath. Setting hardcode_minus_L
+ # to unsupported forces relinking
+ hardcode_minus_L=yes
+ hardcode_libdir_flag_spec='-L$libdir'
+ hardcode_libdir_separator=
+ fi
+ ;;
+ esac
+ shared_flag='-shared'
+ if test "$aix_use_runtimelinking" = yes; then
+ shared_flag="$shared_flag "'${wl}-G'
+ fi
+ else
+ # not using gcc
+ if test "$host_cpu" = ia64; then
+ # VisualAge C++, Version 5.5 for AIX 5L for IA-64, Beta 3 Release
+ # chokes on -Wl,-G. The following line is correct:
+ shared_flag='-G'
+ else
+ if test "$aix_use_runtimelinking" = yes; then
+ shared_flag='${wl}-G'
+ else
+ shared_flag='${wl}-bM:SRE'
+ fi
+ fi
+ fi
+
+ export_dynamic_flag_spec='${wl}-bexpall'
+ # It seems that -bexpall does not export symbols beginning with
+ # underscore (_), so it is better to generate a list of symbols to export.
+ always_export_symbols=yes
+ if test "$aix_use_runtimelinking" = yes; then
+ # Warning - without using the other runtime loading flags (-brtl),
+ # -berok will link without error, but may produce a broken library.
+ allow_undefined_flag='-berok'
+ # Determine the default libpath from the value encoded in an
+ # empty executable.
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+
+lt_aix_libpath_sed='
+ /Import File Strings/,/^$/ {
+ /^0/ {
+ s/^0 *\(.*\)$/\1/
+ p
+ }
+ }'
+aix_libpath=`dump -H conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"`
+# Check for a 64-bit object if we didn't find anything.
+if test -z "$aix_libpath"; then
+ aix_libpath=`dump -HX64 conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"`
+fi
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi
+
+ hardcode_libdir_flag_spec='${wl}-blibpath:$libdir:'"$aix_libpath"
+ archive_expsym_cmds='$CC -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags `if test "x${allow_undefined_flag}" != "x"; then $ECHO "X${wl}${allow_undefined_flag}" | $Xsed; else :; fi` '"\${wl}$exp_sym_flag:\$export_symbols $shared_flag"
+ else
+ if test "$host_cpu" = ia64; then
+ hardcode_libdir_flag_spec='${wl}-R $libdir:/usr/lib:/lib'
+ allow_undefined_flag="-z nodefs"
+ archive_expsym_cmds="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs '"\${wl}$no_entry_flag"' $compiler_flags ${wl}${allow_undefined_flag} '"\${wl}$exp_sym_flag:\$export_symbols"
+ else
+ # Determine the default libpath from the value encoded in an
+ # empty executable.
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+
+lt_aix_libpath_sed='
+ /Import File Strings/,/^$/ {
+ /^0/ {
+ s/^0 *\(.*\)$/\1/
+ p
+ }
+ }'
+aix_libpath=`dump -H conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"`
+# Check for a 64-bit object if we didn't find anything.
+if test -z "$aix_libpath"; then
+ aix_libpath=`dump -HX64 conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"`
+fi
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi
+
+ hardcode_libdir_flag_spec='${wl}-blibpath:$libdir:'"$aix_libpath"
+ # Warning - without using the other run time loading flags,
+ # -berok will link without error, but may produce a broken library.
+ no_undefined_flag=' ${wl}-bernotok'
+ allow_undefined_flag=' ${wl}-berok'
+ # Exported symbols can be pulled into shared objects from archives
+ whole_archive_flag_spec='$convenience'
+ archive_cmds_need_lc=yes
+ # This is similar to how AIX traditionally builds its shared libraries.
+ archive_expsym_cmds="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs ${wl}-bnoentry $compiler_flags ${wl}-bE:$export_symbols${allow_undefined_flag}~$AR $AR_FLAGS $output_objdir/$libname$release.a $output_objdir/$soname'
+ fi
+ fi
+ ;;
+
+ amigaos*)
+ case $host_cpu in
+ powerpc)
+ # see comment about AmigaOS4 .so support
+ archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname $wl$soname -o $lib'
+ archive_expsym_cmds=''
+ ;;
+ m68k)
+ archive_cmds='$RM $output_objdir/a2ixlibrary.data~$ECHO "#define NAME $libname" > $output_objdir/a2ixlibrary.data~$ECHO "#define LIBRARY_ID 1" >> $output_objdir/a2ixlibrary.data~$ECHO "#define VERSION $major" >> $output_objdir/a2ixlibrary.data~$ECHO "#define REVISION $revision" >> $output_objdir/a2ixlibrary.data~$AR $AR_FLAGS $lib $libobjs~$RANLIB $lib~(cd $output_objdir && a2ixlibrary -32)'
+ hardcode_libdir_flag_spec='-L$libdir'
+ hardcode_minus_L=yes
+ ;;
+ esac
+ ;;
+
+ bsdi[45]*)
+ export_dynamic_flag_spec=-rdynamic
+ ;;
+
+ cygwin* | mingw* | pw32* | cegcc*)
+ # When not using gcc, we currently assume that we are using
+ # Microsoft Visual C++.
+ # hardcode_libdir_flag_spec is actually meaningless, as there is
+ # no search path for DLLs.
+ hardcode_libdir_flag_spec=' '
+ allow_undefined_flag=unsupported
+ # Tell ltmain to make .lib files, not .a files.
+ libext=lib
+ # Tell ltmain to make .dll files, not .so files.
+ shrext_cmds=".dll"
+ # FIXME: Setting linknames here is a bad hack.
+ archive_cmds='$CC -o $lib $libobjs $compiler_flags `$ECHO "X$deplibs" | $Xsed -e '\''s/ -lc$//'\''` -link -dll~linknames='
+ # The linker will automatically build a .lib file if we build a DLL.
+ old_archive_from_new_cmds='true'
+ # FIXME: Should let the user specify the lib program.
+ old_archive_cmds='lib -OUT:$oldlib$oldobjs$old_deplibs'
+ fix_srcfile_path='`cygpath -w "$srcfile"`'
+ enable_shared_with_static_runtimes=yes
+ ;;
+
+ darwin* | rhapsody*)
+
+
+ archive_cmds_need_lc=no
+ hardcode_direct=no
+ hardcode_automatic=yes
+ hardcode_shlibpath_var=unsupported
+ whole_archive_flag_spec=''
+ link_all_deplibs=yes
+ allow_undefined_flag="$_lt_dar_allow_undefined"
+ case $cc_basename in
+ ifort*) _lt_dar_can_shared=yes ;;
+ *) _lt_dar_can_shared=$GCC ;;
+ esac
+ if test "$_lt_dar_can_shared" = "yes"; then
+ output_verbose_link_cmd=echo
+ archive_cmds="\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod${_lt_dsymutil}"
+ module_cmds="\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags${_lt_dsymutil}"
+ archive_expsym_cmds="sed 's,^,_,' < \$export_symbols > \$output_objdir/\${libname}-symbols.expsym~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring ${_lt_dar_single_mod}${_lt_dar_export_syms}${_lt_dsymutil}"
+ module_expsym_cmds="sed -e 's,^,_,' < \$export_symbols > \$output_objdir/\${libname}-symbols.expsym~\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags${_lt_dar_export_syms}${_lt_dsymutil}"
+
+ else
+ ld_shlibs=no
+ fi
+
+ ;;
+
+ dgux*)
+ archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ hardcode_libdir_flag_spec='-L$libdir'
+ hardcode_shlibpath_var=no
+ ;;
+
+ freebsd1*)
+ ld_shlibs=no
+ ;;
+
+ # FreeBSD 2.2.[012] allows us to include c++rt0.o to get C++ constructor
+ # support. Future versions do this automatically, but an explicit c++rt0.o
+ # does not break anything, and helps significantly (at the cost of a little
+ # extra space).
+ freebsd2.2*)
+ archive_cmds='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags /usr/lib/c++rt0.o'
+ hardcode_libdir_flag_spec='-R$libdir'
+ hardcode_direct=yes
+ hardcode_shlibpath_var=no
+ ;;
+
+ # Unfortunately, older versions of FreeBSD 2 do not have this feature.
+ freebsd2*)
+ archive_cmds='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags'
+ hardcode_direct=yes
+ hardcode_minus_L=yes
+ hardcode_shlibpath_var=no
+ ;;
+
+ # FreeBSD 3 and greater uses gcc -shared to do shared libraries.
+ freebsd* | dragonfly*)
+ archive_cmds='$CC -shared -o $lib $libobjs $deplibs $compiler_flags'
+ hardcode_libdir_flag_spec='-R$libdir'
+ hardcode_direct=yes
+ hardcode_shlibpath_var=no
+ ;;
+
+ hpux9*)
+ if test "$GCC" = yes; then
+ archive_cmds='$RM $output_objdir/$soname~$CC -shared -fPIC ${wl}+b ${wl}$install_libdir -o $output_objdir/$soname $libobjs $deplibs $compiler_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib'
+ else
+ archive_cmds='$RM $output_objdir/$soname~$LD -b +b $install_libdir -o $output_objdir/$soname $libobjs $deplibs $linker_flags~test $output_objdir/$soname = $lib || mv $output_objdir/$soname $lib'
+ fi
+ hardcode_libdir_flag_spec='${wl}+b ${wl}$libdir'
+ hardcode_libdir_separator=:
+ hardcode_direct=yes
+
+ # hardcode_minus_L: Not really in the search PATH,
+ # but as the default location of the library.
+ hardcode_minus_L=yes
+ export_dynamic_flag_spec='${wl}-E'
+ ;;
+
+ hpux10*)
+ if test "$GCC" = yes -a "$with_gnu_ld" = no; then
+ archive_cmds='$CC -shared -fPIC ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $libobjs $deplibs $compiler_flags'
+ else
+ archive_cmds='$LD -b +h $soname +b $install_libdir -o $lib $libobjs $deplibs $linker_flags'
+ fi
+ if test "$with_gnu_ld" = no; then
+ hardcode_libdir_flag_spec='${wl}+b ${wl}$libdir'
+ hardcode_libdir_flag_spec_ld='+b $libdir'
+ hardcode_libdir_separator=:
+ hardcode_direct=yes
+ hardcode_direct_absolute=yes
+ export_dynamic_flag_spec='${wl}-E'
+ # hardcode_minus_L: Not really in the search PATH,
+ # but as the default location of the library.
+ hardcode_minus_L=yes
+ fi
+ ;;
+
+ hpux11*)
+ if test "$GCC" = yes -a "$with_gnu_ld" = no; then
+ case $host_cpu in
+ hppa*64*)
+ archive_cmds='$CC -shared ${wl}+h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ ia64*)
+ archive_cmds='$CC -shared -fPIC ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ *)
+ archive_cmds='$CC -shared -fPIC ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ esac
+ else
+ case $host_cpu in
+ hppa*64*)
+ archive_cmds='$CC -b ${wl}+h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ ia64*)
+ archive_cmds='$CC -b ${wl}+h ${wl}$soname ${wl}+nodefaultrpath -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ *)
+ archive_cmds='$CC -b ${wl}+h ${wl}$soname ${wl}+b ${wl}$install_libdir -o $lib $libobjs $deplibs $compiler_flags'
+ ;;
+ esac
+ fi
+ if test "$with_gnu_ld" = no; then
+ hardcode_libdir_flag_spec='${wl}+b ${wl}$libdir'
+ hardcode_libdir_separator=:
+
+ case $host_cpu in
+ hppa*64*|ia64*)
+ hardcode_direct=no
+ hardcode_shlibpath_var=no
+ ;;
+ *)
+ hardcode_direct=yes
+ hardcode_direct_absolute=yes
+ export_dynamic_flag_spec='${wl}-E'
+
+ # hardcode_minus_L: Not really in the search PATH,
+ # but as the default location of the library.
+ hardcode_minus_L=yes
+ ;;
+ esac
+ fi
+ ;;
+
+ irix5* | irix6* | nonstopux*)
+ if test "$GCC" = yes; then
+ archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && $ECHO "X${wl}-set_version ${wl}$verstring" | $Xsed` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib'
+ # Try to use the -exported_symbol ld option, if it does not
+ # work, assume that -exports_file does not work either and
+ # implicitly export all symbols.
+ save_LDFLAGS="$LDFLAGS"
+ LDFLAGS="$LDFLAGS -shared ${wl}-exported_symbol ${wl}foo ${wl}-update_registry ${wl}/dev/null"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+int foo(void) {}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ archive_expsym_cmds='$CC -shared $libobjs $deplibs $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && $ECHO "X${wl}-set_version ${wl}$verstring" | $Xsed` ${wl}-update_registry ${wl}${output_objdir}/so_locations ${wl}-exports_file ${wl}$export_symbols -o $lib'
+
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+ LDFLAGS="$save_LDFLAGS"
+ else
+ archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && $ECHO "X-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -o $lib'
+ archive_expsym_cmds='$CC -shared $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && $ECHO "X-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -exports_file $export_symbols -o $lib'
+ fi
+ archive_cmds_need_lc='no'
+ hardcode_libdir_flag_spec='${wl}-rpath ${wl}$libdir'
+ hardcode_libdir_separator=:
+ inherit_rpath=yes
+ link_all_deplibs=yes
+ ;;
+
+ netbsd*)
+ if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then
+ archive_cmds='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags' # a.out
+ else
+ archive_cmds='$LD -shared -o $lib $libobjs $deplibs $linker_flags' # ELF
+ fi
+ hardcode_libdir_flag_spec='-R$libdir'
+ hardcode_direct=yes
+ hardcode_shlibpath_var=no
+ ;;
+
+ newsos6)
+ archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ hardcode_direct=yes
+ hardcode_libdir_flag_spec='${wl}-rpath ${wl}$libdir'
+ hardcode_libdir_separator=:
+ hardcode_shlibpath_var=no
+ ;;
+
+ *nto* | *qnx*)
+ ;;
+
+ openbsd*)
+ if test -f /usr/libexec/ld.so; then
+ hardcode_direct=yes
+ hardcode_shlibpath_var=no
+ hardcode_direct_absolute=yes
+ if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then
+ archive_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags'
+ archive_expsym_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags ${wl}-retain-symbols-file,$export_symbols'
+ hardcode_libdir_flag_spec='${wl}-rpath,$libdir'
+ export_dynamic_flag_spec='${wl}-E'
+ else
+ case $host_os in
+ openbsd[01].* | openbsd2.[0-7] | openbsd2.[0-7].*)
+ archive_cmds='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags'
+ hardcode_libdir_flag_spec='-R$libdir'
+ ;;
+ *)
+ archive_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags'
+ hardcode_libdir_flag_spec='${wl}-rpath,$libdir'
+ ;;
+ esac
+ fi
+ else
+ ld_shlibs=no
+ fi
+ ;;
+
+ os2*)
+ hardcode_libdir_flag_spec='-L$libdir'
+ hardcode_minus_L=yes
+ allow_undefined_flag=unsupported
+ archive_cmds='$ECHO "LIBRARY $libname INITINSTANCE" > $output_objdir/$libname.def~$ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~$ECHO DATA >> $output_objdir/$libname.def~$ECHO " SINGLE NONSHARED" >> $output_objdir/$libname.def~$ECHO EXPORTS >> $output_objdir/$libname.def~emxexp $libobjs >> $output_objdir/$libname.def~$CC -Zdll -Zcrtdll -o $lib $libobjs $deplibs $compiler_flags $output_objdir/$libname.def'
+ old_archive_from_new_cmds='emximp -o $output_objdir/$libname.a $output_objdir/$libname.def'
+ ;;
+
+ osf3*)
+ if test "$GCC" = yes; then
+ allow_undefined_flag=' ${wl}-expect_unresolved ${wl}\*'
+ archive_cmds='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags ${wl}-soname ${wl}$soname `test -n "$verstring" && $ECHO "X${wl}-set_version ${wl}$verstring" | $Xsed` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib'
+ else
+ allow_undefined_flag=' -expect_unresolved \*'
+ archive_cmds='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && $ECHO "X-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -o $lib'
+ fi
+ archive_cmds_need_lc='no'
+ hardcode_libdir_flag_spec='${wl}-rpath ${wl}$libdir'
+ hardcode_libdir_separator=:
+ ;;
+
+ osf4* | osf5*) # as osf3* with the addition of -msym flag
+ if test "$GCC" = yes; then
+ allow_undefined_flag=' ${wl}-expect_unresolved ${wl}\*'
+ archive_cmds='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags ${wl}-msym ${wl}-soname ${wl}$soname `test -n "$verstring" && $ECHO "X${wl}-set_version ${wl}$verstring" | $Xsed` ${wl}-update_registry ${wl}${output_objdir}/so_locations -o $lib'
+ hardcode_libdir_flag_spec='${wl}-rpath ${wl}$libdir'
+ else
+ allow_undefined_flag=' -expect_unresolved \*'
+ archive_cmds='$CC -shared${allow_undefined_flag} $libobjs $deplibs $compiler_flags -msym -soname $soname `test -n "$verstring" && $ECHO "X-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -o $lib'
+ archive_expsym_cmds='for i in `cat $export_symbols`; do printf "%s %s\\n" -exported_symbol "\$i" >> $lib.exp; done; printf "%s\\n" "-hidden">> $lib.exp~
+ $CC -shared${allow_undefined_flag} ${wl}-input ${wl}$lib.exp $compiler_flags $libobjs $deplibs -soname $soname `test -n "$verstring" && $ECHO "X-set_version $verstring" | $Xsed` -update_registry ${output_objdir}/so_locations -o $lib~$RM $lib.exp'
+
+ # Both c and cxx compiler support -rpath directly
+ hardcode_libdir_flag_spec='-rpath $libdir'
+ fi
+ archive_cmds_need_lc='no'
+ hardcode_libdir_separator=:
+ ;;
+
+ solaris*)
+ no_undefined_flag=' -z defs'
+ if test "$GCC" = yes; then
+ wlarc='${wl}'
+ archive_cmds='$CC -shared ${wl}-z ${wl}text ${wl}-h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags'
+ archive_expsym_cmds='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~
+ $CC -shared ${wl}-z ${wl}text ${wl}-M ${wl}$lib.exp ${wl}-h ${wl}$soname -o $lib $libobjs $deplibs $compiler_flags~$RM $lib.exp'
+ else
+ case `$CC -V 2>&1` in
+ *"Compilers 5.0"*)
+ wlarc=''
+ archive_cmds='$LD -G${allow_undefined_flag} -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ archive_expsym_cmds='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~
+ $LD -G${allow_undefined_flag} -M $lib.exp -h $soname -o $lib $libobjs $deplibs $linker_flags~$RM $lib.exp'
+ ;;
+ *)
+ wlarc='${wl}'
+ archive_cmds='$CC -G${allow_undefined_flag} -h $soname -o $lib $libobjs $deplibs $compiler_flags'
+ archive_expsym_cmds='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~
+ $CC -G${allow_undefined_flag} -M $lib.exp -h $soname -o $lib $libobjs $deplibs $compiler_flags~$RM $lib.exp'
+ ;;
+ esac
+ fi
+ hardcode_libdir_flag_spec='-R$libdir'
+ hardcode_shlibpath_var=no
+ case $host_os in
+ solaris2.[0-5] | solaris2.[0-5].*) ;;
+ *)
+ # The compiler driver will combine and reorder linker options,
+ # but understands `-z linker_flag'. GCC discards it without `$wl',
+ # but is careful enough not to reorder.
+ # Supported since Solaris 2.6 (maybe 2.5.1?)
+ if test "$GCC" = yes; then
+ whole_archive_flag_spec='${wl}-z ${wl}allextract$convenience ${wl}-z ${wl}defaultextract'
+ else
+ whole_archive_flag_spec='-z allextract$convenience -z defaultextract'
+ fi
+ ;;
+ esac
+ link_all_deplibs=yes
+ ;;
+
+ sunos4*)
+ if test "x$host_vendor" = xsequent; then
+ # Use $CC to link under sequent, because it throws in some extra .o
+ # files that make .init and .fini sections work.
+ archive_cmds='$CC -G ${wl}-h $soname -o $lib $libobjs $deplibs $compiler_flags'
+ else
+ archive_cmds='$LD -assert pure-text -Bstatic -o $lib $libobjs $deplibs $linker_flags'
+ fi
+ hardcode_libdir_flag_spec='-L$libdir'
+ hardcode_direct=yes
+ hardcode_minus_L=yes
+ hardcode_shlibpath_var=no
+ ;;
+
+ sysv4)
+ case $host_vendor in
+ sni)
+ archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ hardcode_direct=yes # is this really true???
+ ;;
+ siemens)
+ ## LD is ld it makes a PLAMLIB
+ ## CC just makes a GrossModule.
+ archive_cmds='$LD -G -o $lib $libobjs $deplibs $linker_flags'
+ reload_cmds='$CC -r -o $output$reload_objs'
+ hardcode_direct=no
+ ;;
+ motorola)
+ archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ hardcode_direct=no #Motorola manual says yes, but my tests say they lie
+ ;;
+ esac
+ runpath_var='LD_RUN_PATH'
+ hardcode_shlibpath_var=no
+ ;;
+
+ sysv4.3*)
+ archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ hardcode_shlibpath_var=no
+ export_dynamic_flag_spec='-Bexport'
+ ;;
+
+ sysv4*MP*)
+ if test -d /usr/nec; then
+ archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ hardcode_shlibpath_var=no
+ runpath_var=LD_RUN_PATH
+ hardcode_runpath_var=yes
+ ld_shlibs=yes
+ fi
+ ;;
+
+ sysv4*uw2* | sysv5OpenUNIX* | sysv5UnixWare7.[01].[10]* | unixware7* | sco3.2v5.0.[024]*)
+ no_undefined_flag='${wl}-z,text'
+ archive_cmds_need_lc=no
+ hardcode_shlibpath_var=no
+ runpath_var='LD_RUN_PATH'
+
+ if test "$GCC" = yes; then
+ archive_cmds='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ archive_expsym_cmds='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ else
+ archive_cmds='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ archive_expsym_cmds='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ fi
+ ;;
+
+ sysv5* | sco3.2v5* | sco5v6*)
+ # Note: We can NOT use -z defs as we might desire, because we do not
+ # link with -lc, and that would cause any symbols used from libc to
+ # always be unresolved, which means just about no library would
+ # ever link correctly. If we're not using GNU ld we use -z text
+ # though, which does catch some bad symbols but isn't as heavy-handed
+ # as -z defs.
+ no_undefined_flag='${wl}-z,text'
+ allow_undefined_flag='${wl}-z,nodefs'
+ archive_cmds_need_lc=no
+ hardcode_shlibpath_var=no
+ hardcode_libdir_flag_spec='${wl}-R,$libdir'
+ hardcode_libdir_separator=':'
+ link_all_deplibs=yes
+ export_dynamic_flag_spec='${wl}-Bexport'
+ runpath_var='LD_RUN_PATH'
+
+ if test "$GCC" = yes; then
+ archive_cmds='$CC -shared ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ archive_expsym_cmds='$CC -shared ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ else
+ archive_cmds='$CC -G ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ archive_expsym_cmds='$CC -G ${wl}-Bexport:$export_symbols ${wl}-h,$soname -o $lib $libobjs $deplibs $compiler_flags'
+ fi
+ ;;
+
+ uts4*)
+ archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags'
+ hardcode_libdir_flag_spec='-L$libdir'
+ hardcode_shlibpath_var=no
+ ;;
+
+ *)
+ ld_shlibs=no
+ ;;
+ esac
+
+ if test x$host_vendor = xsni; then
+ case $host in
+ sysv4 | sysv4.2uw2* | sysv4.3* | sysv5*)
+ export_dynamic_flag_spec='${wl}-Blargedynsym'
+ ;;
+ esac
+ fi
+ fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ld_shlibs" >&5
+$as_echo "$ld_shlibs" >&6; }
+test "$ld_shlibs" = no && can_build_shared=no
+
+with_gnu_ld=$with_gnu_ld
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+#
+# Do we need to explicitly link libc?
+#
+case "x$archive_cmds_need_lc" in
+x|xyes)
+ # Assume -lc should be added
+ archive_cmds_need_lc=yes
+
+ if test "$enable_shared" = yes && test "$GCC" = yes; then
+ case $archive_cmds in
+ *'~'*)
+ # FIXME: we may have to deal with multi-command sequences.
+ ;;
+ '$CC '*)
+ # Test whether the compiler implicitly links with -lc since on some
+ # systems, -lgcc has to come before -lc. If gcc already passes -lc
+ # to ld, don't add -lc before -lgcc.
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether -lc should be explicitly linked in" >&5
+$as_echo_n "checking whether -lc should be explicitly linked in... " >&6; }
+ $RM conftest*
+ echo "$lt_simple_compile_test_code" > conftest.$ac_ext
+
+ if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
+ (eval $ac_compile) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; } 2>conftest.err; then
+ soname=conftest
+ lib=conftest
+ libobjs=conftest.$ac_objext
+ deplibs=
+ wl=$lt_prog_compiler_wl
+ pic_flag=$lt_prog_compiler_pic
+ compiler_flags=-v
+ linker_flags=-v
+ verstring=
+ output_objdir=.
+ libname=conftest
+ lt_save_allow_undefined_flag=$allow_undefined_flag
+ allow_undefined_flag=
+ if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$archive_cmds 2\>\&1 \| $GREP \" -lc \" \>/dev/null 2\>\&1\""; } >&5
+ (eval $archive_cmds 2\>\&1 \| $GREP \" -lc \" \>/dev/null 2\>\&1) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }
+ then
+ archive_cmds_need_lc=no
+ else
+ archive_cmds_need_lc=yes
+ fi
+ allow_undefined_flag=$lt_save_allow_undefined_flag
+ else
+ cat conftest.err 1>&5
+ fi
+ $RM conftest*
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $archive_cmds_need_lc" >&5
+$as_echo "$archive_cmds_need_lc" >&6; }
+ ;;
+ esac
+ fi
+ ;;
+esac
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking dynamic linker characteristics" >&5
+$as_echo_n "checking dynamic linker characteristics... " >&6; }
+
+if test "$GCC" = yes; then
+ case $host_os in
+ darwin*) lt_awk_arg="/^libraries:/,/LR/" ;;
+ *) lt_awk_arg="/^libraries:/" ;;
+ esac
+ lt_search_path_spec=`$CC -print-search-dirs | awk $lt_awk_arg | $SED -e "s/^libraries://" -e "s,=/,/,g"`
+ if $ECHO "$lt_search_path_spec" | $GREP ';' >/dev/null ; then
+ # if the path contains ";" then we assume it to be the separator
+ # otherwise default to the standard path separator (i.e. ":") - it is
+ # assumed that no part of a normal pathname contains ";" but that should
+ # okay in the real world where ";" in dirpaths is itself problematic.
+ lt_search_path_spec=`$ECHO "$lt_search_path_spec" | $SED -e 's/;/ /g'`
+ else
+ lt_search_path_spec=`$ECHO "$lt_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"`
+ fi
+ # Ok, now we have the path, separated by spaces, we can step through it
+ # and add multilib dir if necessary.
+ lt_tmp_lt_search_path_spec=
+ lt_multi_os_dir=`$CC $CPPFLAGS $CFLAGS $LDFLAGS -print-multi-os-directory 2>/dev/null`
+ for lt_sys_path in $lt_search_path_spec; do
+ if test -d "$lt_sys_path/$lt_multi_os_dir"; then
+ lt_tmp_lt_search_path_spec="$lt_tmp_lt_search_path_spec $lt_sys_path/$lt_multi_os_dir"
+ else
+ test -d "$lt_sys_path" && \
+ lt_tmp_lt_search_path_spec="$lt_tmp_lt_search_path_spec $lt_sys_path"
+ fi
+ done
+ lt_search_path_spec=`$ECHO $lt_tmp_lt_search_path_spec | awk '
+BEGIN {RS=" "; FS="/|\n";} {
+ lt_foo="";
+ lt_count=0;
+ for (lt_i = NF; lt_i > 0; lt_i--) {
+ if ($lt_i != "" && $lt_i != ".") {
+ if ($lt_i == "..") {
+ lt_count++;
+ } else {
+ if (lt_count == 0) {
+ lt_foo="/" $lt_i lt_foo;
+ } else {
+ lt_count--;
+ }
+ }
+ }
+ }
+ if (lt_foo != "") { lt_freq[lt_foo]++; }
+ if (lt_freq[lt_foo] == 1) { print lt_foo; }
+}'`
+ sys_lib_search_path_spec=`$ECHO $lt_search_path_spec`
+else
+ sys_lib_search_path_spec="/lib /usr/lib /usr/local/lib"
+fi
+library_names_spec=
+libname_spec='lib$name'
+soname_spec=
+shrext_cmds=".so"
+postinstall_cmds=
+postuninstall_cmds=
+finish_cmds=
+finish_eval=
+shlibpath_var=
+shlibpath_overrides_runpath=unknown
+version_type=none
+dynamic_linker="$host_os ld.so"
+sys_lib_dlsearch_path_spec="/lib /usr/lib"
+need_lib_prefix=unknown
+hardcode_into_libs=no
+
+# when you set need_version to no, make sure it does not cause -set_version
+# flags to be left without arguments
+need_version=unknown
+
+case $host_os in
+aix3*)
+ version_type=linux
+ library_names_spec='${libname}${release}${shared_ext}$versuffix $libname.a'
+ shlibpath_var=LIBPATH
+
+ # AIX 3 has no versioning support, so we append a major version to the name.
+ soname_spec='${libname}${release}${shared_ext}$major'
+ ;;
+
+aix[4-9]*)
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ hardcode_into_libs=yes
+ if test "$host_cpu" = ia64; then
+ # AIX 5 supports IA64
+ library_names_spec='${libname}${release}${shared_ext}$major ${libname}${release}${shared_ext}$versuffix $libname${shared_ext}'
+ shlibpath_var=LD_LIBRARY_PATH
+ else
+ # With GCC up to 2.95.x, collect2 would create an import file
+ # for dependence libraries. The import file would start with
+ # the line `#! .'. This would cause the generated library to
+ # depend on `.', always an invalid library. This was fixed in
+ # development snapshots of GCC prior to 3.0.
+ case $host_os in
+ aix4 | aix4.[01] | aix4.[01].*)
+ if { echo '#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 97)'
+ echo ' yes '
+ echo '#endif'; } | ${CC} -E - | $GREP yes > /dev/null; then
+ :
+ else
+ can_build_shared=no
+ fi
+ ;;
+ esac
+ # AIX (on Power*) has no versioning support, so currently we can not hardcode correct
+ # soname into executable. Probably we can add versioning support to
+ # collect2, so additional links can be useful in future.
+ if test "$aix_use_runtimelinking" = yes; then
+ # If using run time linking (on AIX 4.2 or later) use lib<name>.so
+ # instead of lib<name>.a to let people know that these are not
+ # typical AIX shared libraries.
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ else
+ # We preserve .a as extension for shared libraries through AIX4.2
+ # and later when we are not doing run time linking.
+ library_names_spec='${libname}${release}.a $libname.a'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ fi
+ shlibpath_var=LIBPATH
+ fi
+ ;;
+
+amigaos*)
+ case $host_cpu in
+ powerpc)
+ # Since July 2007 AmigaOS4 officially supports .so libraries.
+ # When compiling the executable, add -use-dynld -Lsobjs: to the compileline.
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ ;;
+ m68k)
+ library_names_spec='$libname.ixlibrary $libname.a'
+ # Create ${libname}_ixlibrary.a entries in /sys/libs.
+ finish_eval='for lib in `ls $libdir/*.ixlibrary 2>/dev/null`; do libname=`$ECHO "X$lib" | $Xsed -e '\''s%^.*/\([^/]*\)\.ixlibrary$%\1%'\''`; test $RM /sys/libs/${libname}_ixlibrary.a; $show "cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a"; cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a || exit 1; done'
+ ;;
+ esac
+ ;;
+
+beos*)
+ library_names_spec='${libname}${shared_ext}'
+ dynamic_linker="$host_os ld.so"
+ shlibpath_var=LIBRARY_PATH
+ ;;
+
+bsdi[45]*)
+ version_type=linux
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ finish_cmds='PATH="\$PATH:/sbin" ldconfig $libdir'
+ shlibpath_var=LD_LIBRARY_PATH
+ sys_lib_search_path_spec="/shlib /usr/lib /usr/X11/lib /usr/contrib/lib /lib /usr/local/lib"
+ sys_lib_dlsearch_path_spec="/shlib /usr/lib /usr/local/lib"
+ # the default ld.so.conf also contains /usr/contrib/lib and
+ # /usr/X11R6/lib (/usr/X11 is a link to /usr/X11R6), but let us allow
+ # libtool to hard-code these into programs
+ ;;
+
+cygwin* | mingw* | pw32* | cegcc*)
+ version_type=windows
+ shrext_cmds=".dll"
+ need_version=no
+ need_lib_prefix=no
+
+ case $GCC,$host_os in
+ yes,cygwin* | yes,mingw* | yes,pw32* | yes,cegcc*)
+ library_names_spec='$libname.dll.a'
+ # DLL is installed to $(libdir)/../bin by postinstall_cmds
+ postinstall_cmds='base_file=`basename \${file}`~
+ dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\${base_file}'\''i; echo \$dlname'\''`~
+ dldir=$destdir/`dirname \$dlpath`~
+ test -d \$dldir || mkdir -p \$dldir~
+ $install_prog $dir/$dlname \$dldir/$dlname~
+ chmod a+x \$dldir/$dlname~
+ if test -n '\''$stripme'\'' && test -n '\''$striplib'\''; then
+ eval '\''$striplib \$dldir/$dlname'\'' || exit \$?;
+ fi'
+ postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; echo \$dlname'\''`~
+ dlpath=$dir/\$dldll~
+ $RM \$dlpath'
+ shlibpath_overrides_runpath=yes
+
+ case $host_os in
+ cygwin*)
+ # Cygwin DLLs use 'cyg' prefix rather than 'lib'
+ soname_spec='`echo ${libname} | sed -e 's/^lib/cyg/'``echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext}'
+ sys_lib_search_path_spec="/usr/lib /lib/w32api /lib /usr/local/lib"
+ ;;
+ mingw* | cegcc*)
+ # MinGW DLLs use traditional 'lib' prefix
+ soname_spec='${libname}`echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext}'
+ sys_lib_search_path_spec=`$CC -print-search-dirs | $GREP "^libraries:" | $SED -e "s/^libraries://" -e "s,=/,/,g"`
+ if $ECHO "$sys_lib_search_path_spec" | $GREP ';[c-zC-Z]:/' >/dev/null; then
+ # It is most probably a Windows format PATH printed by
+ # mingw gcc, but we are running on Cygwin. Gcc prints its search
+ # path with ; separators, and with drive letters. We can handle the
+ # drive letters (cygwin fileutils understands them), so leave them,
+ # especially as we might pass files found there to a mingw objdump,
+ # which wouldn't understand a cygwinified path. Ahh.
+ sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e 's/;/ /g'`
+ else
+ sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"`
+ fi
+ ;;
+ pw32*)
+ # pw32 DLLs use 'pw' prefix rather than 'lib'
+ library_names_spec='`echo ${libname} | sed -e 's/^lib/pw/'``echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext}'
+ ;;
+ esac
+ ;;
+
+ *)
+ library_names_spec='${libname}`echo ${release} | $SED -e 's/[.]/-/g'`${versuffix}${shared_ext} $libname.lib'
+ ;;
+ esac
+ dynamic_linker='Win32 ld.exe'
+ # FIXME: first we should search . and the directory the executable is in
+ shlibpath_var=PATH
+ ;;
+
+darwin* | rhapsody*)
+ dynamic_linker="$host_os dyld"
+ version_type=darwin
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${major}$shared_ext ${libname}$shared_ext'
+ soname_spec='${libname}${release}${major}$shared_ext'
+ shlibpath_overrides_runpath=yes
+ shlibpath_var=DYLD_LIBRARY_PATH
+ shrext_cmds='`test .$module = .yes && echo .so || echo .dylib`'
+
+ sys_lib_search_path_spec="$sys_lib_search_path_spec /usr/local/lib"
+ sys_lib_dlsearch_path_spec='/usr/local/lib /lib /usr/lib'
+ ;;
+
+dgux*)
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname$shared_ext'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ ;;
+
+freebsd1*)
+ dynamic_linker=no
+ ;;
+
+freebsd* | dragonfly*)
+ # DragonFly does not have aout. When/if they implement a new
+ # versioning mechanism, adjust this.
+ if test -x /usr/bin/objformat; then
+ objformat=`/usr/bin/objformat`
+ else
+ case $host_os in
+ freebsd[123]*) objformat=aout ;;
+ *) objformat=elf ;;
+ esac
+ fi
+ version_type=freebsd-$objformat
+ case $version_type in
+ freebsd-elf*)
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext} $libname${shared_ext}'
+ need_version=no
+ need_lib_prefix=no
+ ;;
+ freebsd-*)
+ library_names_spec='${libname}${release}${shared_ext}$versuffix $libname${shared_ext}$versuffix'
+ need_version=yes
+ ;;
+ esac
+ shlibpath_var=LD_LIBRARY_PATH
+ case $host_os in
+ freebsd2*)
+ shlibpath_overrides_runpath=yes
+ ;;
+ freebsd3.[01]* | freebsdelf3.[01]*)
+ shlibpath_overrides_runpath=yes
+ hardcode_into_libs=yes
+ ;;
+ freebsd3.[2-9]* | freebsdelf3.[2-9]* | \
+ freebsd4.[0-5] | freebsdelf4.[0-5] | freebsd4.1.1 | freebsdelf4.1.1)
+ shlibpath_overrides_runpath=no
+ hardcode_into_libs=yes
+ ;;
+ *) # from 4.6 on, and DragonFly
+ shlibpath_overrides_runpath=yes
+ hardcode_into_libs=yes
+ ;;
+ esac
+ ;;
+
+gnu*)
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}${major} ${libname}${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ hardcode_into_libs=yes
+ ;;
+
+hpux9* | hpux10* | hpux11*)
+ # Give a soname corresponding to the major version so that dld.sl refuses to
+ # link against other versions.
+ version_type=sunos
+ need_lib_prefix=no
+ need_version=no
+ case $host_cpu in
+ ia64*)
+ shrext_cmds='.so'
+ hardcode_into_libs=yes
+ dynamic_linker="$host_os dld.so"
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=yes # Unless +noenvvar is specified.
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ if test "X$HPUX_IA64_MODE" = X32; then
+ sys_lib_search_path_spec="/usr/lib/hpux32 /usr/local/lib/hpux32 /usr/local/lib"
+ else
+ sys_lib_search_path_spec="/usr/lib/hpux64 /usr/local/lib/hpux64"
+ fi
+ sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec
+ ;;
+ hppa*64*)
+ shrext_cmds='.sl'
+ hardcode_into_libs=yes
+ dynamic_linker="$host_os dld.sl"
+ shlibpath_var=LD_LIBRARY_PATH # How should we handle SHLIB_PATH
+ shlibpath_overrides_runpath=yes # Unless +noenvvar is specified.
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ sys_lib_search_path_spec="/usr/lib/pa20_64 /usr/ccs/lib/pa20_64"
+ sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec
+ ;;
+ *)
+ shrext_cmds='.sl'
+ dynamic_linker="$host_os dld.sl"
+ shlibpath_var=SHLIB_PATH
+ shlibpath_overrides_runpath=no # +s is required to enable SHLIB_PATH
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ ;;
+ esac
+ # HP-UX runs *really* slowly unless shared libraries are mode 555.
+ postinstall_cmds='chmod 555 $lib'
+ ;;
+
+interix[3-9]*)
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ dynamic_linker='Interix 3.x ld.so.1 (PE, like ELF)'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=no
+ hardcode_into_libs=yes
+ ;;
+
+irix5* | irix6* | nonstopux*)
+ case $host_os in
+ nonstopux*) version_type=nonstopux ;;
+ *)
+ if test "$lt_cv_prog_gnu_ld" = yes; then
+ version_type=linux
+ else
+ version_type=irix
+ fi ;;
+ esac
+ need_lib_prefix=no
+ need_version=no
+ soname_spec='${libname}${release}${shared_ext}$major'
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${release}${shared_ext} $libname${shared_ext}'
+ case $host_os in
+ irix5* | nonstopux*)
+ libsuff= shlibsuff=
+ ;;
+ *)
+ case $LD in # libtool.m4 will add one of these switches to LD
+ *-32|*"-32 "|*-melf32bsmip|*"-melf32bsmip ")
+ libsuff= shlibsuff= libmagic=32-bit;;
+ *-n32|*"-n32 "|*-melf32bmipn32|*"-melf32bmipn32 ")
+ libsuff=32 shlibsuff=N32 libmagic=N32;;
+ *-64|*"-64 "|*-melf64bmip|*"-melf64bmip ")
+ libsuff=64 shlibsuff=64 libmagic=64-bit;;
+ *) libsuff= shlibsuff= libmagic=never-match;;
+ esac
+ ;;
+ esac
+ shlibpath_var=LD_LIBRARY${shlibsuff}_PATH
+ shlibpath_overrides_runpath=no
+ sys_lib_search_path_spec="/usr/lib${libsuff} /lib${libsuff} /usr/local/lib${libsuff}"
+ sys_lib_dlsearch_path_spec="/usr/lib${libsuff} /lib${libsuff}"
+ hardcode_into_libs=yes
+ ;;
+
+# No shared lib support for Linux oldld, aout, or coff.
+linux*oldld* | linux*aout* | linux*coff*)
+ dynamic_linker=no
+ ;;
+
+# This must be Linux ELF.
+linux* | k*bsd*-gnu)
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ finish_cmds='PATH="\$PATH:/sbin" ldconfig -n $libdir'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=no
+ # Some binutils ld are patched to set DT_RUNPATH
+ save_LDFLAGS=$LDFLAGS
+ save_libdir=$libdir
+ eval "libdir=/foo; wl=\"$lt_prog_compiler_wl\"; \
+ LDFLAGS=\"\$LDFLAGS $hardcode_libdir_flag_spec\""
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ if ($OBJDUMP -p conftest$ac_exeext) 2>/dev/null | grep "RUNPATH.*$libdir" >/dev/null; then :
+ shlibpath_overrides_runpath=yes
+fi
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+ LDFLAGS=$save_LDFLAGS
+ libdir=$save_libdir
+
+ # This implies no fast_install, which is unacceptable.
+ # Some rework will be needed to allow for fast_install
+ # before this can be enabled.
+ hardcode_into_libs=yes
+
+ # Add ABI-specific directories to the system library path.
+ sys_lib_dlsearch_path_spec="/lib64 /usr/lib64 /lib /usr/lib"
+
+ # Append ld.so.conf contents to the search path
+ if test -f /etc/ld.so.conf; then
+ lt_ld_extra=`awk '/^include / { system(sprintf("cd /etc; cat %s 2>/dev/null", \$2)); skip = 1; } { if (!skip) print \$0; skip = 0; }' < /etc/ld.so.conf | $SED -e 's/#.*//;/^[ ]*hwcap[ ]/d;s/[:, ]/ /g;s/=[^=]*$//;s/=[^= ]* / /g;/^$/d' | tr '\n' ' '`
+ sys_lib_dlsearch_path_spec="$sys_lib_dlsearch_path_spec $lt_ld_extra"
+ fi
+
+ # We used to test for /lib/ld.so.1 and disable shared libraries on
+ # powerpc, because MkLinux only supported shared libraries with the
+ # GNU dynamic linker. Since this was broken with cross compilers,
+ # most powerpc-linux boxes support dynamic linking these days and
+ # people can always --disable-shared, the test was removed, and we
+ # assume the GNU/Linux dynamic linker is in use.
+ dynamic_linker='GNU/Linux ld.so'
+ ;;
+
+netbsd*)
+ version_type=sunos
+ need_lib_prefix=no
+ need_version=no
+ if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix'
+ finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir'
+ dynamic_linker='NetBSD (a.out) ld.so'
+ else
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ dynamic_linker='NetBSD ld.elf_so'
+ fi
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=yes
+ hardcode_into_libs=yes
+ ;;
+
+newsos6)
+ version_type=linux
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=yes
+ ;;
+
+*nto* | *qnx*)
+ version_type=qnx
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=no
+ hardcode_into_libs=yes
+ dynamic_linker='ldqnx.so'
+ ;;
+
+openbsd*)
+ version_type=sunos
+ sys_lib_dlsearch_path_spec="/usr/lib"
+ need_lib_prefix=no
+ # Some older versions of OpenBSD (3.3 at least) *do* need versioned libs.
+ case $host_os in
+ openbsd3.3 | openbsd3.3.*) need_version=yes ;;
+ *) need_version=no ;;
+ esac
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix'
+ finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir'
+ shlibpath_var=LD_LIBRARY_PATH
+ if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`" || test "$host_os-$host_cpu" = "openbsd2.8-powerpc"; then
+ case $host_os in
+ openbsd2.[89] | openbsd2.[89].*)
+ shlibpath_overrides_runpath=no
+ ;;
+ *)
+ shlibpath_overrides_runpath=yes
+ ;;
+ esac
+ else
+ shlibpath_overrides_runpath=yes
+ fi
+ ;;
+
+os2*)
+ libname_spec='$name'
+ shrext_cmds=".dll"
+ need_lib_prefix=no
+ library_names_spec='$libname${shared_ext} $libname.a'
+ dynamic_linker='OS/2 ld.exe'
+ shlibpath_var=LIBPATH
+ ;;
+
+osf3* | osf4* | osf5*)
+ version_type=osf
+ need_lib_prefix=no
+ need_version=no
+ soname_spec='${libname}${release}${shared_ext}$major'
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ shlibpath_var=LD_LIBRARY_PATH
+ sys_lib_search_path_spec="/usr/shlib /usr/ccs/lib /usr/lib/cmplrs/cc /usr/lib /usr/local/lib /var/shlib"
+ sys_lib_dlsearch_path_spec="$sys_lib_search_path_spec"
+ ;;
+
+rdos*)
+ dynamic_linker=no
+ ;;
+
+solaris*)
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=yes
+ hardcode_into_libs=yes
+ # ldd complains unless libraries are executable
+ postinstall_cmds='chmod +x $lib'
+ ;;
+
+sunos4*)
+ version_type=sunos
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${shared_ext}$versuffix'
+ finish_cmds='PATH="\$PATH:/usr/etc" ldconfig $libdir'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=yes
+ if test "$with_gnu_ld" = yes; then
+ need_lib_prefix=no
+ fi
+ need_version=yes
+ ;;
+
+sysv4 | sysv4.3*)
+ version_type=linux
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ case $host_vendor in
+ sni)
+ shlibpath_overrides_runpath=no
+ need_lib_prefix=no
+ runpath_var=LD_RUN_PATH
+ ;;
+ siemens)
+ need_lib_prefix=no
+ ;;
+ motorola)
+ need_lib_prefix=no
+ need_version=no
+ shlibpath_overrides_runpath=no
+ sys_lib_search_path_spec='/lib /usr/lib /usr/ccs/lib'
+ ;;
+ esac
+ ;;
+
+sysv4*MP*)
+ if test -d /usr/nec ;then
+ version_type=linux
+ library_names_spec='$libname${shared_ext}.$versuffix $libname${shared_ext}.$major $libname${shared_ext}'
+ soname_spec='$libname${shared_ext}.$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ fi
+ ;;
+
+sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX* | sysv4*uw2*)
+ version_type=freebsd-elf
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext} $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=yes
+ hardcode_into_libs=yes
+ if test "$with_gnu_ld" = yes; then
+ sys_lib_search_path_spec='/usr/local/lib /usr/gnu/lib /usr/ccs/lib /usr/lib /lib'
+ else
+ sys_lib_search_path_spec='/usr/ccs/lib /usr/lib'
+ case $host_os in
+ sco3.2v5*)
+ sys_lib_search_path_spec="$sys_lib_search_path_spec /lib"
+ ;;
+ esac
+ fi
+ sys_lib_dlsearch_path_spec='/usr/lib'
+ ;;
+
+tpf*)
+ # TPF is a cross-target only. Preferred cross-host = GNU/Linux.
+ version_type=linux
+ need_lib_prefix=no
+ need_version=no
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ shlibpath_var=LD_LIBRARY_PATH
+ shlibpath_overrides_runpath=no
+ hardcode_into_libs=yes
+ ;;
+
+uts4*)
+ version_type=linux
+ library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major $libname${shared_ext}'
+ soname_spec='${libname}${release}${shared_ext}$major'
+ shlibpath_var=LD_LIBRARY_PATH
+ ;;
+
+*)
+ dynamic_linker=no
+ ;;
+esac
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $dynamic_linker" >&5
+$as_echo "$dynamic_linker" >&6; }
+test "$dynamic_linker" = no && can_build_shared=no
+
+variables_saved_for_relink="PATH $shlibpath_var $runpath_var"
+if test "$GCC" = yes; then
+ variables_saved_for_relink="$variables_saved_for_relink GCC_EXEC_PREFIX COMPILER_PATH LIBRARY_PATH"
+fi
+
+if test "${lt_cv_sys_lib_search_path_spec+set}" = set; then
+ sys_lib_search_path_spec="$lt_cv_sys_lib_search_path_spec"
+fi
+if test "${lt_cv_sys_lib_dlsearch_path_spec+set}" = set; then
+ sys_lib_dlsearch_path_spec="$lt_cv_sys_lib_dlsearch_path_spec"
+fi
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking how to hardcode library paths into programs" >&5
+$as_echo_n "checking how to hardcode library paths into programs... " >&6; }
+hardcode_action=
+if test -n "$hardcode_libdir_flag_spec" ||
+ test -n "$runpath_var" ||
+ test "X$hardcode_automatic" = "Xyes" ; then
+
+ # We can hardcode non-existent directories.
+ if test "$hardcode_direct" != no &&
+ # If the only mechanism to avoid hardcoding is shlibpath_var, we
+ # have to relink, otherwise we might link with an installed library
+ # when we should be linking with a yet-to-be-installed one
+ ## test "$_LT_TAGVAR(hardcode_shlibpath_var, )" != no &&
+ test "$hardcode_minus_L" != no; then
+ # Linking always hardcodes the temporary library directory.
+ hardcode_action=relink
+ else
+ # We can link without hardcoding, and we can hardcode nonexisting dirs.
+ hardcode_action=immediate
+ fi
+else
+ # We cannot hardcode anything, or else we can only hardcode existing
+ # directories.
+ hardcode_action=unsupported
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $hardcode_action" >&5
+$as_echo "$hardcode_action" >&6; }
+
+if test "$hardcode_action" = relink ||
+ test "$inherit_rpath" = yes; then
+ # Fast installation is not supported
+ enable_fast_install=no
+elif test "$shlibpath_overrides_runpath" = yes ||
+ test "$enable_shared" = no; then
+ # Fast installation is not necessary
+ enable_fast_install=needless
+fi
+
+
+
+
+
+
+ if test "x$enable_dlopen" != xyes; then
+ enable_dlopen=unknown
+ enable_dlopen_self=unknown
+ enable_dlopen_self_static=unknown
+else
+ lt_cv_dlopen=no
+ lt_cv_dlopen_libs=
+
+ case $host_os in
+ beos*)
+ lt_cv_dlopen="load_add_on"
+ lt_cv_dlopen_libs=
+ lt_cv_dlopen_self=yes
+ ;;
+
+ mingw* | pw32* | cegcc*)
+ lt_cv_dlopen="LoadLibrary"
+ lt_cv_dlopen_libs=
+ ;;
+
+ cygwin*)
+ lt_cv_dlopen="dlopen"
+ lt_cv_dlopen_libs=
+ ;;
+
+ darwin*)
+ # if libdl is installed we need to link against it
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -ldl" >&5
+$as_echo_n "checking for dlopen in -ldl... " >&6; }
+if test "${ac_cv_lib_dl_dlopen+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-ldl $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char dlopen ();
+int
+main ()
+{
+return dlopen ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_dl_dlopen=yes
+else
+ ac_cv_lib_dl_dlopen=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlopen" >&5
+$as_echo "$ac_cv_lib_dl_dlopen" >&6; }
+if test "x$ac_cv_lib_dl_dlopen" = x""yes; then :
+ lt_cv_dlopen="dlopen" lt_cv_dlopen_libs="-ldl"
+else
+
+ lt_cv_dlopen="dyld"
+ lt_cv_dlopen_libs=
+ lt_cv_dlopen_self=yes
+
+fi
+
+ ;;
+
+ *)
+ ac_fn_c_check_func "$LINENO" "shl_load" "ac_cv_func_shl_load"
+if test "x$ac_cv_func_shl_load" = x""yes; then :
+ lt_cv_dlopen="shl_load"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shl_load in -ldld" >&5
+$as_echo_n "checking for shl_load in -ldld... " >&6; }
+if test "${ac_cv_lib_dld_shl_load+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-ldld $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shl_load ();
+int
+main ()
+{
+return shl_load ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_dld_shl_load=yes
+else
+ ac_cv_lib_dld_shl_load=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dld_shl_load" >&5
+$as_echo "$ac_cv_lib_dld_shl_load" >&6; }
+if test "x$ac_cv_lib_dld_shl_load" = x""yes; then :
+ lt_cv_dlopen="shl_load" lt_cv_dlopen_libs="-ldld"
+else
+ ac_fn_c_check_func "$LINENO" "dlopen" "ac_cv_func_dlopen"
+if test "x$ac_cv_func_dlopen" = x""yes; then :
+ lt_cv_dlopen="dlopen"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -ldl" >&5
+$as_echo_n "checking for dlopen in -ldl... " >&6; }
+if test "${ac_cv_lib_dl_dlopen+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-ldl $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char dlopen ();
+int
+main ()
+{
+return dlopen ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_dl_dlopen=yes
+else
+ ac_cv_lib_dl_dlopen=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlopen" >&5
+$as_echo "$ac_cv_lib_dl_dlopen" >&6; }
+if test "x$ac_cv_lib_dl_dlopen" = x""yes; then :
+ lt_cv_dlopen="dlopen" lt_cv_dlopen_libs="-ldl"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -lsvld" >&5
+$as_echo_n "checking for dlopen in -lsvld... " >&6; }
+if test "${ac_cv_lib_svld_dlopen+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lsvld $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char dlopen ();
+int
+main ()
+{
+return dlopen ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_svld_dlopen=yes
+else
+ ac_cv_lib_svld_dlopen=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_svld_dlopen" >&5
+$as_echo "$ac_cv_lib_svld_dlopen" >&6; }
+if test "x$ac_cv_lib_svld_dlopen" = x""yes; then :
+ lt_cv_dlopen="dlopen" lt_cv_dlopen_libs="-lsvld"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dld_link in -ldld" >&5
+$as_echo_n "checking for dld_link in -ldld... " >&6; }
+if test "${ac_cv_lib_dld_dld_link+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-ldld $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char dld_link ();
+int
+main ()
+{
+return dld_link ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_dld_dld_link=yes
+else
+ ac_cv_lib_dld_dld_link=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dld_dld_link" >&5
+$as_echo "$ac_cv_lib_dld_dld_link" >&6; }
+if test "x$ac_cv_lib_dld_dld_link" = x""yes; then :
+ lt_cv_dlopen="dld_link" lt_cv_dlopen_libs="-ldld"
+fi
+
+
+fi
+
+
+fi
+
+
+fi
+
+
+fi
+
+
+fi
+
+ ;;
+ esac
+
+ if test "x$lt_cv_dlopen" != xno; then
+ enable_dlopen=yes
+ else
+ enable_dlopen=no
+ fi
+
+ case $lt_cv_dlopen in
+ dlopen)
+ save_CPPFLAGS="$CPPFLAGS"
+ test "x$ac_cv_header_dlfcn_h" = xyes && CPPFLAGS="$CPPFLAGS -DHAVE_DLFCN_H"
+
+ save_LDFLAGS="$LDFLAGS"
+ wl=$lt_prog_compiler_wl eval LDFLAGS=\"\$LDFLAGS $export_dynamic_flag_spec\"
+
+ save_LIBS="$LIBS"
+ LIBS="$lt_cv_dlopen_libs $LIBS"
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether a program can dlopen itself" >&5
+$as_echo_n "checking whether a program can dlopen itself... " >&6; }
+if test "${lt_cv_dlopen_self+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test "$cross_compiling" = yes; then :
+ lt_cv_dlopen_self=cross
+else
+ lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
+ lt_status=$lt_dlunknown
+ cat > conftest.$ac_ext <<_LT_EOF
+#line 11016 "configure"
+#include "confdefs.h"
+
+#if HAVE_DLFCN_H
+#include <dlfcn.h>
+#endif
+
+#include <stdio.h>
+
+#ifdef RTLD_GLOBAL
+# define LT_DLGLOBAL RTLD_GLOBAL
+#else
+# ifdef DL_GLOBAL
+# define LT_DLGLOBAL DL_GLOBAL
+# else
+# define LT_DLGLOBAL 0
+# endif
+#endif
+
+/* We may have to define LT_DLLAZY_OR_NOW in the command line if we
+ find out it does not work in some platform. */
+#ifndef LT_DLLAZY_OR_NOW
+# ifdef RTLD_LAZY
+# define LT_DLLAZY_OR_NOW RTLD_LAZY
+# else
+# ifdef DL_LAZY
+# define LT_DLLAZY_OR_NOW DL_LAZY
+# else
+# ifdef RTLD_NOW
+# define LT_DLLAZY_OR_NOW RTLD_NOW
+# else
+# ifdef DL_NOW
+# define LT_DLLAZY_OR_NOW DL_NOW
+# else
+# define LT_DLLAZY_OR_NOW 0
+# endif
+# endif
+# endif
+# endif
+#endif
+
+void fnord() { int i=42;}
+int main ()
+{
+ void *self = dlopen (0, LT_DLGLOBAL|LT_DLLAZY_OR_NOW);
+ int status = $lt_dlunknown;
+
+ if (self)
+ {
+ if (dlsym (self,"fnord")) status = $lt_dlno_uscore;
+ else if (dlsym( self,"_fnord")) status = $lt_dlneed_uscore;
+ /* dlclose (self); */
+ }
+ else
+ puts (dlerror ());
+
+ return status;
+}
+_LT_EOF
+ if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_link\""; } >&5
+ (eval $ac_link) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; } && test -s conftest${ac_exeext} 2>/dev/null; then
+ (./conftest; exit; ) >&5 2>/dev/null
+ lt_status=$?
+ case x$lt_status in
+ x$lt_dlno_uscore) lt_cv_dlopen_self=yes ;;
+ x$lt_dlneed_uscore) lt_cv_dlopen_self=yes ;;
+ x$lt_dlunknown|x*) lt_cv_dlopen_self=no ;;
+ esac
+ else :
+ # compilation failed
+ lt_cv_dlopen_self=no
+ fi
+fi
+rm -fr conftest*
+
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_dlopen_self" >&5
+$as_echo "$lt_cv_dlopen_self" >&6; }
+
+ if test "x$lt_cv_dlopen_self" = xyes; then
+ wl=$lt_prog_compiler_wl eval LDFLAGS=\"\$LDFLAGS $lt_prog_compiler_static\"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether a statically linked program can dlopen itself" >&5
+$as_echo_n "checking whether a statically linked program can dlopen itself... " >&6; }
+if test "${lt_cv_dlopen_self_static+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test "$cross_compiling" = yes; then :
+ lt_cv_dlopen_self_static=cross
+else
+ lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
+ lt_status=$lt_dlunknown
+ cat > conftest.$ac_ext <<_LT_EOF
+#line 11112 "configure"
+#include "confdefs.h"
+
+#if HAVE_DLFCN_H
+#include <dlfcn.h>
+#endif
+
+#include <stdio.h>
+
+#ifdef RTLD_GLOBAL
+# define LT_DLGLOBAL RTLD_GLOBAL
+#else
+# ifdef DL_GLOBAL
+# define LT_DLGLOBAL DL_GLOBAL
+# else
+# define LT_DLGLOBAL 0
+# endif
+#endif
+
+/* We may have to define LT_DLLAZY_OR_NOW in the command line if we
+ find out it does not work in some platform. */
+#ifndef LT_DLLAZY_OR_NOW
+# ifdef RTLD_LAZY
+# define LT_DLLAZY_OR_NOW RTLD_LAZY
+# else
+# ifdef DL_LAZY
+# define LT_DLLAZY_OR_NOW DL_LAZY
+# else
+# ifdef RTLD_NOW
+# define LT_DLLAZY_OR_NOW RTLD_NOW
+# else
+# ifdef DL_NOW
+# define LT_DLLAZY_OR_NOW DL_NOW
+# else
+# define LT_DLLAZY_OR_NOW 0
+# endif
+# endif
+# endif
+# endif
+#endif
+
+void fnord() { int i=42;}
+int main ()
+{
+ void *self = dlopen (0, LT_DLGLOBAL|LT_DLLAZY_OR_NOW);
+ int status = $lt_dlunknown;
+
+ if (self)
+ {
+ if (dlsym (self,"fnord")) status = $lt_dlno_uscore;
+ else if (dlsym( self,"_fnord")) status = $lt_dlneed_uscore;
+ /* dlclose (self); */
+ }
+ else
+ puts (dlerror ());
+
+ return status;
+}
+_LT_EOF
+ if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_link\""; } >&5
+ (eval $ac_link) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; } && test -s conftest${ac_exeext} 2>/dev/null; then
+ (./conftest; exit; ) >&5 2>/dev/null
+ lt_status=$?
+ case x$lt_status in
+ x$lt_dlno_uscore) lt_cv_dlopen_self_static=yes ;;
+ x$lt_dlneed_uscore) lt_cv_dlopen_self_static=yes ;;
+ x$lt_dlunknown|x*) lt_cv_dlopen_self_static=no ;;
+ esac
+ else :
+ # compilation failed
+ lt_cv_dlopen_self_static=no
+ fi
+fi
+rm -fr conftest*
+
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_dlopen_self_static" >&5
+$as_echo "$lt_cv_dlopen_self_static" >&6; }
+ fi
+
+ CPPFLAGS="$save_CPPFLAGS"
+ LDFLAGS="$save_LDFLAGS"
+ LIBS="$save_LIBS"
+ ;;
+ esac
+
+ case $lt_cv_dlopen_self in
+ yes|no) enable_dlopen_self=$lt_cv_dlopen_self ;;
+ *) enable_dlopen_self=unknown ;;
+ esac
+
+ case $lt_cv_dlopen_self_static in
+ yes|no) enable_dlopen_self_static=$lt_cv_dlopen_self_static ;;
+ *) enable_dlopen_self_static=unknown ;;
+ esac
+fi
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+striplib=
+old_striplib=
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether stripping libraries is possible" >&5
+$as_echo_n "checking whether stripping libraries is possible... " >&6; }
+if test -n "$STRIP" && $STRIP -V 2>&1 | $GREP "GNU strip" >/dev/null; then
+ test -z "$old_striplib" && old_striplib="$STRIP --strip-debug"
+ test -z "$striplib" && striplib="$STRIP --strip-unneeded"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+# FIXME - insert some real tests, host_os isn't really good enough
+ case $host_os in
+ darwin*)
+ if test -n "$STRIP" ; then
+ striplib="$STRIP -x"
+ old_striplib="$STRIP -S"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ fi
+ ;;
+ *)
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ ;;
+ esac
+fi
+
+
+
+
+
+
+
+
+
+
+
+
+ # Report which library types will actually be built
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking if libtool supports shared libraries" >&5
+$as_echo_n "checking if libtool supports shared libraries... " >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $can_build_shared" >&5
+$as_echo "$can_build_shared" >&6; }
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build shared libraries" >&5
+$as_echo_n "checking whether to build shared libraries... " >&6; }
+ test "$can_build_shared" = "no" && enable_shared=no
+
+ # On AIX, shared libraries and static libraries use the same namespace, and
+ # are all built from PIC.
+ case $host_os in
+ aix3*)
+ test "$enable_shared" = yes && enable_static=no
+ if test -n "$RANLIB"; then
+ archive_cmds="$archive_cmds~\$RANLIB \$lib"
+ postinstall_cmds='$RANLIB $lib'
+ fi
+ ;;
+
+ aix[4-9]*)
+ if test "$host_cpu" != ia64 && test "$aix_use_runtimelinking" = no ; then
+ test "$enable_shared" = yes && enable_static=no
+ fi
+ ;;
+ esac
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_shared" >&5
+$as_echo "$enable_shared" >&6; }
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build static libraries" >&5
+$as_echo_n "checking whether to build static libraries... " >&6; }
+ # Make sure either enable_shared or enable_static is yes.
+ test "$enable_shared" = yes || enable_static=yes
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_static" >&5
+$as_echo "$enable_static" >&6; }
+
+
+
+
+fi
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+CC="$lt_save_CC"
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ac_config_commands="$ac_config_commands libtool"
+
+
+
+
+# Only expand once:
+
+
+: ${CONFIG_LT=./config.lt}
+{ $as_echo "$as_me:${as_lineno-$LINENO}: creating $CONFIG_LT" >&5
+$as_echo "$as_me: creating $CONFIG_LT" >&6;}
+cat >"$CONFIG_LT" <<_LTEOF
+#! $SHELL
+# Generated by $as_me.
+# Run this file to recreate a libtool stub with the current configuration.
+
+lt_cl_silent=false
+SHELL=\${CONFIG_SHELL-$SHELL}
+_LTEOF
+
+cat >>"$CONFIG_LT" <<\_LTEOF
+## -------------------- ##
+## M4sh Initialization. ##
+## -------------------- ##
+
+# Be more Bourne compatible
+DUALCASE=1; export DUALCASE # for MKS sh
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then :
+ emulate sh
+ NULLCMD=:
+ # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which
+ # is contrary to our usage. Disable this feature.
+ alias -g '${1+"$@"}'='"$@"'
+ setopt NO_GLOB_SUBST
+else
+ case `(set -o) 2>/dev/null` in #(
+ *posix*) :
+ set -o posix ;; #(
+ *) :
+ ;;
+esac
+fi
+
+
+as_nl='
+'
+export as_nl
+# Printing a long string crashes Solaris 7 /usr/bin/printf.
+as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'
+as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo
+as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo
+# Prefer a ksh shell builtin over an external printf program on Solaris,
+# but without wasting forks for bash or zsh.
+if test -z "$BASH_VERSION$ZSH_VERSION" \
+ && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then
+ as_echo='print -r --'
+ as_echo_n='print -rn --'
+elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then
+ as_echo='printf %s\n'
+ as_echo_n='printf %s'
+else
+ if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then
+ as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"'
+ as_echo_n='/usr/ucb/echo -n'
+ else
+ as_echo_body='eval expr "X$1" : "X\\(.*\\)"'
+ as_echo_n_body='eval
+ arg=$1;
+ case $arg in #(
+ *"$as_nl"*)
+ expr "X$arg" : "X\\(.*\\)$as_nl";
+ arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;;
+ esac;
+ expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl"
+ '
+ export as_echo_n_body
+ as_echo_n='sh -c $as_echo_n_body as_echo'
+ fi
+ export as_echo_body
+ as_echo='sh -c $as_echo_body as_echo'
+fi
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+ PATH_SEPARATOR=:
+ (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && {
+ (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 ||
+ PATH_SEPARATOR=';'
+ }
+fi
+
+
+# IFS
+# We need space, tab and new line, in precisely that order. Quoting is
+# there to prevent editors from complaining about space-tab.
+# (If _AS_PATH_WALK were called with IFS unset, it would disable word
+# splitting by setting IFS to empty value.)
+IFS=" "" $as_nl"
+
+# Find who we are. Look in the path if we contain no directory separator.
+case $0 in #((
+ *[\\/]* ) as_myself=$0 ;;
+ *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+ done
+IFS=$as_save_IFS
+
+ ;;
+esac
+# We did not find ourselves, most probably we were run as `sh COMMAND'
+# in which case we are not to be found in the path.
+if test "x$as_myself" = x; then
+ as_myself=$0
+fi
+if test ! -f "$as_myself"; then
+ $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2
+ exit 1
+fi
+
+# Unset variables that we do not need and which cause bugs (e.g. in
+# pre-3.0 UWIN ksh). But do not cause bugs in bash 2.01; the "|| exit 1"
+# suppresses any "Segmentation fault" message there. '((' could
+# trigger a bug in pdksh 5.2.14.
+for as_var in BASH_ENV ENV MAIL MAILPATH
+do eval test x\${$as_var+set} = xset \
+ && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || :
+done
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+LC_ALL=C
+export LC_ALL
+LANGUAGE=C
+export LANGUAGE
+
+# CDPATH.
+(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
+
+
+
+# as_fn_error STATUS ERROR [LINENO LOG_FD]
+# ----------------------------------------
+# Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are
+# provided, also output the error to LOG_FD, referencing LINENO. Then exit the
+# script with STATUS, using 1 if that was 0.
+as_fn_error ()
+{
+ as_status=$1; test $as_status -eq 0 && as_status=1
+ if test "$4"; then
+ as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+ $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4
+ fi
+ $as_echo "$as_me: error: $2" >&2
+ as_fn_exit $as_status
+} # as_fn_error
+
+
+# as_fn_set_status STATUS
+# -----------------------
+# Set $? to STATUS, without forking.
+as_fn_set_status ()
+{
+ return $1
+} # as_fn_set_status
+
+# as_fn_exit STATUS
+# -----------------
+# Exit the shell with STATUS, even in a "trap 0" or "set -e" context.
+as_fn_exit ()
+{
+ set +e
+ as_fn_set_status $1
+ exit $1
+} # as_fn_exit
+
+# as_fn_unset VAR
+# ---------------
+# Portably unset VAR.
+as_fn_unset ()
+{
+ { eval $1=; unset $1;}
+}
+as_unset=as_fn_unset
+# as_fn_append VAR VALUE
+# ----------------------
+# Append the text in VALUE to the end of the definition contained in VAR. Take
+# advantage of any shell optimizations that allow amortized linear growth over
+# repeated appends, instead of the typical quadratic growth present in naive
+# implementations.
+if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null; then :
+ eval 'as_fn_append ()
+ {
+ eval $1+=\$2
+ }'
+else
+ as_fn_append ()
+ {
+ eval $1=\$$1\$2
+ }
+fi # as_fn_append
+
+# as_fn_arith ARG...
+# ------------------
+# Perform arithmetic evaluation on the ARGs, and store the result in the
+# global $as_val. Take advantage of shells that can avoid forks. The arguments
+# must be portable across $(()) and expr.
+if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null; then :
+ eval 'as_fn_arith ()
+ {
+ as_val=$(( $* ))
+ }'
+else
+ as_fn_arith ()
+ {
+ as_val=`expr "$@" || test $? -eq 1`
+ }
+fi # as_fn_arith
+
+
+if expr a : '\(a\)' >/dev/null 2>&1 &&
+ test "X`expr 00001 : '.*\(...\)'`" = X001; then
+ as_expr=expr
+else
+ as_expr=false
+fi
+
+if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then
+ as_basename=basename
+else
+ as_basename=false
+fi
+
+if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then
+ as_dirname=dirname
+else
+ as_dirname=false
+fi
+
+as_me=`$as_basename -- "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+ X"$0" : 'X\(//\)$' \| \
+ X"$0" : 'X\(/\)' \| . 2>/dev/null ||
+$as_echo X/"$0" |
+ sed '/^.*\/\([^/][^/]*\)\/*$/{
+ s//\1/
+ q
+ }
+ /^X\/\(\/\/\)$/{
+ s//\1/
+ q
+ }
+ /^X\/\(\/\).*/{
+ s//\1/
+ q
+ }
+ s/.*/./; q'`
+
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+ECHO_C= ECHO_N= ECHO_T=
+case `echo -n x` in #(((((
+-n*)
+ case `echo 'xy\c'` in
+ *c*) ECHO_T=' ';; # ECHO_T is single tab character.
+ xy) ECHO_C='\c';;
+ *) echo `echo ksh88 bug on AIX 6.1` > /dev/null
+ ECHO_T=' ';;
+ esac;;
+*)
+ ECHO_N='-n';;
+esac
+
+rm -f conf$$ conf$$.exe conf$$.file
+if test -d conf$$.dir; then
+ rm -f conf$$.dir/conf$$.file
+else
+ rm -f conf$$.dir
+ mkdir conf$$.dir 2>/dev/null
+fi
+if (echo >conf$$.file) 2>/dev/null; then
+ if ln -s conf$$.file conf$$ 2>/dev/null; then
+ as_ln_s='ln -s'
+ # ... but there are two gotchas:
+ # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail.
+ # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable.
+ # In both cases, we have to default to `cp -p'.
+ ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe ||
+ as_ln_s='cp -p'
+ elif ln conf$$.file conf$$ 2>/dev/null; then
+ as_ln_s=ln
+ else
+ as_ln_s='cp -p'
+ fi
+else
+ as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file
+rmdir conf$$.dir 2>/dev/null
+
+
+# as_fn_mkdir_p
+# -------------
+# Create "$as_dir" as a directory, including parents if necessary.
+as_fn_mkdir_p ()
+{
+
+ case $as_dir in #(
+ -*) as_dir=./$as_dir;;
+ esac
+ test -d "$as_dir" || eval $as_mkdir_p || {
+ as_dirs=
+ while :; do
+ case $as_dir in #(
+ *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'(
+ *) as_qdir=$as_dir;;
+ esac
+ as_dirs="'$as_qdir' $as_dirs"
+ as_dir=`$as_dirname -- "$as_dir" ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+ X"$as_dir" : 'X\(//\)[^/]' \| \
+ X"$as_dir" : 'X\(//\)$' \| \
+ X"$as_dir" : 'X\(/\)' \| . 2>/dev/null ||
+$as_echo X"$as_dir" |
+ sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)[^/].*/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\).*/{
+ s//\1/
+ q
+ }
+ s/.*/./; q'`
+ test -d "$as_dir" && break
+ done
+ test -z "$as_dirs" || eval "mkdir $as_dirs"
+ } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" "$LINENO" 5
+
+
+} # as_fn_mkdir_p
+if mkdir -p . 2>/dev/null; then
+ as_mkdir_p='mkdir -p "$as_dir"'
+else
+ test -d ./-p && rmdir ./-p
+ as_mkdir_p=false
+fi
+
+if test -x / >/dev/null 2>&1; then
+ as_test_x='test -x'
+else
+ if ls -dL / >/dev/null 2>&1; then
+ as_ls_L_option=L
+ else
+ as_ls_L_option=
+ fi
+ as_test_x='
+ eval sh -c '\''
+ if test -d "$1"; then
+ test -d "$1/.";
+ else
+ case $1 in #(
+ -*)set "./$1";;
+ esac;
+ case `ls -ld'$as_ls_L_option' "$1" 2>/dev/null` in #((
+ ???[sx]*):;;*)false;;esac;fi
+ '\'' sh
+ '
+fi
+as_executable_p=$as_test_x
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+
+exec 6>&1
+exec 5>>config.log
+{
+ echo
+ sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX
+## Running $as_me. ##
+_ASBOX
+} >&5
+
+lt_cl_help="\
+\`$as_me' creates a local libtool stub from the current configuration,
+for use in further configure time tests before the real libtool is
+generated.
+
+Usage: $0 [OPTIONS]
+
+ -h, --help print this help, then exit
+ -V, --version print version number, then exit
+ -q, --quiet do not print progress messages
+ -d, --debug don't remove temporary files
+
+Report bugs to <bug-libtool at gnu.org>."
+
+lt_cl_version="\
+libfabric config.lt 1.4.0rc2
+configured by $0, generated by GNU Autoconf 2.67.
+
+Copyright (C) 2008 Free Software Foundation, Inc.
+This config.lt script is free software; the Free Software Foundation
+gives unlimited permision to copy, distribute and modify it."
+
+while test $# != 0
+do
+ case $1 in
+ --version | --v* | -V )
+ echo "$lt_cl_version"; exit 0 ;;
+ --help | --h* | -h )
+ echo "$lt_cl_help"; exit 0 ;;
+ --debug | --d* | -d )
+ debug=: ;;
+ --quiet | --q* | --silent | --s* | -q )
+ lt_cl_silent=: ;;
+
+ -*) as_fn_error $? "unrecognized option: $1
+Try \`$0 --help' for more information." "$LINENO" 5 ;;
+
+ *) as_fn_error $? "unrecognized argument: $1
+Try \`$0 --help' for more information." "$LINENO" 5 ;;
+ esac
+ shift
+done
+
+if $lt_cl_silent; then
+ exec 6>/dev/null
+fi
+_LTEOF
+
+cat >>"$CONFIG_LT" <<_LTEOF
+
+
+# The HP-UX ksh and POSIX shell print the target directory to stdout
+# if CDPATH is set.
+(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
+
+sed_quote_subst='$sed_quote_subst'
+double_quote_subst='$double_quote_subst'
+delay_variable_subst='$delay_variable_subst'
+macro_version='`$ECHO "X$macro_version" | $Xsed -e "$delay_single_quote_subst"`'
+macro_revision='`$ECHO "X$macro_revision" | $Xsed -e "$delay_single_quote_subst"`'
+enable_shared='`$ECHO "X$enable_shared" | $Xsed -e "$delay_single_quote_subst"`'
+enable_static='`$ECHO "X$enable_static" | $Xsed -e "$delay_single_quote_subst"`'
+pic_mode='`$ECHO "X$pic_mode" | $Xsed -e "$delay_single_quote_subst"`'
+enable_fast_install='`$ECHO "X$enable_fast_install" | $Xsed -e "$delay_single_quote_subst"`'
+host_alias='`$ECHO "X$host_alias" | $Xsed -e "$delay_single_quote_subst"`'
+host='`$ECHO "X$host" | $Xsed -e "$delay_single_quote_subst"`'
+host_os='`$ECHO "X$host_os" | $Xsed -e "$delay_single_quote_subst"`'
+build_alias='`$ECHO "X$build_alias" | $Xsed -e "$delay_single_quote_subst"`'
+build='`$ECHO "X$build" | $Xsed -e "$delay_single_quote_subst"`'
+build_os='`$ECHO "X$build_os" | $Xsed -e "$delay_single_quote_subst"`'
+SED='`$ECHO "X$SED" | $Xsed -e "$delay_single_quote_subst"`'
+Xsed='`$ECHO "X$Xsed" | $Xsed -e "$delay_single_quote_subst"`'
+GREP='`$ECHO "X$GREP" | $Xsed -e "$delay_single_quote_subst"`'
+EGREP='`$ECHO "X$EGREP" | $Xsed -e "$delay_single_quote_subst"`'
+FGREP='`$ECHO "X$FGREP" | $Xsed -e "$delay_single_quote_subst"`'
+LD='`$ECHO "X$LD" | $Xsed -e "$delay_single_quote_subst"`'
+NM='`$ECHO "X$NM" | $Xsed -e "$delay_single_quote_subst"`'
+LN_S='`$ECHO "X$LN_S" | $Xsed -e "$delay_single_quote_subst"`'
+max_cmd_len='`$ECHO "X$max_cmd_len" | $Xsed -e "$delay_single_quote_subst"`'
+ac_objext='`$ECHO "X$ac_objext" | $Xsed -e "$delay_single_quote_subst"`'
+exeext='`$ECHO "X$exeext" | $Xsed -e "$delay_single_quote_subst"`'
+lt_unset='`$ECHO "X$lt_unset" | $Xsed -e "$delay_single_quote_subst"`'
+lt_SP2NL='`$ECHO "X$lt_SP2NL" | $Xsed -e "$delay_single_quote_subst"`'
+lt_NL2SP='`$ECHO "X$lt_NL2SP" | $Xsed -e "$delay_single_quote_subst"`'
+reload_flag='`$ECHO "X$reload_flag" | $Xsed -e "$delay_single_quote_subst"`'
+reload_cmds='`$ECHO "X$reload_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+OBJDUMP='`$ECHO "X$OBJDUMP" | $Xsed -e "$delay_single_quote_subst"`'
+deplibs_check_method='`$ECHO "X$deplibs_check_method" | $Xsed -e "$delay_single_quote_subst"`'
+file_magic_cmd='`$ECHO "X$file_magic_cmd" | $Xsed -e "$delay_single_quote_subst"`'
+AR='`$ECHO "X$AR" | $Xsed -e "$delay_single_quote_subst"`'
+AR_FLAGS='`$ECHO "X$AR_FLAGS" | $Xsed -e "$delay_single_quote_subst"`'
+STRIP='`$ECHO "X$STRIP" | $Xsed -e "$delay_single_quote_subst"`'
+RANLIB='`$ECHO "X$RANLIB" | $Xsed -e "$delay_single_quote_subst"`'
+old_postinstall_cmds='`$ECHO "X$old_postinstall_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+old_postuninstall_cmds='`$ECHO "X$old_postuninstall_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+old_archive_cmds='`$ECHO "X$old_archive_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+CC='`$ECHO "X$CC" | $Xsed -e "$delay_single_quote_subst"`'
+CFLAGS='`$ECHO "X$CFLAGS" | $Xsed -e "$delay_single_quote_subst"`'
+compiler='`$ECHO "X$compiler" | $Xsed -e "$delay_single_quote_subst"`'
+GCC='`$ECHO "X$GCC" | $Xsed -e "$delay_single_quote_subst"`'
+lt_cv_sys_global_symbol_pipe='`$ECHO "X$lt_cv_sys_global_symbol_pipe" | $Xsed -e "$delay_single_quote_subst"`'
+lt_cv_sys_global_symbol_to_cdecl='`$ECHO "X$lt_cv_sys_global_symbol_to_cdecl" | $Xsed -e "$delay_single_quote_subst"`'
+lt_cv_sys_global_symbol_to_c_name_address='`$ECHO "X$lt_cv_sys_global_symbol_to_c_name_address" | $Xsed -e "$delay_single_quote_subst"`'
+lt_cv_sys_global_symbol_to_c_name_address_lib_prefix='`$ECHO "X$lt_cv_sys_global_symbol_to_c_name_address_lib_prefix" | $Xsed -e "$delay_single_quote_subst"`'
+objdir='`$ECHO "X$objdir" | $Xsed -e "$delay_single_quote_subst"`'
+SHELL='`$ECHO "X$SHELL" | $Xsed -e "$delay_single_quote_subst"`'
+ECHO='`$ECHO "X$ECHO" | $Xsed -e "$delay_single_quote_subst"`'
+MAGIC_CMD='`$ECHO "X$MAGIC_CMD" | $Xsed -e "$delay_single_quote_subst"`'
+lt_prog_compiler_no_builtin_flag='`$ECHO "X$lt_prog_compiler_no_builtin_flag" | $Xsed -e "$delay_single_quote_subst"`'
+lt_prog_compiler_wl='`$ECHO "X$lt_prog_compiler_wl" | $Xsed -e "$delay_single_quote_subst"`'
+lt_prog_compiler_pic='`$ECHO "X$lt_prog_compiler_pic" | $Xsed -e "$delay_single_quote_subst"`'
+lt_prog_compiler_static='`$ECHO "X$lt_prog_compiler_static" | $Xsed -e "$delay_single_quote_subst"`'
+lt_cv_prog_compiler_c_o='`$ECHO "X$lt_cv_prog_compiler_c_o" | $Xsed -e "$delay_single_quote_subst"`'
+need_locks='`$ECHO "X$need_locks" | $Xsed -e "$delay_single_quote_subst"`'
+DSYMUTIL='`$ECHO "X$DSYMUTIL" | $Xsed -e "$delay_single_quote_subst"`'
+NMEDIT='`$ECHO "X$NMEDIT" | $Xsed -e "$delay_single_quote_subst"`'
+LIPO='`$ECHO "X$LIPO" | $Xsed -e "$delay_single_quote_subst"`'
+OTOOL='`$ECHO "X$OTOOL" | $Xsed -e "$delay_single_quote_subst"`'
+OTOOL64='`$ECHO "X$OTOOL64" | $Xsed -e "$delay_single_quote_subst"`'
+libext='`$ECHO "X$libext" | $Xsed -e "$delay_single_quote_subst"`'
+shrext_cmds='`$ECHO "X$shrext_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+extract_expsyms_cmds='`$ECHO "X$extract_expsyms_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+archive_cmds_need_lc='`$ECHO "X$archive_cmds_need_lc" | $Xsed -e "$delay_single_quote_subst"`'
+enable_shared_with_static_runtimes='`$ECHO "X$enable_shared_with_static_runtimes" | $Xsed -e "$delay_single_quote_subst"`'
+export_dynamic_flag_spec='`$ECHO "X$export_dynamic_flag_spec" | $Xsed -e "$delay_single_quote_subst"`'
+whole_archive_flag_spec='`$ECHO "X$whole_archive_flag_spec" | $Xsed -e "$delay_single_quote_subst"`'
+compiler_needs_object='`$ECHO "X$compiler_needs_object" | $Xsed -e "$delay_single_quote_subst"`'
+old_archive_from_new_cmds='`$ECHO "X$old_archive_from_new_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+old_archive_from_expsyms_cmds='`$ECHO "X$old_archive_from_expsyms_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+archive_cmds='`$ECHO "X$archive_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+archive_expsym_cmds='`$ECHO "X$archive_expsym_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+module_cmds='`$ECHO "X$module_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+module_expsym_cmds='`$ECHO "X$module_expsym_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+with_gnu_ld='`$ECHO "X$with_gnu_ld" | $Xsed -e "$delay_single_quote_subst"`'
+allow_undefined_flag='`$ECHO "X$allow_undefined_flag" | $Xsed -e "$delay_single_quote_subst"`'
+no_undefined_flag='`$ECHO "X$no_undefined_flag" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_libdir_flag_spec='`$ECHO "X$hardcode_libdir_flag_spec" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_libdir_flag_spec_ld='`$ECHO "X$hardcode_libdir_flag_spec_ld" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_libdir_separator='`$ECHO "X$hardcode_libdir_separator" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_direct='`$ECHO "X$hardcode_direct" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_direct_absolute='`$ECHO "X$hardcode_direct_absolute" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_minus_L='`$ECHO "X$hardcode_minus_L" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_shlibpath_var='`$ECHO "X$hardcode_shlibpath_var" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_automatic='`$ECHO "X$hardcode_automatic" | $Xsed -e "$delay_single_quote_subst"`'
+inherit_rpath='`$ECHO "X$inherit_rpath" | $Xsed -e "$delay_single_quote_subst"`'
+link_all_deplibs='`$ECHO "X$link_all_deplibs" | $Xsed -e "$delay_single_quote_subst"`'
+fix_srcfile_path='`$ECHO "X$fix_srcfile_path" | $Xsed -e "$delay_single_quote_subst"`'
+always_export_symbols='`$ECHO "X$always_export_symbols" | $Xsed -e "$delay_single_quote_subst"`'
+export_symbols_cmds='`$ECHO "X$export_symbols_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+exclude_expsyms='`$ECHO "X$exclude_expsyms" | $Xsed -e "$delay_single_quote_subst"`'
+include_expsyms='`$ECHO "X$include_expsyms" | $Xsed -e "$delay_single_quote_subst"`'
+prelink_cmds='`$ECHO "X$prelink_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+file_list_spec='`$ECHO "X$file_list_spec" | $Xsed -e "$delay_single_quote_subst"`'
+variables_saved_for_relink='`$ECHO "X$variables_saved_for_relink" | $Xsed -e "$delay_single_quote_subst"`'
+need_lib_prefix='`$ECHO "X$need_lib_prefix" | $Xsed -e "$delay_single_quote_subst"`'
+need_version='`$ECHO "X$need_version" | $Xsed -e "$delay_single_quote_subst"`'
+version_type='`$ECHO "X$version_type" | $Xsed -e "$delay_single_quote_subst"`'
+runpath_var='`$ECHO "X$runpath_var" | $Xsed -e "$delay_single_quote_subst"`'
+shlibpath_var='`$ECHO "X$shlibpath_var" | $Xsed -e "$delay_single_quote_subst"`'
+shlibpath_overrides_runpath='`$ECHO "X$shlibpath_overrides_runpath" | $Xsed -e "$delay_single_quote_subst"`'
+libname_spec='`$ECHO "X$libname_spec" | $Xsed -e "$delay_single_quote_subst"`'
+library_names_spec='`$ECHO "X$library_names_spec" | $Xsed -e "$delay_single_quote_subst"`'
+soname_spec='`$ECHO "X$soname_spec" | $Xsed -e "$delay_single_quote_subst"`'
+postinstall_cmds='`$ECHO "X$postinstall_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+postuninstall_cmds='`$ECHO "X$postuninstall_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+finish_cmds='`$ECHO "X$finish_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+finish_eval='`$ECHO "X$finish_eval" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_into_libs='`$ECHO "X$hardcode_into_libs" | $Xsed -e "$delay_single_quote_subst"`'
+sys_lib_search_path_spec='`$ECHO "X$sys_lib_search_path_spec" | $Xsed -e "$delay_single_quote_subst"`'
+sys_lib_dlsearch_path_spec='`$ECHO "X$sys_lib_dlsearch_path_spec" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_action='`$ECHO "X$hardcode_action" | $Xsed -e "$delay_single_quote_subst"`'
+enable_dlopen='`$ECHO "X$enable_dlopen" | $Xsed -e "$delay_single_quote_subst"`'
+enable_dlopen_self='`$ECHO "X$enable_dlopen_self" | $Xsed -e "$delay_single_quote_subst"`'
+enable_dlopen_self_static='`$ECHO "X$enable_dlopen_self_static" | $Xsed -e "$delay_single_quote_subst"`'
+old_striplib='`$ECHO "X$old_striplib" | $Xsed -e "$delay_single_quote_subst"`'
+striplib='`$ECHO "X$striplib" | $Xsed -e "$delay_single_quote_subst"`'
+
+LTCC='$LTCC'
+LTCFLAGS='$LTCFLAGS'
+compiler='$compiler_DEFAULT'
+
+# Quote evaled strings.
+for var in SED \
+GREP \
+EGREP \
+FGREP \
+LD \
+NM \
+LN_S \
+lt_SP2NL \
+lt_NL2SP \
+reload_flag \
+OBJDUMP \
+deplibs_check_method \
+file_magic_cmd \
+AR \
+AR_FLAGS \
+STRIP \
+RANLIB \
+CC \
+CFLAGS \
+compiler \
+lt_cv_sys_global_symbol_pipe \
+lt_cv_sys_global_symbol_to_cdecl \
+lt_cv_sys_global_symbol_to_c_name_address \
+lt_cv_sys_global_symbol_to_c_name_address_lib_prefix \
+SHELL \
+ECHO \
+lt_prog_compiler_no_builtin_flag \
+lt_prog_compiler_wl \
+lt_prog_compiler_pic \
+lt_prog_compiler_static \
+lt_cv_prog_compiler_c_o \
+need_locks \
+DSYMUTIL \
+NMEDIT \
+LIPO \
+OTOOL \
+OTOOL64 \
+shrext_cmds \
+export_dynamic_flag_spec \
+whole_archive_flag_spec \
+compiler_needs_object \
+with_gnu_ld \
+allow_undefined_flag \
+no_undefined_flag \
+hardcode_libdir_flag_spec \
+hardcode_libdir_flag_spec_ld \
+hardcode_libdir_separator \
+fix_srcfile_path \
+exclude_expsyms \
+include_expsyms \
+file_list_spec \
+variables_saved_for_relink \
+libname_spec \
+library_names_spec \
+soname_spec \
+finish_eval \
+old_striplib \
+striplib; do
+ case \`eval \\\\\$ECHO "X\\\\\$\$var"\` in
+ *[\\\\\\\`\\"\\\$]*)
+ eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"X\\\$\$var\\" | \\\$Xsed -e \\"\\\$sed_quote_subst\\"\\\`\\\\\\""
+ ;;
+ *)
+ eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\""
+ ;;
+ esac
+done
+
+# Double-quote double-evaled strings.
+for var in reload_cmds \
+old_postinstall_cmds \
+old_postuninstall_cmds \
+old_archive_cmds \
+extract_expsyms_cmds \
+old_archive_from_new_cmds \
+old_archive_from_expsyms_cmds \
+archive_cmds \
+archive_expsym_cmds \
+module_cmds \
+module_expsym_cmds \
+export_symbols_cmds \
+prelink_cmds \
+postinstall_cmds \
+postuninstall_cmds \
+finish_cmds \
+sys_lib_search_path_spec \
+sys_lib_dlsearch_path_spec; do
+ case \`eval \\\\\$ECHO "X\\\\\$\$var"\` in
+ *[\\\\\\\`\\"\\\$]*)
+ eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"X\\\$\$var\\" | \\\$Xsed -e \\"\\\$double_quote_subst\\" -e \\"\\\$sed_quote_subst\\" -e \\"\\\$delay_variable_subst\\"\\\`\\\\\\""
+ ;;
+ *)
+ eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\""
+ ;;
+ esac
+done
+
+# Fix-up fallback echo if it was mangled by the above quoting rules.
+case \$lt_ECHO in
+*'\\\$0 --fallback-echo"') lt_ECHO=\`\$ECHO "X\$lt_ECHO" | \$Xsed -e 's/\\\\\\\\\\\\\\\$0 --fallback-echo"\$/\$0 --fallback-echo"/'\`
+ ;;
+esac
+
+ac_aux_dir='$ac_aux_dir'
+xsi_shell='$xsi_shell'
+lt_shell_append='$lt_shell_append'
+
+# See if we are running on zsh, and set the options which allow our
+# commands through without removal of \ escapes INIT.
+if test -n "\${ZSH_VERSION+set}" ; then
+ setopt NO_GLOB_SUBST
+fi
+
+
+ PACKAGE='$PACKAGE'
+ VERSION='$VERSION'
+ TIMESTAMP='$TIMESTAMP'
+ RM='$RM'
+ ofile='$ofile'
+
+
+
+_LTEOF
+
+cat >>"$CONFIG_LT" <<\_LTEOF
+{ $as_echo "$as_me:${as_lineno-$LINENO}: creating $ofile" >&5
+$as_echo "$as_me: creating $ofile" >&6;}
+
+
+ # See if we are running on zsh, and set the options which allow our
+ # commands through without removal of \ escapes.
+ if test -n "${ZSH_VERSION+set}" ; then
+ setopt NO_GLOB_SUBST
+ fi
+
+ cfgfile="${ofile}T"
+ trap "$RM \"$cfgfile\"; exit 1" 1 2 15
+ $RM "$cfgfile"
+
+ cat <<_LT_EOF >> "$cfgfile"
+#! $SHELL
+
+# `$ECHO "$ofile" | sed 's%^.*/%%'` - Provide generalized library-building support services.
+# Generated automatically by $as_me ($PACKAGE$TIMESTAMP) $VERSION
+# Libtool was configured on host `(hostname || uname -n) 2>/dev/null | sed 1q`:
+# NOTE: Changes made to this file will be lost: look at ltmain.sh.
+#
+# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, 2005,
+# 2006, 2007, 2008 Free Software Foundation, Inc.
+# Written by Gordon Matzigkeit, 1996
+#
+# This file is part of GNU Libtool.
+#
+# GNU Libtool is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of
+# the License, or (at your option) any later version.
+#
+# As a special exception to the GNU General Public License,
+# if you distribute this file as part of a program or library that
+# is built using GNU Libtool, you may include this file under the
+# same distribution terms that you use for the rest of that program.
+#
+# GNU Libtool is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Libtool; see the file COPYING. If not, a copy
+# can be downloaded from http://www.gnu.org/licenses/gpl.html, or
+# obtained by writing to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+
+# The names of the tagged configurations supported by this script.
+available_tags=""
+
+# ### BEGIN LIBTOOL CONFIG
+
+# Which release of libtool.m4 was used?
+macro_version=$macro_version
+macro_revision=$macro_revision
+
+# Whether or not to build shared libraries.
+build_libtool_libs=$enable_shared
+
+# Whether or not to build static libraries.
+build_old_libs=$enable_static
+
+# What type of objects to build.
+pic_mode=$pic_mode
+
+# Whether or not to optimize for fast installation.
+fast_install=$enable_fast_install
+
+# The host system.
+host_alias=$host_alias
+host=$host
+host_os=$host_os
+
+# The build system.
+build_alias=$build_alias
+build=$build
+build_os=$build_os
+
+# A sed program that does not truncate output.
+SED=$lt_SED
+
+# Sed that helps us avoid accidentally triggering echo(1) options like -n.
+Xsed="\$SED -e 1s/^X//"
+
+# A grep program that handles long lines.
+GREP=$lt_GREP
+
+# An ERE matcher.
+EGREP=$lt_EGREP
+
+# A literal string matcher.
+FGREP=$lt_FGREP
+
+# A BSD- or MS-compatible name lister.
+NM=$lt_NM
+
+# Whether we need soft or hard links.
+LN_S=$lt_LN_S
+
+# What is the maximum length of a command?
+max_cmd_len=$max_cmd_len
+
+# Object file suffix (normally "o").
+objext=$ac_objext
+
+# Executable file suffix (normally "").
+exeext=$exeext
+
+# whether the shell understands "unset".
+lt_unset=$lt_unset
+
+# turn spaces into newlines.
+SP2NL=$lt_lt_SP2NL
+
+# turn newlines into spaces.
+NL2SP=$lt_lt_NL2SP
+
+# How to create reloadable object files.
+reload_flag=$lt_reload_flag
+reload_cmds=$lt_reload_cmds
+
+# An object symbol dumper.
+OBJDUMP=$lt_OBJDUMP
+
+# Method to check whether dependent libraries are shared objects.
+deplibs_check_method=$lt_deplibs_check_method
+
+# Command to use when deplibs_check_method == "file_magic".
+file_magic_cmd=$lt_file_magic_cmd
+
+# The archiver.
+AR=$lt_AR
+AR_FLAGS=$lt_AR_FLAGS
+
+# A symbol stripping program.
+STRIP=$lt_STRIP
+
+# Commands used to install an old-style archive.
+RANLIB=$lt_RANLIB
+old_postinstall_cmds=$lt_old_postinstall_cmds
+old_postuninstall_cmds=$lt_old_postuninstall_cmds
+
+# A C compiler.
+LTCC=$lt_CC
+
+# LTCC compiler flags.
+LTCFLAGS=$lt_CFLAGS
+
+# Take the output of nm and produce a listing of raw symbols and C names.
+global_symbol_pipe=$lt_lt_cv_sys_global_symbol_pipe
+
+# Transform the output of nm in a proper C declaration.
+global_symbol_to_cdecl=$lt_lt_cv_sys_global_symbol_to_cdecl
+
+# Transform the output of nm in a C name address pair.
+global_symbol_to_c_name_address=$lt_lt_cv_sys_global_symbol_to_c_name_address
+
+# Transform the output of nm in a C name address pair when lib prefix is needed.
+global_symbol_to_c_name_address_lib_prefix=$lt_lt_cv_sys_global_symbol_to_c_name_address_lib_prefix
+
+# The name of the directory that contains temporary libtool files.
+objdir=$objdir
+
+# Shell to use when invoking shell scripts.
+SHELL=$lt_SHELL
+
+# An echo program that does not interpret backslashes.
+ECHO=$lt_ECHO
+
+# Used to examine libraries when file_magic_cmd begins with "file".
+MAGIC_CMD=$MAGIC_CMD
+
+# Must we lock files when doing compilation?
+need_locks=$lt_need_locks
+
+# Tool to manipulate archived DWARF debug symbol files on Mac OS X.
+DSYMUTIL=$lt_DSYMUTIL
+
+# Tool to change global to local symbols on Mac OS X.
+NMEDIT=$lt_NMEDIT
+
+# Tool to manipulate fat objects and archives on Mac OS X.
+LIPO=$lt_LIPO
+
+# ldd/readelf like tool for Mach-O binaries on Mac OS X.
+OTOOL=$lt_OTOOL
+
+# ldd/readelf like tool for 64 bit Mach-O binaries on Mac OS X 10.4.
+OTOOL64=$lt_OTOOL64
+
+# Old archive suffix (normally "a").
+libext=$libext
+
+# Shared library suffix (normally ".so").
+shrext_cmds=$lt_shrext_cmds
+
+# The commands to extract the exported symbol list from a shared archive.
+extract_expsyms_cmds=$lt_extract_expsyms_cmds
+
+# Variables whose values should be saved in libtool wrapper scripts and
+# restored at link time.
+variables_saved_for_relink=$lt_variables_saved_for_relink
+
+# Do we need the "lib" prefix for modules?
+need_lib_prefix=$need_lib_prefix
+
+# Do we need a version for libraries?
+need_version=$need_version
+
+# Library versioning type.
+version_type=$version_type
+
+# Shared library runtime path variable.
+runpath_var=$runpath_var
+
+# Shared library path variable.
+shlibpath_var=$shlibpath_var
+
+# Is shlibpath searched before the hard-coded library search path?
+shlibpath_overrides_runpath=$shlibpath_overrides_runpath
+
+# Format of library name prefix.
+libname_spec=$lt_libname_spec
+
+# List of archive names. First name is the real one, the rest are links.
+# The last name is the one that the linker finds with -lNAME
+library_names_spec=$lt_library_names_spec
+
+# The coded name of the library, if different from the real name.
+soname_spec=$lt_soname_spec
+
+# Command to use after installation of a shared archive.
+postinstall_cmds=$lt_postinstall_cmds
+
+# Command to use after uninstallation of a shared archive.
+postuninstall_cmds=$lt_postuninstall_cmds
+
+# Commands used to finish a libtool library installation in a directory.
+finish_cmds=$lt_finish_cmds
+
+# As "finish_cmds", except a single script fragment to be evaled but
+# not shown.
+finish_eval=$lt_finish_eval
+
+# Whether we should hardcode library paths into libraries.
+hardcode_into_libs=$hardcode_into_libs
+
+# Compile-time system search path for libraries.
+sys_lib_search_path_spec=$lt_sys_lib_search_path_spec
+
+# Run-time system search path for libraries.
+sys_lib_dlsearch_path_spec=$lt_sys_lib_dlsearch_path_spec
+
+# Whether dlopen is supported.
+dlopen_support=$enable_dlopen
+
+# Whether dlopen of programs is supported.
+dlopen_self=$enable_dlopen_self
+
+# Whether dlopen of statically linked programs is supported.
+dlopen_self_static=$enable_dlopen_self_static
+
+# Commands to strip libraries.
+old_striplib=$lt_old_striplib
+striplib=$lt_striplib
+
+
+# The linker used to build libraries.
+LD=$lt_LD
+
+# Commands used to build an old-style archive.
+old_archive_cmds=$lt_old_archive_cmds
+
+# A language specific compiler.
+CC=$lt_compiler
+
+# Is the compiler the GNU compiler?
+with_gcc=$GCC
+
+# Compiler flag to turn off builtin functions.
+no_builtin_flag=$lt_lt_prog_compiler_no_builtin_flag
+
+# How to pass a linker flag through the compiler.
+wl=$lt_lt_prog_compiler_wl
+
+# Additional compiler flags for building library objects.
+pic_flag=$lt_lt_prog_compiler_pic
+
+# Compiler flag to prevent dynamic linking.
+link_static_flag=$lt_lt_prog_compiler_static
+
+# Does compiler simultaneously support -c and -o options?
+compiler_c_o=$lt_lt_cv_prog_compiler_c_o
+
+# Whether or not to add -lc for building shared libraries.
+build_libtool_need_lc=$archive_cmds_need_lc
+
+# Whether or not to disallow shared libs when runtime libs are static.
+allow_libtool_libs_with_static_runtimes=$enable_shared_with_static_runtimes
+
+# Compiler flag to allow reflexive dlopens.
+export_dynamic_flag_spec=$lt_export_dynamic_flag_spec
+
+# Compiler flag to generate shared objects directly from archives.
+whole_archive_flag_spec=$lt_whole_archive_flag_spec
+
+# Whether the compiler copes with passing no objects directly.
+compiler_needs_object=$lt_compiler_needs_object
+
+# Create an old-style archive from a shared archive.
+old_archive_from_new_cmds=$lt_old_archive_from_new_cmds
+
+# Create a temporary old-style archive to link instead of a shared archive.
+old_archive_from_expsyms_cmds=$lt_old_archive_from_expsyms_cmds
+
+# Commands used to build a shared archive.
+archive_cmds=$lt_archive_cmds
+archive_expsym_cmds=$lt_archive_expsym_cmds
+
+# Commands used to build a loadable module if different from building
+# a shared archive.
+module_cmds=$lt_module_cmds
+module_expsym_cmds=$lt_module_expsym_cmds
+
+# Whether we are building with GNU ld or not.
+with_gnu_ld=$lt_with_gnu_ld
+
+# Flag that allows shared libraries with undefined symbols to be built.
+allow_undefined_flag=$lt_allow_undefined_flag
+
+# Flag that enforces no undefined symbols.
+no_undefined_flag=$lt_no_undefined_flag
+
+# Flag to hardcode \$libdir into a binary during linking.
+# This must work even if \$libdir does not exist
+hardcode_libdir_flag_spec=$lt_hardcode_libdir_flag_spec
+
+# If ld is used when linking, flag to hardcode \$libdir into a binary
+# during linking. This must work even if \$libdir does not exist.
+hardcode_libdir_flag_spec_ld=$lt_hardcode_libdir_flag_spec_ld
+
+# Whether we need a single "-rpath" flag with a separated argument.
+hardcode_libdir_separator=$lt_hardcode_libdir_separator
+
+# Set to "yes" if using DIR/libNAME\${shared_ext} during linking hardcodes
+# DIR into the resulting binary.
+hardcode_direct=$hardcode_direct
+
+# Set to "yes" if using DIR/libNAME\${shared_ext} during linking hardcodes
+# DIR into the resulting binary and the resulting library dependency is
+# "absolute",i.e impossible to change by setting \${shlibpath_var} if the
+# library is relocated.
+hardcode_direct_absolute=$hardcode_direct_absolute
+
+# Set to "yes" if using the -LDIR flag during linking hardcodes DIR
+# into the resulting binary.
+hardcode_minus_L=$hardcode_minus_L
+
+# Set to "yes" if using SHLIBPATH_VAR=DIR during linking hardcodes DIR
+# into the resulting binary.
+hardcode_shlibpath_var=$hardcode_shlibpath_var
+
+# Set to "yes" if building a shared library automatically hardcodes DIR
+# into the library and all subsequent libraries and executables linked
+# against it.
+hardcode_automatic=$hardcode_automatic
+
+# Set to yes if linker adds runtime paths of dependent libraries
+# to runtime path list.
+inherit_rpath=$inherit_rpath
+
+# Whether libtool must link a program against all its dependency libraries.
+link_all_deplibs=$link_all_deplibs
+
+# Fix the shell variable \$srcfile for the compiler.
+fix_srcfile_path=$lt_fix_srcfile_path
+
+# Set to "yes" if exported symbols are required.
+always_export_symbols=$always_export_symbols
+
+# The commands to list exported symbols.
+export_symbols_cmds=$lt_export_symbols_cmds
+
+# Symbols that should not be listed in the preloaded symbols.
+exclude_expsyms=$lt_exclude_expsyms
+
+# Symbols that must always be exported.
+include_expsyms=$lt_include_expsyms
+
+# Commands necessary for linking programs (against libraries) with templates.
+prelink_cmds=$lt_prelink_cmds
+
+# Specify filename containing input files.
+file_list_spec=$lt_file_list_spec
+
+# How to hardcode a shared library path into an executable.
+hardcode_action=$hardcode_action
+
+# ### END LIBTOOL CONFIG
+
+_LT_EOF
+
+ case $host_os in
+ aix3*)
+ cat <<\_LT_EOF >> "$cfgfile"
+# AIX sometimes has problems with the GCC collect2 program. For some
+# reason, if we set the COLLECT_NAMES environment variable, the problems
+# vanish in a puff of smoke.
+if test "X${COLLECT_NAMES+set}" != Xset; then
+ COLLECT_NAMES=
+ export COLLECT_NAMES
+fi
+_LT_EOF
+ ;;
+ esac
+
+
+ltmain="$ac_aux_dir/ltmain.sh"
+
+
+ # We use sed instead of cat because bash on DJGPP gets confused if
+ # if finds mixed CR/LF and LF-only lines. Since sed operates in
+ # text mode, it properly converts lines to CR/LF. This bash problem
+ # is reportedly fixed, but why not run on old versions too?
+ sed '/^# Generated shell functions inserted here/q' "$ltmain" >> "$cfgfile" \
+ || (rm -f "$cfgfile"; exit 1)
+
+ case $xsi_shell in
+ yes)
+ cat << \_LT_EOF >> "$cfgfile"
+
+# func_dirname file append nondir_replacement
+# Compute the dirname of FILE. If nonempty, add APPEND to the result,
+# otherwise set result to NONDIR_REPLACEMENT.
+func_dirname ()
+{
+ case ${1} in
+ */*) func_dirname_result="${1%/*}${2}" ;;
+ * ) func_dirname_result="${3}" ;;
+ esac
+}
+
+# func_basename file
+func_basename ()
+{
+ func_basename_result="${1##*/}"
+}
+
+# func_dirname_and_basename file append nondir_replacement
+# perform func_basename and func_dirname in a single function
+# call:
+# dirname: Compute the dirname of FILE. If nonempty,
+# add APPEND to the result, otherwise set result
+# to NONDIR_REPLACEMENT.
+# value returned in "$func_dirname_result"
+# basename: Compute filename of FILE.
+# value retuned in "$func_basename_result"
+# Implementation must be kept synchronized with func_dirname
+# and func_basename. For efficiency, we do not delegate to
+# those functions but instead duplicate the functionality here.
+func_dirname_and_basename ()
+{
+ case ${1} in
+ */*) func_dirname_result="${1%/*}${2}" ;;
+ * ) func_dirname_result="${3}" ;;
+ esac
+ func_basename_result="${1##*/}"
+}
+
+# func_stripname prefix suffix name
+# strip PREFIX and SUFFIX off of NAME.
+# PREFIX and SUFFIX must not contain globbing or regex special
+# characters, hashes, percent signs, but SUFFIX may contain a leading
+# dot (in which case that matches only a dot).
+func_stripname ()
+{
+ # pdksh 5.2.14 does not do ${X%$Y} correctly if both X and Y are
+ # positional parameters, so assign one to ordinary parameter first.
+ func_stripname_result=${3}
+ func_stripname_result=${func_stripname_result#"${1}"}
+ func_stripname_result=${func_stripname_result%"${2}"}
+}
+
+# func_opt_split
+func_opt_split ()
+{
+ func_opt_split_opt=${1%%=*}
+ func_opt_split_arg=${1#*=}
+}
+
+# func_lo2o object
+func_lo2o ()
+{
+ case ${1} in
+ *.lo) func_lo2o_result=${1%.lo}.${objext} ;;
+ *) func_lo2o_result=${1} ;;
+ esac
+}
+
+# func_xform libobj-or-source
+func_xform ()
+{
+ func_xform_result=${1%.*}.lo
+}
+
+# func_arith arithmetic-term...
+func_arith ()
+{
+ func_arith_result=$(( $* ))
+}
+
+# func_len string
+# STRING may not start with a hyphen.
+func_len ()
+{
+ func_len_result=${#1}
+}
+
+_LT_EOF
+ ;;
+ *) # Bourne compatible functions.
+ cat << \_LT_EOF >> "$cfgfile"
+
+# func_dirname file append nondir_replacement
+# Compute the dirname of FILE. If nonempty, add APPEND to the result,
+# otherwise set result to NONDIR_REPLACEMENT.
+func_dirname ()
+{
+ # Extract subdirectory from the argument.
+ func_dirname_result=`$ECHO "X${1}" | $Xsed -e "$dirname"`
+ if test "X$func_dirname_result" = "X${1}"; then
+ func_dirname_result="${3}"
+ else
+ func_dirname_result="$func_dirname_result${2}"
+ fi
+}
+
+# func_basename file
+func_basename ()
+{
+ func_basename_result=`$ECHO "X${1}" | $Xsed -e "$basename"`
+}
+
+
+# func_stripname prefix suffix name
+# strip PREFIX and SUFFIX off of NAME.
+# PREFIX and SUFFIX must not contain globbing or regex special
+# characters, hashes, percent signs, but SUFFIX may contain a leading
+# dot (in which case that matches only a dot).
+# func_strip_suffix prefix name
+func_stripname ()
+{
+ case ${2} in
+ .*) func_stripname_result=`$ECHO "X${3}" \
+ | $Xsed -e "s%^${1}%%" -e "s%\\\\${2}\$%%"`;;
+ *) func_stripname_result=`$ECHO "X${3}" \
+ | $Xsed -e "s%^${1}%%" -e "s%${2}\$%%"`;;
+ esac
+}
+
+# sed scripts:
+my_sed_long_opt='1s/^\(-[^=]*\)=.*/\1/;q'
+my_sed_long_arg='1s/^-[^=]*=//'
+
+# func_opt_split
+func_opt_split ()
+{
+ func_opt_split_opt=`$ECHO "X${1}" | $Xsed -e "$my_sed_long_opt"`
+ func_opt_split_arg=`$ECHO "X${1}" | $Xsed -e "$my_sed_long_arg"`
+}
+
+# func_lo2o object
+func_lo2o ()
+{
+ func_lo2o_result=`$ECHO "X${1}" | $Xsed -e "$lo2o"`
+}
+
+# func_xform libobj-or-source
+func_xform ()
+{
+ func_xform_result=`$ECHO "X${1}" | $Xsed -e 's/\.[^.]*$/.lo/'`
+}
+
+# func_arith arithmetic-term...
+func_arith ()
+{
+ func_arith_result=`expr "$@"`
+}
+
+# func_len string
+# STRING may not start with a hyphen.
+func_len ()
+{
+ func_len_result=`expr "$1" : ".*" 2>/dev/null || echo $max_cmd_len`
+}
+
+_LT_EOF
+esac
+
+case $lt_shell_append in
+ yes)
+ cat << \_LT_EOF >> "$cfgfile"
+
+# func_append var value
+# Append VALUE to the end of shell variable VAR.
+func_append ()
+{
+ eval "$1+=\$2"
+}
+_LT_EOF
+ ;;
+ *)
+ cat << \_LT_EOF >> "$cfgfile"
+
+# func_append var value
+# Append VALUE to the end of shell variable VAR.
+func_append ()
+{
+ eval "$1=\$$1\$2"
+}
+
+_LT_EOF
+ ;;
+ esac
+
+
+ sed -n '/^# Generated shell functions inserted here/,$p' "$ltmain" >> "$cfgfile" \
+ || (rm -f "$cfgfile"; exit 1)
+
+ mv -f "$cfgfile" "$ofile" ||
+ (rm -f "$ofile" && cp "$cfgfile" "$ofile" && rm -f "$cfgfile")
+ chmod +x "$ofile"
+
+
+as_fn_exit 0
+_LTEOF
+chmod +x "$CONFIG_LT"
+
+# configure is writing to config.log, but config.lt does its own redirection,
+# appending to config.log, which fails on DOS, as config.log is still kept
+# open by configure. Here we exec the FD to /dev/null, effectively closing
+# config.log, so it can be properly (re)opened and appended to by config.lt.
+if test "$no_create" != yes; then
+ lt_cl_success=:
+ test "$silent" = yes &&
+ lt_config_lt_args="$lt_config_lt_args --quiet"
+ exec 5>/dev/null
+ $SHELL "$CONFIG_LT" $lt_config_lt_args || lt_cl_success=false
+ exec 5>>config.log
+ $lt_cl_success || as_fn_exit 1
+fi
+
+
+
+# Check whether --with-dlopen was given.
+if test "${with_dlopen+set}" = set; then :
+ withval=$with_dlopen;
+fi
+
+
+if test "$freebsd" == "0"; then
+if test x"$with_dlopen" != x"no"; then :
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -ldl" >&5
+$as_echo_n "checking for dlopen in -ldl... " >&6; }
+if test "${ac_cv_lib_dl_dlopen+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-ldl $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char dlopen ();
+int
+main ()
+{
+return dlopen ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_dl_dlopen=yes
+else
+ ac_cv_lib_dl_dlopen=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlopen" >&5
+$as_echo "$ac_cv_lib_dl_dlopen" >&6; }
+if test "x$ac_cv_lib_dl_dlopen" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBDL 1
+_ACEOF
+
+ LIBS="-ldl $LIBS"
+
+else
+ as_fn_error $? "dlopen not found. libfabric requires libdl." "$LINENO" 5
+fi
+
+
+fi
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for pthread_mutex_init in -lpthread" >&5
+$as_echo_n "checking for pthread_mutex_init in -lpthread... " >&6; }
+if test "${ac_cv_lib_pthread_pthread_mutex_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpthread $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char pthread_mutex_init ();
+int
+main ()
+{
+return pthread_mutex_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_pthread_pthread_mutex_init=yes
+else
+ ac_cv_lib_pthread_pthread_mutex_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_pthread_pthread_mutex_init" >&5
+$as_echo "$ac_cv_lib_pthread_pthread_mutex_init" >&6; }
+if test "x$ac_cv_lib_pthread_pthread_mutex_init" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBPTHREAD 1
+_ACEOF
+
+ LIBS="-lpthread $LIBS"
+
+else
+ as_fn_error $? "pthread_mutex_init() not found. libfabric requires libpthread." "$LINENO" 5
+fi
+
+
+ac_fn_c_check_func "$LINENO" "pthread_spin_init" "ac_cv_func_pthread_spin_init"
+if test "x$ac_cv_func_pthread_spin_init" = x""yes; then :
+ have_spinlock=1
+else
+ have_spinlock=0
+fi
+
+
+
+if test "$macos" -eq 0; then
+ac_fn_c_check_func "$LINENO" "shm_open" "ac_cv_func_shm_open"
+if test "x$ac_cv_func_shm_open" = x""yes; then :
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing shm_open" >&5
+$as_echo_n "checking for library containing shm_open... " >&6; }
+if test "${ac_cv_search_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+for ac_lib in '' rt; do
+ if test -z "$ac_lib"; then
+ ac_res="none required"
+ else
+ ac_res=-l$ac_lib
+ LIBS="-l$ac_lib $ac_func_search_save_LIBS"
+ fi
+ if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_search_shm_open=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext
+ if test "${ac_cv_search_shm_open+set}" = set; then :
+ break
+fi
+done
+if test "${ac_cv_search_shm_open+set}" = set; then :
+
+else
+ ac_cv_search_shm_open=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_shm_open" >&5
+$as_echo "$ac_cv_search_shm_open" >&6; }
+ac_res=$ac_cv_search_shm_open
+if test "$ac_res" != no; then :
+ test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+
+else
+ as_fn_error $? "shm_open() not found. libfabric requires shm_open." "$LINENO" 5
+fi
+
+fi
+
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define PT_LOCK_SPIN $have_spinlock
+_ACEOF
+
+
+have_clock_gettime=0
+have_host_get_clock_service=0
+
+for ac_func in epoll_create
+do :
+ ac_fn_c_check_func "$LINENO" "epoll_create" "ac_cv_func_epoll_create"
+if test "x$ac_cv_func_epoll_create" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_EPOLL_CREATE 1
+_ACEOF
+
+fi
+done
+
+if test "$ac_cv_func_epoll_create" = yes; then
+
+$as_echo "#define HAVE_EPOLL 1" >>confdefs.h
+
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing clock_gettime" >&5
+$as_echo_n "checking for library containing clock_gettime... " >&6; }
+if test "${ac_cv_search_clock_gettime+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char clock_gettime ();
+int
+main ()
+{
+return clock_gettime ();
+ ;
+ return 0;
+}
+_ACEOF
+for ac_lib in '' rt; do
+ if test -z "$ac_lib"; then
+ ac_res="none required"
+ else
+ ac_res=-l$ac_lib
+ LIBS="-l$ac_lib $ac_func_search_save_LIBS"
+ fi
+ if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_search_clock_gettime=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext
+ if test "${ac_cv_search_clock_gettime+set}" = set; then :
+ break
+fi
+done
+if test "${ac_cv_search_clock_gettime+set}" = set; then :
+
+else
+ ac_cv_search_clock_gettime=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_clock_gettime" >&5
+$as_echo "$ac_cv_search_clock_gettime" >&6; }
+ac_res=$ac_cv_search_clock_gettime
+if test "$ac_res" != no; then :
+ test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+ have_clock_gettime=1
+else
+ for ac_func in host_get_clock_service
+do :
+ ac_fn_c_check_func "$LINENO" "host_get_clock_service" "ac_cv_func_host_get_clock_service"
+if test "x$ac_cv_func_host_get_clock_service" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_HOST_GET_CLOCK_SERVICE 1
+_ACEOF
+ have_host_get_clock_service=1
+else
+ as_fn_error $? "clock_gettime or host_get_clock_service
+ not found." "$LINENO" 5
+fi
+done
+
+fi
+
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_CLOCK_GETTIME $have_clock_gettime
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_HOST_GET_CLOCK_SERVICE $have_host_get_clock_service
+_ACEOF
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking compiler support for c11 atomics" >&5
+$as_echo_n "checking compiler support for c11 atomics... " >&6; }
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <stdatomic.h>
+int
+main ()
+{
+atomic_int a;
+ atomic_init(&a, 0);
+ #ifdef __STDC_NO_ATOMICS__
+ #error c11 atomics are not supported
+ #else
+ return 0;
+ #endif
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+$as_echo "#define HAVE_ATOMICS 1" >>confdefs.h
+
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+
+if test "$with_valgrind" != "" && test "$with_valgrind" != "no"; then
+ac_fn_c_check_header_mongrel "$LINENO" "valgrind/memcheck.h" "ac_cv_header_valgrind_memcheck_h" "$ac_includes_default"
+if test "x$ac_cv_header_valgrind_memcheck_h" = x""yes; then :
+
+else
+ as_fn_error $? "valgrind requested but <valgrind/memcheck.h> not found." "$LINENO" 5
+fi
+
+
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ld accepts --version-script" >&5
+$as_echo_n "checking whether ld accepts --version-script... " >&6; }
+if test "${ac_cv_version_script+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "`$LD --help < /dev/null 2>/dev/null | grep version-script`"; then
+ ac_cv_version_script=yes
+ else
+ ac_cv_version_script=no
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_version_script" >&5
+$as_echo "$ac_cv_version_script" >&6; }
+
+# Check whether --enable-embedded was given.
+if test "${enable_embedded+set}" = set; then :
+ enableval=$enable_embedded; ac_asm_symver_support=0
+ icc_symver_hack=1
+else
+ enable_embedded=no
+fi
+
+ if test x"$enable_embedded" = x"yes"; then
+ EMBEDDED_TRUE=
+ EMBEDDED_FALSE='#'
+else
+ EMBEDDED_TRUE='#'
+ EMBEDDED_FALSE=
+fi
+
+
+ if test "$ac_cv_version_script" = "yes"; then
+ HAVE_LD_VERSION_SCRIPT_TRUE=
+ HAVE_LD_VERSION_SCRIPT_FALSE='#'
+else
+ HAVE_LD_VERSION_SCRIPT_TRUE='#'
+ HAVE_LD_VERSION_SCRIPT_FALSE=
+fi
+
+
+case $CFLAGS in #(
+ *-ipo*) :
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: disabling symbol versioning support with -ipo CFLAG" >&5
+$as_echo "$as_me: disabling symbol versioning support with -ipo CFLAG" >&6;}
+ icc_symver_hack=1
+ ac_asm_symver_support=0
+ ;; #(
+ *) :
+
+ ;;
+esac
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for .symver assembler support" >&5
+$as_echo_n "checking for .symver assembler support... " >&6; }
+if test "$icc_symver_hack"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: disabled" >&5
+$as_echo "disabled" >&6; }
+else
+
+
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main ()
+{
+__asm__(".symver main_, main at ABIVER_1.0");
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ ac_asm_symver_support=1
+
+else
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ ac_asm_symver_support=0
+
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+
+
+fi
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_SYMVER_SUPPORT $ac_asm_symver_support
+_ACEOF
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __alias__ attribute support" >&5
+$as_echo_n "checking for __alias__ attribute support... " >&6; }
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+ int foo(int arg);
+ int foo(int arg) { return arg + 3; };
+ int foo2(int arg) __attribute__ (( __alias__("foo")));
+
+int
+main ()
+{
+ /* empty main */
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ ac_prog_cc_alias_symbols=1
+
+else
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ ac_prog_cc_alias_symbols=0
+
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_ALIAS_ATTRIBUTE $ac_prog_cc_alias_symbols
+_ACEOF
+
+for ac_func in getifaddrs
+do :
+ ac_fn_c_check_func "$LINENO" "getifaddrs" "ac_cv_func_getifaddrs"
+if test "x$ac_cv_func_getifaddrs" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_GETIFADDRS 1
+_ACEOF
+
+fi
+done
+
+
+
+ PROVIDERS_TO_BUILD=
+ PROVIDERS_DL=
+ PROVIDERS_STATIC=
+ PROVIDERS_COUNT=
+
+
+
+
+
+
+
+
+
+
+
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: *** Configuring psm provider" >&5
+$as_echo "$as_me: *** Configuring psm provider" >&6;}
+ # Check whether --enable-psm was given.
+if test "${enable_psm+set}" = set; then :
+ enableval=$enable_psm;
+else
+ enable_psm=auto
+fi
+
+
+ # Save CPPFLAGS and LDFLAGS before they are modified by FI_CHECK_PREFIX_DIR.
+ # Provider's local macros could use the value if needed.
+ # Also save LIBS, as a matter of principle.
+ psm_orig_CPPFLAGS=$CPPFLAGS
+ psm_orig_LDFLAGS=$LDFLAGS
+ psm_orig_LIBS=$LIBS
+
+ # Check the --enable-<psm> value
+ psm_dl=0
+ case $enable_psm in #(
+ yes|no) :
+ ;; #(
+ dl) :
+ enable_psm=yes psm_dl=1 ;; #(
+ auto) :
+ ;; #(
+ *) :
+
+ # Check that the base directory exists
+ if test ! -d "$enable_psm"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: supplied directory \"$enable_psm\" does not exist" >&5
+$as_echo "$as_me: WARNING: supplied directory \"$enable_psm\" does not exist" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/include exists
+ if test -d "$enable_psm/include"; then :
+ psm_PREFIX="$enable_psm"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"include\" subdirectory in supplied \"$enable_psm\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"include\" subdirectory in supplied \"$enable_psm\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/lib or base/lib64 exists
+ if test -d "$enable_psm/lib"; then :
+ psm_LIBDIR="$enable_psm/lib"
+else
+ if test -d "$enable_psm/lib64"; then :
+ psm_LIBDIR="$enable_psm/lib64"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_psm\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_psm\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+fi
+
+ enable_psm=yes
+ ;;
+esac
+
+ # Call the provider's CONFIGURE and CONDITIONALS macros
+
+
+
+
+
+ # Determine if we can support the psm provider
+ psm_happy=0
+
+ # Save CPPFLAGS and LDFLAGS because if PSM is found, we need
+ # to do additional tests which require resetting them (and
+ # then we want to set them back to their original values when
+ # done).
+ psm_CPPFLAGS_SAVE=$CPPFLAGS
+ psm_LDFLAGS_SAVE=$LDFLAGS
+
+ if test x"$enable_psm" != x"no"; then :
+
+ fi_check_package_psm_save_CPPFLAGS="$CPPFLAGS"
+ fi_check_package_psm_save_LDFLAGS="$LDFLAGS"
+ fi_check_package_psm_save_LIBS="$LIBS"
+
+ fi_check_package_psm_orig_CPPFLAGS="$psm_CPPFLAGS"
+ fi_check_package_psm_orig_LDFLAGS="$psm_LDFLAGS"
+ fi_check_package_psm_orig_LIBS="$psm_LIBS"
+
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+
+ # There's unfortunately no way to get through the progression of
+ # header includes without killing off the cache variable and
+ # trying again...
+ unset ac_cv_header_psm_h
+
+ fi_check_package_header_happy="no"
+ if test "$psm_PREFIX" = "/usr" || test "$psm_PREFIX" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for header without includes" >&5
+$as_echo "looking for header without includes" >&6; }
+ for ac_header in psm.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "psm.h" "ac_cv_header_psm_h" "$ac_includes_default"
+if test "x$ac_cv_header_psm_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_PSM_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ # no go on the as is - reset the cache and try again
+ unset ac_cv_header_psm_h
+fi
+fi
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ if test "$psm_PREFIX" != ""; then :
+ psm_CPPFLAGS="$psm_CPPFLAGS -I$psm_PREFIX/include"
+ CPPFLAGS="$CPPFLAGS -I$psm_PREFIX/include"
+fi
+ for ac_header in psm.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "psm.h" "ac_cv_header_psm_h" "$ac_includes_default"
+if test "x$ac_cv_header_psm_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_PSM_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "yes"; then :
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_psm_infinipath_psm_init
+ fi_check_package_lib_happy="no"
+ if test "$psm_LIBDIR" != ""; then :
+ # libdir was specified - search only there
+ psm_LDFLAGS="$psm_LDFLAGS -L$psm_LIBDIR"
+ LDFLAGS="$LDFLAGS -L$psm_LIBDIR"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm_init in -lpsm_infinipath" >&5
+$as_echo_n "checking for psm_init in -lpsm_infinipath... " >&6; }
+if test "${ac_cv_lib_psm_infinipath_psm_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm_infinipath $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm_init ();
+int
+main ()
+{
+return psm_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm_infinipath_psm_init=yes
+else
+ ac_cv_lib_psm_infinipath_psm_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm_infinipath_psm_init" >&5
+$as_echo "$ac_cv_lib_psm_infinipath_psm_init" >&6; }
+if test "x$ac_cv_lib_psm_infinipath_psm_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_psm_save_LDFLAGS"
+ psm_LDFLAGS="$fi_check_package_psm_orig_LDFLAGS"
+ unset ac_cv_lib_psm_infinipath_psm_init
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$psm_PREFIX"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm_init in -lpsm_infinipath" >&5
+$as_echo_n "checking for psm_init in -lpsm_infinipath... " >&6; }
+if test "${ac_cv_lib_psm_infinipath_psm_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm_infinipath $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm_init ();
+int
+main ()
+{
+return psm_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm_infinipath_psm_init=yes
+else
+ ac_cv_lib_psm_infinipath_psm_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm_infinipath_psm_init" >&5
+$as_echo "$ac_cv_lib_psm_infinipath_psm_init" >&6; }
+if test "x$ac_cv_lib_psm_infinipath_psm_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_psm_save_LDFLAGS"
+ psm_LDFLAGS="$fi_check_package_psm_orig_LDFLAGS"
+ unset ac_cv_lib_psm_infinipath_psm_init
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ psm_LDFLAGS="$psm_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm_init in -lpsm_infinipath" >&5
+$as_echo_n "checking for psm_init in -lpsm_infinipath... " >&6; }
+if test "${ac_cv_lib_psm_infinipath_psm_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm_infinipath $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm_init ();
+int
+main ()
+{
+return psm_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm_infinipath_psm_init=yes
+else
+ ac_cv_lib_psm_infinipath_psm_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm_infinipath_psm_init" >&5
+$as_echo "$ac_cv_lib_psm_infinipath_psm_init" >&6; }
+if test "x$ac_cv_lib_psm_infinipath_psm_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_psm_save_LDFLAGS"
+ psm_LDFLAGS="$fi_check_package_psm_orig_LDFLAGS"
+ unset ac_cv_lib_psm_infinipath_psm_init
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ psm_LDFLAGS="$psm_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm_init in -lpsm_infinipath" >&5
+$as_echo_n "checking for psm_init in -lpsm_infinipath... " >&6; }
+if test "${ac_cv_lib_psm_infinipath_psm_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm_infinipath $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm_init ();
+int
+main ()
+{
+return psm_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm_infinipath_psm_init=yes
+else
+ ac_cv_lib_psm_infinipath_psm_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm_infinipath_psm_init" >&5
+$as_echo "$ac_cv_lib_psm_infinipath_psm_init" >&6; }
+if test "x$ac_cv_lib_psm_infinipath_psm_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_psm_save_LDFLAGS"
+ psm_LDFLAGS="$fi_check_package_psm_orig_LDFLAGS"
+ unset ac_cv_lib_psm_infinipath_psm_init
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ psm_LIBS="-lpsm_infinipath "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+else
+ fi_check_package_happy="no"
+fi
+else
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_psm_infinipath_psm_init
+ fi_check_package_lib_happy="no"
+ if test "$psm_LIBDIR" != ""; then :
+ # libdir was specified - search only there
+ psm_LDFLAGS="$psm_LDFLAGS -L$psm_LIBDIR"
+ LDFLAGS="$LDFLAGS -L$psm_LIBDIR"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm_init in -lpsm_infinipath" >&5
+$as_echo_n "checking for psm_init in -lpsm_infinipath... " >&6; }
+if test "${ac_cv_lib_psm_infinipath_psm_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm_infinipath $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm_init ();
+int
+main ()
+{
+return psm_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm_infinipath_psm_init=yes
+else
+ ac_cv_lib_psm_infinipath_psm_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm_infinipath_psm_init" >&5
+$as_echo "$ac_cv_lib_psm_infinipath_psm_init" >&6; }
+if test "x$ac_cv_lib_psm_infinipath_psm_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_psm_save_LDFLAGS"
+ psm_LDFLAGS="$fi_check_package_psm_orig_LDFLAGS"
+ unset ac_cv_lib_psm_infinipath_psm_init
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$psm_PREFIX"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm_init in -lpsm_infinipath" >&5
+$as_echo_n "checking for psm_init in -lpsm_infinipath... " >&6; }
+if test "${ac_cv_lib_psm_infinipath_psm_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm_infinipath $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm_init ();
+int
+main ()
+{
+return psm_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm_infinipath_psm_init=yes
+else
+ ac_cv_lib_psm_infinipath_psm_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm_infinipath_psm_init" >&5
+$as_echo "$ac_cv_lib_psm_infinipath_psm_init" >&6; }
+if test "x$ac_cv_lib_psm_infinipath_psm_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_psm_save_LDFLAGS"
+ psm_LDFLAGS="$fi_check_package_psm_orig_LDFLAGS"
+ unset ac_cv_lib_psm_infinipath_psm_init
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ psm_LDFLAGS="$psm_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm_init in -lpsm_infinipath" >&5
+$as_echo_n "checking for psm_init in -lpsm_infinipath... " >&6; }
+if test "${ac_cv_lib_psm_infinipath_psm_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm_infinipath $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm_init ();
+int
+main ()
+{
+return psm_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm_infinipath_psm_init=yes
+else
+ ac_cv_lib_psm_infinipath_psm_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm_infinipath_psm_init" >&5
+$as_echo "$ac_cv_lib_psm_infinipath_psm_init" >&6; }
+if test "x$ac_cv_lib_psm_infinipath_psm_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_psm_save_LDFLAGS"
+ psm_LDFLAGS="$fi_check_package_psm_orig_LDFLAGS"
+ unset ac_cv_lib_psm_infinipath_psm_init
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ psm_LDFLAGS="$psm_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm_init in -lpsm_infinipath" >&5
+$as_echo_n "checking for psm_init in -lpsm_infinipath... " >&6; }
+if test "${ac_cv_lib_psm_infinipath_psm_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm_infinipath $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm_init ();
+int
+main ()
+{
+return psm_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm_infinipath_psm_init=yes
+else
+ ac_cv_lib_psm_infinipath_psm_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm_infinipath_psm_init" >&5
+$as_echo "$ac_cv_lib_psm_infinipath_psm_init" >&6; }
+if test "x$ac_cv_lib_psm_infinipath_psm_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_psm_save_LDFLAGS"
+ psm_LDFLAGS="$fi_check_package_psm_orig_LDFLAGS"
+ unset ac_cv_lib_psm_infinipath_psm_init
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ psm_LIBS="-lpsm_infinipath "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+fi
+ unset fi_check_package_header_happy
+
+
+
+ if test "$fi_check_package_happy" = "yes"; then :
+ psm_happy=1
+else
+ psm_CPPFLAGS="$fi_check_package_psm_orig_CPPFLAGS"
+ psm_LDFLAGS="$fi_check_package_psm_orig_LDFLAGS"
+ psm_LIBS="$fi_check_package_psm_orig_LIBS"
+ psm_happy=0
+fi
+
+
+
+
+
+ CPPFLAGS="$fi_check_package_psm_save_CPPFLAGS"
+ LDFLAGS="$fi_check_package_psm_save_LDFLAGS"
+ LIBS="$fi_check_package_psm_save_LIBS"
+
+ if test $psm_happy -eq 1; then :
+ LDFLAGS="$LDFLAGS $psm_LDFLAGS"
+ CPPFLAGS="$CPPFLAGS $psm_CPPFLAGS"
+fi
+ if test $psm_happy -eq 1; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm_am_get_source in -lpsm_infinipath" >&5
+$as_echo_n "checking for psm_am_get_source in -lpsm_infinipath... " >&6; }
+if test "${ac_cv_lib_psm_infinipath_psm_am_get_source+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm_infinipath $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm_am_get_source ();
+int
+main ()
+{
+return psm_am_get_source ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm_infinipath_psm_am_get_source=yes
+else
+ ac_cv_lib_psm_infinipath_psm_am_get_source=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm_infinipath_psm_am_get_source" >&5
+$as_echo "$ac_cv_lib_psm_infinipath_psm_am_get_source" >&6; }
+if test "x$ac_cv_lib_psm_infinipath_psm_am_get_source" = x""yes; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: PSM version is not 1.x" >&5
+$as_echo "PSM version is not 1.x" >&6; }; psm_happy=0
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: PSM version is 1.x" >&5
+$as_echo "PSM version is 1.x" >&6; }
+fi
+
+
+fi
+ if test $psm_happy -eq 1; then :
+ ac_fn_c_check_type "$LINENO" "psm_epconn_t" "ac_cv_type_psm_epconn_t" "#include <psm.h>
+"
+if test "x$ac_cv_type_psm_epconn_t" = x""yes; then :
+
+else
+ psm_happy=0
+fi
+
+fi
+
+fi
+
+ if test $psm_happy -eq 1; then :
+ psm_happy=1
+else
+ psm_happy=0
+fi
+
+ # Reset CPPFLAGS and LDFLAGS back to their original values and
+ # tidy up
+ CPPFLAGS=$psm_CPPFLAGS_SAVE
+ unset psm_CPPFLAGS_SAVE
+ LDFLAGS=$psm_LDFLAGS_SAVE
+ unset psm_LDFLAGS_SAVE
+
+
+
+
+
+
+
+ # See if the provider configured successfully
+ if test $psm_happy -eq 1; then :
+ PROVIDERS_TO_BUILD="$PROVIDERS_TO_BUILD psm"
+ PROVIDERS_COUNT=$((PROVIDERS_COUNT+1))
+
+ FI_PC_CFLAGS="$FI_PC_CFLAGS $psm_CFLAGS $psm_CPPFLAGS"
+ FI_PC_PRIVATE_LIBS="$FI_PC_PRIVATE_LIBS $psm_LDFLAGS $psm_LIBS"
+
+ if test $psm_dl -eq 1; then :
+ PROVIDERS_DL="prov/psm/libpsm.la $PROVIDERS_DL"
+ if test x"$enable_static" = x"yes" &&
+ test x"$enable_shared" = x"no"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: psm provider was selected to be built as DL" >&5
+$as_echo "$as_me: WARNING: psm provider was selected to be built as DL" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: but libfabric is being built as static-only" >&5
+$as_echo "$as_me: WARNING: but libfabric is being built as static-only" >&2;}
+ as_fn_error $? "This is an impossible situation. Cannot continue." "$LINENO" 5
+fi
+ { $as_echo "$as_me:${as_lineno-$LINENO}: psm provider: build as plugin" >&5
+$as_echo "$as_me: psm provider: build as plugin" >&6;}
+
+else
+ PROVIDERS_STATIC="prov/psm/libpsm.la $PROVIDERS_STATIC"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: psm provider: include in libfabric" >&5
+$as_echo "$as_me: psm provider: include in libfabric" >&6;}
+fi
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: psm provider: disabled" >&5
+$as_echo "$as_me: psm provider: disabled" >&6;}
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_PSM $psm_happy
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_PSM_DL $psm_dl
+_ACEOF
+
+
+ # Set AM conditionals for HAVE_<provider> and HAVE_<provider>_DL
+ # as well as AC defines
+ if test $psm_happy -eq 1; then
+ HAVE_PSM_TRUE=
+ HAVE_PSM_FALSE='#'
+else
+ HAVE_PSM_TRUE='#'
+ HAVE_PSM_FALSE=
+fi
+
+ if test $psm_dl -eq 1; then
+ HAVE_PSM_DL_TRUE=
+ HAVE_PSM_DL_FALSE='#'
+else
+ HAVE_PSM_DL_TRUE='#'
+ HAVE_PSM_DL_FALSE=
+fi
+
+
+ # If this provider was specifically requested but we can't
+ # build it, error.
+ if test "$enable_psm $psm_happy" = "yes 0"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: psm provider was requested, but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: psm provider was requested, but cannot be compiled" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+ # If this provider was requested for direct build, ensure that
+ # provider's fi_direct.h exists in tree. Error otherwise.
+ if test x"$enable_direct" = x"psm"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $srcdir/prov/psm/include/rdma/fi_direct.h" >&5
+$as_echo_n "checking for $srcdir/prov/psm/include/rdma/fi_direct.h... " >&6; }
+ if test -f "$srcdir/prov/psm/include/rdma/fi_direct.h"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ as_fn_error $? "psm provider was requested as direct, but is missing fi_direct.h" "$LINENO" 5
+
+fi
+fi
+
+ # Restore CPPFLAGS/LDFLAGS/LIBS
+ CPPFLAGS=$psm_orig_CPPFLAGS
+ unset psm_orig_CPPFLAGS
+ LDFLAGS=$psm_orig_LDFLAGS
+ unset psm_orig_LDFLAGS
+ LIBS=$psm_orig_LIBS
+ unset psm_orig_LIBS
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: *** Configuring psm2 provider" >&5
+$as_echo "$as_me: *** Configuring psm2 provider" >&6;}
+ # Check whether --enable-psm2 was given.
+if test "${enable_psm2+set}" = set; then :
+ enableval=$enable_psm2;
+else
+ enable_psm2=auto
+fi
+
+
+ # Save CPPFLAGS and LDFLAGS before they are modified by FI_CHECK_PREFIX_DIR.
+ # Provider's local macros could use the value if needed.
+ # Also save LIBS, as a matter of principle.
+ psm2_orig_CPPFLAGS=$CPPFLAGS
+ psm2_orig_LDFLAGS=$LDFLAGS
+ psm2_orig_LIBS=$LIBS
+
+ # Check the --enable-<psm2> value
+ psm2_dl=0
+ case $enable_psm2 in #(
+ yes|no) :
+ ;; #(
+ dl) :
+ enable_psm2=yes psm2_dl=1 ;; #(
+ auto) :
+ ;; #(
+ *) :
+
+ # Check that the base directory exists
+ if test ! -d "$enable_psm2"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: supplied directory \"$enable_psm2\" does not exist" >&5
+$as_echo "$as_me: WARNING: supplied directory \"$enable_psm2\" does not exist" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/include exists
+ if test -d "$enable_psm2/include"; then :
+ psm2_PREFIX="$enable_psm2"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"include\" subdirectory in supplied \"$enable_psm2\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"include\" subdirectory in supplied \"$enable_psm2\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/lib or base/lib64 exists
+ if test -d "$enable_psm2/lib"; then :
+ psm2_LIBDIR="$enable_psm2/lib"
+else
+ if test -d "$enable_psm2/lib64"; then :
+ psm2_LIBDIR="$enable_psm2/lib64"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_psm2\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_psm2\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+fi
+
+ enable_psm2=yes
+ ;;
+esac
+
+ # Call the provider's CONFIGURE and CONDITIONALS macros
+
+
+
+
+
+
+ # Determine if we can support the psm2 provider
+ psm2_happy=0
+ if test x"$enable_psm2" != x"no"; then :
+
+ fi_check_package_psm2_save_CPPFLAGS="$CPPFLAGS"
+ fi_check_package_psm2_save_LDFLAGS="$LDFLAGS"
+ fi_check_package_psm2_save_LIBS="$LIBS"
+
+ fi_check_package_psm2_orig_CPPFLAGS="$psm2_CPPFLAGS"
+ fi_check_package_psm2_orig_LDFLAGS="$psm2_LDFLAGS"
+ fi_check_package_psm2_orig_LIBS="$psm2_LIBS"
+
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+
+ # There's unfortunately no way to get through the progression of
+ # header includes without killing off the cache variable and
+ # trying again...
+ unset ac_cv_header_psm2_h
+
+ fi_check_package_header_happy="no"
+ if test "$psm2_PREFIX" = "/usr" || test "$psm2_PREFIX" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for header without includes" >&5
+$as_echo "looking for header without includes" >&6; }
+ for ac_header in psm2.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "psm2.h" "ac_cv_header_psm2_h" "$ac_includes_default"
+if test "x$ac_cv_header_psm2_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_PSM2_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ # no go on the as is - reset the cache and try again
+ unset ac_cv_header_psm2_h
+fi
+fi
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ if test "$psm2_PREFIX" != ""; then :
+ psm2_CPPFLAGS="$psm2_CPPFLAGS -I$psm2_PREFIX/include"
+ CPPFLAGS="$CPPFLAGS -I$psm2_PREFIX/include"
+fi
+ for ac_header in psm2.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "psm2.h" "ac_cv_header_psm2_h" "$ac_includes_default"
+if test "x$ac_cv_header_psm2_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_PSM2_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "yes"; then :
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_psm2_psm2_init
+ fi_check_package_lib_happy="no"
+ if test "$psm2_LIBDIR" != ""; then :
+ # libdir was specified - search only there
+ psm2_LDFLAGS="$psm2_LDFLAGS -L$psm2_LIBDIR"
+ LDFLAGS="$LDFLAGS -L$psm2_LIBDIR"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm2_init in -lpsm2" >&5
+$as_echo_n "checking for psm2_init in -lpsm2... " >&6; }
+if test "${ac_cv_lib_psm2_psm2_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm2 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm2_init ();
+int
+main ()
+{
+return psm2_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm2_psm2_init=yes
+else
+ ac_cv_lib_psm2_psm2_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm2_psm2_init" >&5
+$as_echo "$ac_cv_lib_psm2_psm2_init" >&6; }
+if test "x$ac_cv_lib_psm2_psm2_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_psm2_save_LDFLAGS"
+ psm2_LDFLAGS="$fi_check_package_psm2_orig_LDFLAGS"
+ unset ac_cv_lib_psm2_psm2_init
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$psm2_PREFIX"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm2_init in -lpsm2" >&5
+$as_echo_n "checking for psm2_init in -lpsm2... " >&6; }
+if test "${ac_cv_lib_psm2_psm2_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm2 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm2_init ();
+int
+main ()
+{
+return psm2_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm2_psm2_init=yes
+else
+ ac_cv_lib_psm2_psm2_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm2_psm2_init" >&5
+$as_echo "$ac_cv_lib_psm2_psm2_init" >&6; }
+if test "x$ac_cv_lib_psm2_psm2_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_psm2_save_LDFLAGS"
+ psm2_LDFLAGS="$fi_check_package_psm2_orig_LDFLAGS"
+ unset ac_cv_lib_psm2_psm2_init
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ psm2_LDFLAGS="$psm2_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm2_init in -lpsm2" >&5
+$as_echo_n "checking for psm2_init in -lpsm2... " >&6; }
+if test "${ac_cv_lib_psm2_psm2_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm2 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm2_init ();
+int
+main ()
+{
+return psm2_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm2_psm2_init=yes
+else
+ ac_cv_lib_psm2_psm2_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm2_psm2_init" >&5
+$as_echo "$ac_cv_lib_psm2_psm2_init" >&6; }
+if test "x$ac_cv_lib_psm2_psm2_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_psm2_save_LDFLAGS"
+ psm2_LDFLAGS="$fi_check_package_psm2_orig_LDFLAGS"
+ unset ac_cv_lib_psm2_psm2_init
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ psm2_LDFLAGS="$psm2_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm2_init in -lpsm2" >&5
+$as_echo_n "checking for psm2_init in -lpsm2... " >&6; }
+if test "${ac_cv_lib_psm2_psm2_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm2 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm2_init ();
+int
+main ()
+{
+return psm2_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm2_psm2_init=yes
+else
+ ac_cv_lib_psm2_psm2_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm2_psm2_init" >&5
+$as_echo "$ac_cv_lib_psm2_psm2_init" >&6; }
+if test "x$ac_cv_lib_psm2_psm2_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_psm2_save_LDFLAGS"
+ psm2_LDFLAGS="$fi_check_package_psm2_orig_LDFLAGS"
+ unset ac_cv_lib_psm2_psm2_init
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ psm2_LIBS="-lpsm2 "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+else
+ fi_check_package_happy="no"
+fi
+else
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_psm2_psm2_init
+ fi_check_package_lib_happy="no"
+ if test "$psm2_LIBDIR" != ""; then :
+ # libdir was specified - search only there
+ psm2_LDFLAGS="$psm2_LDFLAGS -L$psm2_LIBDIR"
+ LDFLAGS="$LDFLAGS -L$psm2_LIBDIR"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm2_init in -lpsm2" >&5
+$as_echo_n "checking for psm2_init in -lpsm2... " >&6; }
+if test "${ac_cv_lib_psm2_psm2_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm2 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm2_init ();
+int
+main ()
+{
+return psm2_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm2_psm2_init=yes
+else
+ ac_cv_lib_psm2_psm2_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm2_psm2_init" >&5
+$as_echo "$ac_cv_lib_psm2_psm2_init" >&6; }
+if test "x$ac_cv_lib_psm2_psm2_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_psm2_save_LDFLAGS"
+ psm2_LDFLAGS="$fi_check_package_psm2_orig_LDFLAGS"
+ unset ac_cv_lib_psm2_psm2_init
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$psm2_PREFIX"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm2_init in -lpsm2" >&5
+$as_echo_n "checking for psm2_init in -lpsm2... " >&6; }
+if test "${ac_cv_lib_psm2_psm2_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm2 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm2_init ();
+int
+main ()
+{
+return psm2_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm2_psm2_init=yes
+else
+ ac_cv_lib_psm2_psm2_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm2_psm2_init" >&5
+$as_echo "$ac_cv_lib_psm2_psm2_init" >&6; }
+if test "x$ac_cv_lib_psm2_psm2_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_psm2_save_LDFLAGS"
+ psm2_LDFLAGS="$fi_check_package_psm2_orig_LDFLAGS"
+ unset ac_cv_lib_psm2_psm2_init
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ psm2_LDFLAGS="$psm2_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm2_init in -lpsm2" >&5
+$as_echo_n "checking for psm2_init in -lpsm2... " >&6; }
+if test "${ac_cv_lib_psm2_psm2_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm2 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm2_init ();
+int
+main ()
+{
+return psm2_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm2_psm2_init=yes
+else
+ ac_cv_lib_psm2_psm2_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm2_psm2_init" >&5
+$as_echo "$ac_cv_lib_psm2_psm2_init" >&6; }
+if test "x$ac_cv_lib_psm2_psm2_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_psm2_save_LDFLAGS"
+ psm2_LDFLAGS="$fi_check_package_psm2_orig_LDFLAGS"
+ unset ac_cv_lib_psm2_psm2_init
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ psm2_LDFLAGS="$psm2_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for psm2_init in -lpsm2" >&5
+$as_echo_n "checking for psm2_init in -lpsm2... " >&6; }
+if test "${ac_cv_lib_psm2_psm2_init+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lpsm2 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char psm2_init ();
+int
+main ()
+{
+return psm2_init ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_psm2_psm2_init=yes
+else
+ ac_cv_lib_psm2_psm2_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_psm2_psm2_init" >&5
+$as_echo "$ac_cv_lib_psm2_psm2_init" >&6; }
+if test "x$ac_cv_lib_psm2_psm2_init" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_psm2_save_LDFLAGS"
+ psm2_LDFLAGS="$fi_check_package_psm2_orig_LDFLAGS"
+ unset ac_cv_lib_psm2_psm2_init
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ psm2_LIBS="-lpsm2 "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+fi
+ unset fi_check_package_header_happy
+
+
+
+ if test "$fi_check_package_happy" = "yes"; then :
+ psm2_happy=1
+else
+ psm2_CPPFLAGS="$fi_check_package_psm2_orig_CPPFLAGS"
+ psm2_LDFLAGS="$fi_check_package_psm2_orig_LDFLAGS"
+ psm2_LIBS="$fi_check_package_psm2_orig_LIBS"
+ psm2_happy=0
+fi
+
+
+
+
+
+ CPPFLAGS="$fi_check_package_psm2_save_CPPFLAGS"
+ LDFLAGS="$fi_check_package_psm2_save_LDFLAGS"
+ LIBS="$fi_check_package_psm2_save_LIBS"
+
+
+fi
+
+ if test $psm2_happy -eq 1; then :
+ psm2_happy=1
+else
+ psm2_happy=0
+fi
+
+
+
+
+
+
+
+ # See if the provider configured successfully
+ if test $psm2_happy -eq 1; then :
+ PROVIDERS_TO_BUILD="$PROVIDERS_TO_BUILD psm2"
+ PROVIDERS_COUNT=$((PROVIDERS_COUNT+1))
+
+ FI_PC_CFLAGS="$FI_PC_CFLAGS $psm2_CFLAGS $psm2_CPPFLAGS"
+ FI_PC_PRIVATE_LIBS="$FI_PC_PRIVATE_LIBS $psm2_LDFLAGS $psm2_LIBS"
+
+ if test $psm2_dl -eq 1; then :
+ PROVIDERS_DL="prov/psm2/libpsm2.la $PROVIDERS_DL"
+ if test x"$enable_static" = x"yes" &&
+ test x"$enable_shared" = x"no"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: psm2 provider was selected to be built as DL" >&5
+$as_echo "$as_me: WARNING: psm2 provider was selected to be built as DL" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: but libfabric is being built as static-only" >&5
+$as_echo "$as_me: WARNING: but libfabric is being built as static-only" >&2;}
+ as_fn_error $? "This is an impossible situation. Cannot continue." "$LINENO" 5
+fi
+ { $as_echo "$as_me:${as_lineno-$LINENO}: psm2 provider: build as plugin" >&5
+$as_echo "$as_me: psm2 provider: build as plugin" >&6;}
+
+else
+ PROVIDERS_STATIC="prov/psm2/libpsm2.la $PROVIDERS_STATIC"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: psm2 provider: include in libfabric" >&5
+$as_echo "$as_me: psm2 provider: include in libfabric" >&6;}
+fi
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: psm2 provider: disabled" >&5
+$as_echo "$as_me: psm2 provider: disabled" >&6;}
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_PSM2 $psm2_happy
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_PSM2_DL $psm2_dl
+_ACEOF
+
+
+ # Set AM conditionals for HAVE_<provider> and HAVE_<provider>_DL
+ # as well as AC defines
+ if test $psm2_happy -eq 1; then
+ HAVE_PSM2_TRUE=
+ HAVE_PSM2_FALSE='#'
+else
+ HAVE_PSM2_TRUE='#'
+ HAVE_PSM2_FALSE=
+fi
+
+ if test $psm2_dl -eq 1; then
+ HAVE_PSM2_DL_TRUE=
+ HAVE_PSM2_DL_FALSE='#'
+else
+ HAVE_PSM2_DL_TRUE='#'
+ HAVE_PSM2_DL_FALSE=
+fi
+
+
+ # If this provider was specifically requested but we can't
+ # build it, error.
+ if test "$enable_psm2 $psm2_happy" = "yes 0"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: psm2 provider was requested, but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: psm2 provider was requested, but cannot be compiled" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+ # If this provider was requested for direct build, ensure that
+ # provider's fi_direct.h exists in tree. Error otherwise.
+ if test x"$enable_direct" = x"psm2"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $srcdir/prov/psm2/include/rdma/fi_direct.h" >&5
+$as_echo_n "checking for $srcdir/prov/psm2/include/rdma/fi_direct.h... " >&6; }
+ if test -f "$srcdir/prov/psm2/include/rdma/fi_direct.h"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ as_fn_error $? "psm2 provider was requested as direct, but is missing fi_direct.h" "$LINENO" 5
+
+fi
+fi
+
+ # Restore CPPFLAGS/LDFLAGS/LIBS
+ CPPFLAGS=$psm2_orig_CPPFLAGS
+ unset psm2_orig_CPPFLAGS
+ LDFLAGS=$psm2_orig_LDFLAGS
+ unset psm2_orig_LDFLAGS
+ LIBS=$psm2_orig_LIBS
+ unset psm2_orig_LIBS
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: *** Configuring sockets provider" >&5
+$as_echo "$as_me: *** Configuring sockets provider" >&6;}
+ # Check whether --enable-sockets was given.
+if test "${enable_sockets+set}" = set; then :
+ enableval=$enable_sockets;
+else
+ enable_sockets=auto
+fi
+
+
+ # Save CPPFLAGS and LDFLAGS before they are modified by FI_CHECK_PREFIX_DIR.
+ # Provider's local macros could use the value if needed.
+ # Also save LIBS, as a matter of principle.
+ sockets_orig_CPPFLAGS=$CPPFLAGS
+ sockets_orig_LDFLAGS=$LDFLAGS
+ sockets_orig_LIBS=$LIBS
+
+ # Check the --enable-<sockets> value
+ sockets_dl=0
+ case $enable_sockets in #(
+ yes|no) :
+ ;; #(
+ dl) :
+ enable_sockets=yes sockets_dl=1 ;; #(
+ auto) :
+ ;; #(
+ *) :
+
+ # Check that the base directory exists
+ if test ! -d "$enable_sockets"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: supplied directory \"$enable_sockets\" does not exist" >&5
+$as_echo "$as_me: WARNING: supplied directory \"$enable_sockets\" does not exist" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/include exists
+ if test -d "$enable_sockets/include"; then :
+ sockets_PREFIX="$enable_sockets"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"include\" subdirectory in supplied \"$enable_sockets\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"include\" subdirectory in supplied \"$enable_sockets\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/lib or base/lib64 exists
+ if test -d "$enable_sockets/lib"; then :
+ sockets_LIBDIR="$enable_sockets/lib"
+else
+ if test -d "$enable_sockets/lib64"; then :
+ sockets_LIBDIR="$enable_sockets/lib64"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_sockets\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_sockets\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+fi
+
+ enable_sockets=yes
+ ;;
+esac
+
+ # Call the provider's CONFIGURE and CONDITIONALS macros
+
+
+
+
+
+ # Determine if we can support the sockets provider
+ sockets_h_happy=0
+ sockets_shm_happy=0
+ if test x"$enable_sockets" != x"no"; then :
+ ac_fn_c_check_header_mongrel "$LINENO" "sys/socket.h" "ac_cv_header_sys_socket_h" "$ac_includes_default"
+if test "x$ac_cv_header_sys_socket_h" = x""yes; then :
+ sockets_h_happy=1
+else
+ sockets_h_happy=0
+fi
+
+
+
+
+ # check if shm_open is already present
+ ac_fn_c_check_func "$LINENO" "shm_open" "ac_cv_func_shm_open"
+if test "x$ac_cv_func_shm_open" = x""yes; then :
+ sockets_shm_happy=1
+else
+ sockets_shm_happy=0
+fi
+
+
+ # look for shm_open in librt if not already present
+ if test $sockets_shm_happy -eq 0; then :
+
+ fi_check_package_sockets_save_CPPFLAGS="$CPPFLAGS"
+ fi_check_package_sockets_save_LDFLAGS="$LDFLAGS"
+ fi_check_package_sockets_save_LIBS="$LIBS"
+
+ fi_check_package_sockets_orig_CPPFLAGS="$sockets_CPPFLAGS"
+ fi_check_package_sockets_orig_LDFLAGS="$sockets_LDFLAGS"
+ fi_check_package_sockets_orig_LIBS="$sockets_LIBS"
+
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+
+ # There's unfortunately no way to get through the progression of
+ # header includes without killing off the cache variable and
+ # trying again...
+ unset ac_cv_header_sys_mman_h
+
+ fi_check_package_header_happy="no"
+ if test "" = "/usr" || test "" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for header without includes" >&5
+$as_echo "looking for header without includes" >&6; }
+ for ac_header in sys/mman.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "sys/mman.h" "ac_cv_header_sys_mman_h" "$ac_includes_default"
+if test "x$ac_cv_header_sys_mman_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_SYS_MMAN_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ # no go on the as is - reset the cache and try again
+ unset ac_cv_header_sys_mman_h
+fi
+fi
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ if test "" != ""; then :
+ sockets_CPPFLAGS="$sockets_CPPFLAGS -I/include"
+ CPPFLAGS="$CPPFLAGS -I/include"
+fi
+ for ac_header in sys/mman.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "sys/mman.h" "ac_cv_header_sys_mman_h" "$ac_includes_default"
+if test "x$ac_cv_header_sys_mman_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_SYS_MMAN_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "yes"; then :
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_rt_shm_open
+ fi_check_package_lib_happy="no"
+ if test "" != ""; then :
+ # libdir was specified - search only there
+ sockets_LDFLAGS="$sockets_LDFLAGS -L"
+ LDFLAGS="$LDFLAGS -L"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_sockets_save_LDFLAGS"
+ sockets_LDFLAGS="$fi_check_package_sockets_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir=""
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_sockets_save_LDFLAGS"
+ sockets_LDFLAGS="$fi_check_package_sockets_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ sockets_LDFLAGS="$sockets_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_sockets_save_LDFLAGS"
+ sockets_LDFLAGS="$fi_check_package_sockets_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ sockets_LDFLAGS="$sockets_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_sockets_save_LDFLAGS"
+ sockets_LDFLAGS="$fi_check_package_sockets_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ sockets_LIBS="-lrt "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+else
+ fi_check_package_happy="no"
+fi
+else
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_rt_shm_open
+ fi_check_package_lib_happy="no"
+ if test "" != ""; then :
+ # libdir was specified - search only there
+ sockets_LDFLAGS="$sockets_LDFLAGS -L"
+ LDFLAGS="$LDFLAGS -L"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_sockets_save_LDFLAGS"
+ sockets_LDFLAGS="$fi_check_package_sockets_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir=""
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_sockets_save_LDFLAGS"
+ sockets_LDFLAGS="$fi_check_package_sockets_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ sockets_LDFLAGS="$sockets_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_sockets_save_LDFLAGS"
+ sockets_LDFLAGS="$fi_check_package_sockets_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ sockets_LDFLAGS="$sockets_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_sockets_save_LDFLAGS"
+ sockets_LDFLAGS="$fi_check_package_sockets_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ sockets_LIBS="-lrt "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+fi
+ unset fi_check_package_header_happy
+
+
+
+ if test "$fi_check_package_happy" = "yes"; then :
+ sockets_shm_happy=1
+else
+ sockets_CPPFLAGS="$fi_check_package_sockets_orig_CPPFLAGS"
+ sockets_LDFLAGS="$fi_check_package_sockets_orig_LDFLAGS"
+ sockets_LIBS="$fi_check_package_sockets_orig_LIBS"
+ sockets_shm_happy=0
+fi
+
+
+
+
+
+ CPPFLAGS="$fi_check_package_sockets_save_CPPFLAGS"
+ LDFLAGS="$fi_check_package_sockets_save_LDFLAGS"
+ LIBS="$fi_check_package_sockets_save_LIBS"
+
+fi
+
+fi
+
+ for ac_func in getifaddrs
+do :
+ ac_fn_c_check_func "$LINENO" "getifaddrs" "ac_cv_func_getifaddrs"
+if test "x$ac_cv_func_getifaddrs" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_GETIFADDRS 1
+_ACEOF
+
+fi
+done
+
+
+ if test $sockets_h_happy -eq 1 && \
+ test $sockets_shm_happy -eq 1; then :
+ sockets_happy=1
+else
+ sockets_happy=0
+fi
+
+
+
+
+
+
+
+ # See if the provider configured successfully
+ if test $sockets_happy -eq 1; then :
+ PROVIDERS_TO_BUILD="$PROVIDERS_TO_BUILD sockets"
+ PROVIDERS_COUNT=$((PROVIDERS_COUNT+1))
+
+ FI_PC_CFLAGS="$FI_PC_CFLAGS $sockets_CFLAGS $sockets_CPPFLAGS"
+ FI_PC_PRIVATE_LIBS="$FI_PC_PRIVATE_LIBS $sockets_LDFLAGS $sockets_LIBS"
+
+ if test $sockets_dl -eq 1; then :
+ PROVIDERS_DL="prov/sockets/libsockets.la $PROVIDERS_DL"
+ if test x"$enable_static" = x"yes" &&
+ test x"$enable_shared" = x"no"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: sockets provider was selected to be built as DL" >&5
+$as_echo "$as_me: WARNING: sockets provider was selected to be built as DL" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: but libfabric is being built as static-only" >&5
+$as_echo "$as_me: WARNING: but libfabric is being built as static-only" >&2;}
+ as_fn_error $? "This is an impossible situation. Cannot continue." "$LINENO" 5
+fi
+ { $as_echo "$as_me:${as_lineno-$LINENO}: sockets provider: build as plugin" >&5
+$as_echo "$as_me: sockets provider: build as plugin" >&6;}
+
+else
+ PROVIDERS_STATIC="prov/sockets/libsockets.la $PROVIDERS_STATIC"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: sockets provider: include in libfabric" >&5
+$as_echo "$as_me: sockets provider: include in libfabric" >&6;}
+fi
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: sockets provider: disabled" >&5
+$as_echo "$as_me: sockets provider: disabled" >&6;}
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_SOCKETS $sockets_happy
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_SOCKETS_DL $sockets_dl
+_ACEOF
+
+
+ # Set AM conditionals for HAVE_<provider> and HAVE_<provider>_DL
+ # as well as AC defines
+ if test $sockets_happy -eq 1; then
+ HAVE_SOCKETS_TRUE=
+ HAVE_SOCKETS_FALSE='#'
+else
+ HAVE_SOCKETS_TRUE='#'
+ HAVE_SOCKETS_FALSE=
+fi
+
+ if test $sockets_dl -eq 1; then
+ HAVE_SOCKETS_DL_TRUE=
+ HAVE_SOCKETS_DL_FALSE='#'
+else
+ HAVE_SOCKETS_DL_TRUE='#'
+ HAVE_SOCKETS_DL_FALSE=
+fi
+
+
+ # If this provider was specifically requested but we can't
+ # build it, error.
+ if test "$enable_sockets $sockets_happy" = "yes 0"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: sockets provider was requested, but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: sockets provider was requested, but cannot be compiled" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+ # If this provider was requested for direct build, ensure that
+ # provider's fi_direct.h exists in tree. Error otherwise.
+ if test x"$enable_direct" = x"sockets"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $srcdir/prov/sockets/include/rdma/fi_direct.h" >&5
+$as_echo_n "checking for $srcdir/prov/sockets/include/rdma/fi_direct.h... " >&6; }
+ if test -f "$srcdir/prov/sockets/include/rdma/fi_direct.h"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ as_fn_error $? "sockets provider was requested as direct, but is missing fi_direct.h" "$LINENO" 5
+
+fi
+fi
+
+ # Restore CPPFLAGS/LDFLAGS/LIBS
+ CPPFLAGS=$sockets_orig_CPPFLAGS
+ unset sockets_orig_CPPFLAGS
+ LDFLAGS=$sockets_orig_LDFLAGS
+ unset sockets_orig_LDFLAGS
+ LIBS=$sockets_orig_LIBS
+ unset sockets_orig_LIBS
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: *** Configuring verbs provider" >&5
+$as_echo "$as_me: *** Configuring verbs provider" >&6;}
+ # Check whether --enable-verbs was given.
+if test "${enable_verbs+set}" = set; then :
+ enableval=$enable_verbs;
+else
+ enable_verbs=auto
+fi
+
+
+ # Save CPPFLAGS and LDFLAGS before they are modified by FI_CHECK_PREFIX_DIR.
+ # Provider's local macros could use the value if needed.
+ # Also save LIBS, as a matter of principle.
+ verbs_orig_CPPFLAGS=$CPPFLAGS
+ verbs_orig_LDFLAGS=$LDFLAGS
+ verbs_orig_LIBS=$LIBS
+
+ # Check the --enable-<verbs> value
+ verbs_dl=0
+ case $enable_verbs in #(
+ yes|no) :
+ ;; #(
+ dl) :
+ enable_verbs=yes verbs_dl=1 ;; #(
+ auto) :
+ ;; #(
+ *) :
+
+ # Check that the base directory exists
+ if test ! -d "$enable_verbs"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: supplied directory \"$enable_verbs\" does not exist" >&5
+$as_echo "$as_me: WARNING: supplied directory \"$enable_verbs\" does not exist" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/include exists
+ if test -d "$enable_verbs/include"; then :
+ verbs_PREFIX="$enable_verbs"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"include\" subdirectory in supplied \"$enable_verbs\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"include\" subdirectory in supplied \"$enable_verbs\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/lib or base/lib64 exists
+ if test -d "$enable_verbs/lib"; then :
+ verbs_LIBDIR="$enable_verbs/lib"
+else
+ if test -d "$enable_verbs/lib64"; then :
+ verbs_LIBDIR="$enable_verbs/lib64"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_verbs\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_verbs\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+fi
+
+ enable_verbs=yes
+ ;;
+esac
+
+ # Call the provider's CONFIGURE and CONDITIONALS macros
+
+
+
+
+
+
+
+ # Determine if we can support the verbs provider
+ verbs_ibverbs_happy=0
+ verbs_rdmacm_happy=0
+ if test x"$enable_verbs" != x"no"; then :
+
+ fi_check_package_verbs_ibverbs_save_CPPFLAGS="$CPPFLAGS"
+ fi_check_package_verbs_ibverbs_save_LDFLAGS="$LDFLAGS"
+ fi_check_package_verbs_ibverbs_save_LIBS="$LIBS"
+
+ fi_check_package_verbs_ibverbs_orig_CPPFLAGS="$verbs_ibverbs_CPPFLAGS"
+ fi_check_package_verbs_ibverbs_orig_LDFLAGS="$verbs_ibverbs_LDFLAGS"
+ fi_check_package_verbs_ibverbs_orig_LIBS="$verbs_ibverbs_LIBS"
+
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+
+ # There's unfortunately no way to get through the progression of
+ # header includes without killing off the cache variable and
+ # trying again...
+ unset ac_cv_header_infiniband_verbs_h
+
+ fi_check_package_header_happy="no"
+ if test "$verbs_PREFIX" = "/usr" || test "$verbs_PREFIX" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for header without includes" >&5
+$as_echo "looking for header without includes" >&6; }
+ for ac_header in infiniband/verbs.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "infiniband/verbs.h" "ac_cv_header_infiniband_verbs_h" "$ac_includes_default"
+if test "x$ac_cv_header_infiniband_verbs_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_INFINIBAND_VERBS_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ # no go on the as is - reset the cache and try again
+ unset ac_cv_header_infiniband_verbs_h
+fi
+fi
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ if test "$verbs_PREFIX" != ""; then :
+ verbs_ibverbs_CPPFLAGS="$verbs_ibverbs_CPPFLAGS -I$verbs_PREFIX/include"
+ CPPFLAGS="$CPPFLAGS -I$verbs_PREFIX/include"
+fi
+ for ac_header in infiniband/verbs.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "infiniband/verbs.h" "ac_cv_header_infiniband_verbs_h" "$ac_includes_default"
+if test "x$ac_cv_header_infiniband_verbs_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_INFINIBAND_VERBS_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "yes"; then :
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_ibverbs_ibv_open_device
+ fi_check_package_lib_happy="no"
+ if test "$verbs_LIBDIR" != ""; then :
+ # libdir was specified - search only there
+ verbs_ibverbs_LDFLAGS="$verbs_ibverbs_LDFLAGS -L$verbs_LIBDIR"
+ LDFLAGS="$LDFLAGS -L$verbs_LIBDIR"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ibv_open_device in -libverbs" >&5
+$as_echo_n "checking for ibv_open_device in -libverbs... " >&6; }
+if test "${ac_cv_lib_ibverbs_ibv_open_device+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-libverbs $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ibv_open_device ();
+int
+main ()
+{
+return ibv_open_device ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_ibverbs_ibv_open_device=yes
+else
+ ac_cv_lib_ibverbs_ibv_open_device=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ibverbs_ibv_open_device" >&5
+$as_echo "$ac_cv_lib_ibverbs_ibv_open_device" >&6; }
+if test "x$ac_cv_lib_ibverbs_ibv_open_device" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_verbs_ibverbs_save_LDFLAGS"
+ verbs_ibverbs_LDFLAGS="$fi_check_package_verbs_ibverbs_orig_LDFLAGS"
+ unset ac_cv_lib_ibverbs_ibv_open_device
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$verbs_PREFIX"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ibv_open_device in -libverbs" >&5
+$as_echo_n "checking for ibv_open_device in -libverbs... " >&6; }
+if test "${ac_cv_lib_ibverbs_ibv_open_device+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-libverbs $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ibv_open_device ();
+int
+main ()
+{
+return ibv_open_device ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_ibverbs_ibv_open_device=yes
+else
+ ac_cv_lib_ibverbs_ibv_open_device=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ibverbs_ibv_open_device" >&5
+$as_echo "$ac_cv_lib_ibverbs_ibv_open_device" >&6; }
+if test "x$ac_cv_lib_ibverbs_ibv_open_device" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_verbs_ibverbs_save_LDFLAGS"
+ verbs_ibverbs_LDFLAGS="$fi_check_package_verbs_ibverbs_orig_LDFLAGS"
+ unset ac_cv_lib_ibverbs_ibv_open_device
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ verbs_ibverbs_LDFLAGS="$verbs_ibverbs_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ibv_open_device in -libverbs" >&5
+$as_echo_n "checking for ibv_open_device in -libverbs... " >&6; }
+if test "${ac_cv_lib_ibverbs_ibv_open_device+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-libverbs $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ibv_open_device ();
+int
+main ()
+{
+return ibv_open_device ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_ibverbs_ibv_open_device=yes
+else
+ ac_cv_lib_ibverbs_ibv_open_device=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ibverbs_ibv_open_device" >&5
+$as_echo "$ac_cv_lib_ibverbs_ibv_open_device" >&6; }
+if test "x$ac_cv_lib_ibverbs_ibv_open_device" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_verbs_ibverbs_save_LDFLAGS"
+ verbs_ibverbs_LDFLAGS="$fi_check_package_verbs_ibverbs_orig_LDFLAGS"
+ unset ac_cv_lib_ibverbs_ibv_open_device
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ verbs_ibverbs_LDFLAGS="$verbs_ibverbs_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ibv_open_device in -libverbs" >&5
+$as_echo_n "checking for ibv_open_device in -libverbs... " >&6; }
+if test "${ac_cv_lib_ibverbs_ibv_open_device+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-libverbs $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ibv_open_device ();
+int
+main ()
+{
+return ibv_open_device ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_ibverbs_ibv_open_device=yes
+else
+ ac_cv_lib_ibverbs_ibv_open_device=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ibverbs_ibv_open_device" >&5
+$as_echo "$ac_cv_lib_ibverbs_ibv_open_device" >&6; }
+if test "x$ac_cv_lib_ibverbs_ibv_open_device" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_verbs_ibverbs_save_LDFLAGS"
+ verbs_ibverbs_LDFLAGS="$fi_check_package_verbs_ibverbs_orig_LDFLAGS"
+ unset ac_cv_lib_ibverbs_ibv_open_device
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ verbs_ibverbs_LIBS="-libverbs "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+else
+ fi_check_package_happy="no"
+fi
+else
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_ibverbs_ibv_open_device
+ fi_check_package_lib_happy="no"
+ if test "$verbs_LIBDIR" != ""; then :
+ # libdir was specified - search only there
+ verbs_ibverbs_LDFLAGS="$verbs_ibverbs_LDFLAGS -L$verbs_LIBDIR"
+ LDFLAGS="$LDFLAGS -L$verbs_LIBDIR"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ibv_open_device in -libverbs" >&5
+$as_echo_n "checking for ibv_open_device in -libverbs... " >&6; }
+if test "${ac_cv_lib_ibverbs_ibv_open_device+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-libverbs $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ibv_open_device ();
+int
+main ()
+{
+return ibv_open_device ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_ibverbs_ibv_open_device=yes
+else
+ ac_cv_lib_ibverbs_ibv_open_device=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ibverbs_ibv_open_device" >&5
+$as_echo "$ac_cv_lib_ibverbs_ibv_open_device" >&6; }
+if test "x$ac_cv_lib_ibverbs_ibv_open_device" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_verbs_ibverbs_save_LDFLAGS"
+ verbs_ibverbs_LDFLAGS="$fi_check_package_verbs_ibverbs_orig_LDFLAGS"
+ unset ac_cv_lib_ibverbs_ibv_open_device
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$verbs_PREFIX"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ibv_open_device in -libverbs" >&5
+$as_echo_n "checking for ibv_open_device in -libverbs... " >&6; }
+if test "${ac_cv_lib_ibverbs_ibv_open_device+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-libverbs $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ibv_open_device ();
+int
+main ()
+{
+return ibv_open_device ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_ibverbs_ibv_open_device=yes
+else
+ ac_cv_lib_ibverbs_ibv_open_device=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ibverbs_ibv_open_device" >&5
+$as_echo "$ac_cv_lib_ibverbs_ibv_open_device" >&6; }
+if test "x$ac_cv_lib_ibverbs_ibv_open_device" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_verbs_ibverbs_save_LDFLAGS"
+ verbs_ibverbs_LDFLAGS="$fi_check_package_verbs_ibverbs_orig_LDFLAGS"
+ unset ac_cv_lib_ibverbs_ibv_open_device
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ verbs_ibverbs_LDFLAGS="$verbs_ibverbs_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ibv_open_device in -libverbs" >&5
+$as_echo_n "checking for ibv_open_device in -libverbs... " >&6; }
+if test "${ac_cv_lib_ibverbs_ibv_open_device+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-libverbs $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ibv_open_device ();
+int
+main ()
+{
+return ibv_open_device ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_ibverbs_ibv_open_device=yes
+else
+ ac_cv_lib_ibverbs_ibv_open_device=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ibverbs_ibv_open_device" >&5
+$as_echo "$ac_cv_lib_ibverbs_ibv_open_device" >&6; }
+if test "x$ac_cv_lib_ibverbs_ibv_open_device" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_verbs_ibverbs_save_LDFLAGS"
+ verbs_ibverbs_LDFLAGS="$fi_check_package_verbs_ibverbs_orig_LDFLAGS"
+ unset ac_cv_lib_ibverbs_ibv_open_device
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ verbs_ibverbs_LDFLAGS="$verbs_ibverbs_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ibv_open_device in -libverbs" >&5
+$as_echo_n "checking for ibv_open_device in -libverbs... " >&6; }
+if test "${ac_cv_lib_ibverbs_ibv_open_device+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-libverbs $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ibv_open_device ();
+int
+main ()
+{
+return ibv_open_device ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_ibverbs_ibv_open_device=yes
+else
+ ac_cv_lib_ibverbs_ibv_open_device=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ibverbs_ibv_open_device" >&5
+$as_echo "$ac_cv_lib_ibverbs_ibv_open_device" >&6; }
+if test "x$ac_cv_lib_ibverbs_ibv_open_device" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_verbs_ibverbs_save_LDFLAGS"
+ verbs_ibverbs_LDFLAGS="$fi_check_package_verbs_ibverbs_orig_LDFLAGS"
+ unset ac_cv_lib_ibverbs_ibv_open_device
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ verbs_ibverbs_LIBS="-libverbs "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+fi
+ unset fi_check_package_header_happy
+
+
+
+ if test "$fi_check_package_happy" = "yes"; then :
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking if libibverbs is linkable by libtool" >&5
+$as_echo_n "checking if libibverbs is linkable by libtool... " >&6; }
+ file=conftemp.$$.c
+ rm -f $file conftemp
+ cat > $file <<-EOF
+char ibv_open_device ();
+int main ()
+{ return ibv_open_device (); }
+EOF
+
+ cmd="./libtool --mode=link --tag=CC $CC $CPPFLAGS $CFLAGS $file -o conftemp $LDFLAGS -libverbs"
+ echo "configure:$LINENO: $cmd" >> config.log 2>&1
+ eval $cmd >> config.log 2>&1
+ status=$?
+ if test $status -eq 0 && test -x conftemp; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ verbs_ibverbs_happy=1
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ echo "configure: failed program was" >> config.log
+ cat $file >> config.log
+ verbs_ibverbs_happy=0
+fi
+
+ rm -f $file conftemp
+
+else
+ verbs_ibverbs_CPPFLAGS="$fi_check_package_verbs_ibverbs_orig_CPPFLAGS"
+ verbs_ibverbs_LDFLAGS="$fi_check_package_verbs_ibverbs_orig_LDFLAGS"
+ verbs_ibverbs_LIBS="$fi_check_package_verbs_ibverbs_orig_LIBS"
+ verbs_ibverbs_happy=0
+fi
+
+
+
+
+
+ CPPFLAGS="$fi_check_package_verbs_ibverbs_save_CPPFLAGS"
+ LDFLAGS="$fi_check_package_verbs_ibverbs_save_LDFLAGS"
+ LIBS="$fi_check_package_verbs_ibverbs_save_LIBS"
+
+
+
+ fi_check_package_verbs_rdmacm_save_CPPFLAGS="$CPPFLAGS"
+ fi_check_package_verbs_rdmacm_save_LDFLAGS="$LDFLAGS"
+ fi_check_package_verbs_rdmacm_save_LIBS="$LIBS"
+
+ fi_check_package_verbs_rdmacm_orig_CPPFLAGS="$verbs_rdmacm_CPPFLAGS"
+ fi_check_package_verbs_rdmacm_orig_LDFLAGS="$verbs_rdmacm_LDFLAGS"
+ fi_check_package_verbs_rdmacm_orig_LIBS="$verbs_rdmacm_LIBS"
+
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+
+ # There's unfortunately no way to get through the progression of
+ # header includes without killing off the cache variable and
+ # trying again...
+ unset ac_cv_header_rdma_rsocket_h
+
+ fi_check_package_header_happy="no"
+ if test "$verbs_PREFIX" = "/usr" || test "$verbs_PREFIX" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for header without includes" >&5
+$as_echo "looking for header without includes" >&6; }
+ for ac_header in rdma/rsocket.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "rdma/rsocket.h" "ac_cv_header_rdma_rsocket_h" "$ac_includes_default"
+if test "x$ac_cv_header_rdma_rsocket_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_RDMA_RSOCKET_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ # no go on the as is - reset the cache and try again
+ unset ac_cv_header_rdma_rsocket_h
+fi
+fi
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ if test "$verbs_PREFIX" != ""; then :
+ verbs_rdmacm_CPPFLAGS="$verbs_rdmacm_CPPFLAGS -I$verbs_PREFIX/include"
+ CPPFLAGS="$CPPFLAGS -I$verbs_PREFIX/include"
+fi
+ for ac_header in rdma/rsocket.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "rdma/rsocket.h" "ac_cv_header_rdma_rsocket_h" "$ac_includes_default"
+if test "x$ac_cv_header_rdma_rsocket_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_RDMA_RSOCKET_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "yes"; then :
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_rdmacm_rdma_create_qp
+ fi_check_package_lib_happy="no"
+ if test "$verbs_LIBDIR" != ""; then :
+ # libdir was specified - search only there
+ verbs_rdmacm_LDFLAGS="$verbs_rdmacm_LDFLAGS -L$verbs_LIBDIR"
+ LDFLAGS="$LDFLAGS -L$verbs_LIBDIR"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for rdma_create_qp in -lrdmacm" >&5
+$as_echo_n "checking for rdma_create_qp in -lrdmacm... " >&6; }
+if test "${ac_cv_lib_rdmacm_rdma_create_qp+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrdmacm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char rdma_create_qp ();
+int
+main ()
+{
+return rdma_create_qp ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rdmacm_rdma_create_qp=yes
+else
+ ac_cv_lib_rdmacm_rdma_create_qp=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rdmacm_rdma_create_qp" >&5
+$as_echo "$ac_cv_lib_rdmacm_rdma_create_qp" >&6; }
+if test "x$ac_cv_lib_rdmacm_rdma_create_qp" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_verbs_rdmacm_save_LDFLAGS"
+ verbs_rdmacm_LDFLAGS="$fi_check_package_verbs_rdmacm_orig_LDFLAGS"
+ unset ac_cv_lib_rdmacm_rdma_create_qp
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$verbs_PREFIX"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for rdma_create_qp in -lrdmacm" >&5
+$as_echo_n "checking for rdma_create_qp in -lrdmacm... " >&6; }
+if test "${ac_cv_lib_rdmacm_rdma_create_qp+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrdmacm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char rdma_create_qp ();
+int
+main ()
+{
+return rdma_create_qp ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rdmacm_rdma_create_qp=yes
+else
+ ac_cv_lib_rdmacm_rdma_create_qp=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rdmacm_rdma_create_qp" >&5
+$as_echo "$ac_cv_lib_rdmacm_rdma_create_qp" >&6; }
+if test "x$ac_cv_lib_rdmacm_rdma_create_qp" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_verbs_rdmacm_save_LDFLAGS"
+ verbs_rdmacm_LDFLAGS="$fi_check_package_verbs_rdmacm_orig_LDFLAGS"
+ unset ac_cv_lib_rdmacm_rdma_create_qp
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ verbs_rdmacm_LDFLAGS="$verbs_rdmacm_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for rdma_create_qp in -lrdmacm" >&5
+$as_echo_n "checking for rdma_create_qp in -lrdmacm... " >&6; }
+if test "${ac_cv_lib_rdmacm_rdma_create_qp+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrdmacm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char rdma_create_qp ();
+int
+main ()
+{
+return rdma_create_qp ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rdmacm_rdma_create_qp=yes
+else
+ ac_cv_lib_rdmacm_rdma_create_qp=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rdmacm_rdma_create_qp" >&5
+$as_echo "$ac_cv_lib_rdmacm_rdma_create_qp" >&6; }
+if test "x$ac_cv_lib_rdmacm_rdma_create_qp" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_verbs_rdmacm_save_LDFLAGS"
+ verbs_rdmacm_LDFLAGS="$fi_check_package_verbs_rdmacm_orig_LDFLAGS"
+ unset ac_cv_lib_rdmacm_rdma_create_qp
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ verbs_rdmacm_LDFLAGS="$verbs_rdmacm_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for rdma_create_qp in -lrdmacm" >&5
+$as_echo_n "checking for rdma_create_qp in -lrdmacm... " >&6; }
+if test "${ac_cv_lib_rdmacm_rdma_create_qp+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrdmacm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char rdma_create_qp ();
+int
+main ()
+{
+return rdma_create_qp ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rdmacm_rdma_create_qp=yes
+else
+ ac_cv_lib_rdmacm_rdma_create_qp=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rdmacm_rdma_create_qp" >&5
+$as_echo "$ac_cv_lib_rdmacm_rdma_create_qp" >&6; }
+if test "x$ac_cv_lib_rdmacm_rdma_create_qp" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_verbs_rdmacm_save_LDFLAGS"
+ verbs_rdmacm_LDFLAGS="$fi_check_package_verbs_rdmacm_orig_LDFLAGS"
+ unset ac_cv_lib_rdmacm_rdma_create_qp
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ verbs_rdmacm_LIBS="-lrdmacm "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+else
+ fi_check_package_happy="no"
+fi
+else
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_rdmacm_rdma_create_qp
+ fi_check_package_lib_happy="no"
+ if test "$verbs_LIBDIR" != ""; then :
+ # libdir was specified - search only there
+ verbs_rdmacm_LDFLAGS="$verbs_rdmacm_LDFLAGS -L$verbs_LIBDIR"
+ LDFLAGS="$LDFLAGS -L$verbs_LIBDIR"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for rdma_create_qp in -lrdmacm" >&5
+$as_echo_n "checking for rdma_create_qp in -lrdmacm... " >&6; }
+if test "${ac_cv_lib_rdmacm_rdma_create_qp+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrdmacm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char rdma_create_qp ();
+int
+main ()
+{
+return rdma_create_qp ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rdmacm_rdma_create_qp=yes
+else
+ ac_cv_lib_rdmacm_rdma_create_qp=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rdmacm_rdma_create_qp" >&5
+$as_echo "$ac_cv_lib_rdmacm_rdma_create_qp" >&6; }
+if test "x$ac_cv_lib_rdmacm_rdma_create_qp" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_verbs_rdmacm_save_LDFLAGS"
+ verbs_rdmacm_LDFLAGS="$fi_check_package_verbs_rdmacm_orig_LDFLAGS"
+ unset ac_cv_lib_rdmacm_rdma_create_qp
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$verbs_PREFIX"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for rdma_create_qp in -lrdmacm" >&5
+$as_echo_n "checking for rdma_create_qp in -lrdmacm... " >&6; }
+if test "${ac_cv_lib_rdmacm_rdma_create_qp+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrdmacm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char rdma_create_qp ();
+int
+main ()
+{
+return rdma_create_qp ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rdmacm_rdma_create_qp=yes
+else
+ ac_cv_lib_rdmacm_rdma_create_qp=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rdmacm_rdma_create_qp" >&5
+$as_echo "$ac_cv_lib_rdmacm_rdma_create_qp" >&6; }
+if test "x$ac_cv_lib_rdmacm_rdma_create_qp" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_verbs_rdmacm_save_LDFLAGS"
+ verbs_rdmacm_LDFLAGS="$fi_check_package_verbs_rdmacm_orig_LDFLAGS"
+ unset ac_cv_lib_rdmacm_rdma_create_qp
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ verbs_rdmacm_LDFLAGS="$verbs_rdmacm_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for rdma_create_qp in -lrdmacm" >&5
+$as_echo_n "checking for rdma_create_qp in -lrdmacm... " >&6; }
+if test "${ac_cv_lib_rdmacm_rdma_create_qp+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrdmacm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char rdma_create_qp ();
+int
+main ()
+{
+return rdma_create_qp ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rdmacm_rdma_create_qp=yes
+else
+ ac_cv_lib_rdmacm_rdma_create_qp=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rdmacm_rdma_create_qp" >&5
+$as_echo "$ac_cv_lib_rdmacm_rdma_create_qp" >&6; }
+if test "x$ac_cv_lib_rdmacm_rdma_create_qp" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_verbs_rdmacm_save_LDFLAGS"
+ verbs_rdmacm_LDFLAGS="$fi_check_package_verbs_rdmacm_orig_LDFLAGS"
+ unset ac_cv_lib_rdmacm_rdma_create_qp
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ verbs_rdmacm_LDFLAGS="$verbs_rdmacm_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for rdma_create_qp in -lrdmacm" >&5
+$as_echo_n "checking for rdma_create_qp in -lrdmacm... " >&6; }
+if test "${ac_cv_lib_rdmacm_rdma_create_qp+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrdmacm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char rdma_create_qp ();
+int
+main ()
+{
+return rdma_create_qp ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rdmacm_rdma_create_qp=yes
+else
+ ac_cv_lib_rdmacm_rdma_create_qp=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rdmacm_rdma_create_qp" >&5
+$as_echo "$ac_cv_lib_rdmacm_rdma_create_qp" >&6; }
+if test "x$ac_cv_lib_rdmacm_rdma_create_qp" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_verbs_rdmacm_save_LDFLAGS"
+ verbs_rdmacm_LDFLAGS="$fi_check_package_verbs_rdmacm_orig_LDFLAGS"
+ unset ac_cv_lib_rdmacm_rdma_create_qp
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ verbs_rdmacm_LIBS="-lrdmacm "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+fi
+ unset fi_check_package_header_happy
+
+
+
+ if test "$fi_check_package_happy" = "yes"; then :
+ verbs_rdmacm_happy=1
+else
+ verbs_rdmacm_CPPFLAGS="$fi_check_package_verbs_rdmacm_orig_CPPFLAGS"
+ verbs_rdmacm_LDFLAGS="$fi_check_package_verbs_rdmacm_orig_LDFLAGS"
+ verbs_rdmacm_LIBS="$fi_check_package_verbs_rdmacm_orig_LIBS"
+ verbs_rdmacm_happy=0
+fi
+
+
+
+
+
+ CPPFLAGS="$fi_check_package_verbs_rdmacm_save_CPPFLAGS"
+ LDFLAGS="$fi_check_package_verbs_rdmacm_save_LDFLAGS"
+ LIBS="$fi_check_package_verbs_rdmacm_save_LIBS"
+
+
+fi
+
+ if test $verbs_ibverbs_happy -eq 1 && \
+ test $verbs_rdmacm_happy -eq 1; then :
+ verbs_happy=1
+else
+ verbs_happy=0
+fi
+
+ # Technically, verbs_ibverbs_CPPFLAGS and
+ # verbs_rdmacm_CPPFLAGS could be different, but it is highly
+ # unlikely that they ever will be. So only list
+ # verbs_ibverbs_CPPFLAGS here. Same with verbs_*_LDFLAGS,
+ # below.
+ verbs_CPPFLAGS=$verbs_ibverbs_CPPFLAGS
+ verbs_LDFLAGS=$verbs_ibverbs_LDFLAGS
+ verbs_LIBS="$verbs_rdmacm_LIBS $verbs_ibverbs_LIBS"
+
+
+
+
+
+
+
+
+
+
+ # See if the provider configured successfully
+ if test $verbs_happy -eq 1; then :
+ PROVIDERS_TO_BUILD="$PROVIDERS_TO_BUILD verbs"
+ PROVIDERS_COUNT=$((PROVIDERS_COUNT+1))
+
+ FI_PC_CFLAGS="$FI_PC_CFLAGS $verbs_CFLAGS $verbs_CPPFLAGS"
+ FI_PC_PRIVATE_LIBS="$FI_PC_PRIVATE_LIBS $verbs_LDFLAGS $verbs_LIBS"
+
+ if test $verbs_dl -eq 1; then :
+ PROVIDERS_DL="prov/verbs/libverbs.la $PROVIDERS_DL"
+ if test x"$enable_static" = x"yes" &&
+ test x"$enable_shared" = x"no"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: verbs provider was selected to be built as DL" >&5
+$as_echo "$as_me: WARNING: verbs provider was selected to be built as DL" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: but libfabric is being built as static-only" >&5
+$as_echo "$as_me: WARNING: but libfabric is being built as static-only" >&2;}
+ as_fn_error $? "This is an impossible situation. Cannot continue." "$LINENO" 5
+fi
+ { $as_echo "$as_me:${as_lineno-$LINENO}: verbs provider: build as plugin" >&5
+$as_echo "$as_me: verbs provider: build as plugin" >&6;}
+
+else
+ PROVIDERS_STATIC="prov/verbs/libverbs.la $PROVIDERS_STATIC"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: verbs provider: include in libfabric" >&5
+$as_echo "$as_me: verbs provider: include in libfabric" >&6;}
+fi
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: verbs provider: disabled" >&5
+$as_echo "$as_me: verbs provider: disabled" >&6;}
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_VERBS $verbs_happy
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_VERBS_DL $verbs_dl
+_ACEOF
+
+
+ # Set AM conditionals for HAVE_<provider> and HAVE_<provider>_DL
+ # as well as AC defines
+ if test $verbs_happy -eq 1; then
+ HAVE_VERBS_TRUE=
+ HAVE_VERBS_FALSE='#'
+else
+ HAVE_VERBS_TRUE='#'
+ HAVE_VERBS_FALSE=
+fi
+
+ if test $verbs_dl -eq 1; then
+ HAVE_VERBS_DL_TRUE=
+ HAVE_VERBS_DL_FALSE='#'
+else
+ HAVE_VERBS_DL_TRUE='#'
+ HAVE_VERBS_DL_FALSE=
+fi
+
+
+ # If this provider was specifically requested but we can't
+ # build it, error.
+ if test "$enable_verbs $verbs_happy" = "yes 0"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: verbs provider was requested, but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: verbs provider was requested, but cannot be compiled" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+ # If this provider was requested for direct build, ensure that
+ # provider's fi_direct.h exists in tree. Error otherwise.
+ if test x"$enable_direct" = x"verbs"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $srcdir/prov/verbs/include/rdma/fi_direct.h" >&5
+$as_echo_n "checking for $srcdir/prov/verbs/include/rdma/fi_direct.h... " >&6; }
+ if test -f "$srcdir/prov/verbs/include/rdma/fi_direct.h"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ as_fn_error $? "verbs provider was requested as direct, but is missing fi_direct.h" "$LINENO" 5
+
+fi
+fi
+
+ # Restore CPPFLAGS/LDFLAGS/LIBS
+ CPPFLAGS=$verbs_orig_CPPFLAGS
+ unset verbs_orig_CPPFLAGS
+ LDFLAGS=$verbs_orig_LDFLAGS
+ unset verbs_orig_LDFLAGS
+ LIBS=$verbs_orig_LIBS
+ unset verbs_orig_LIBS
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: *** Configuring usnic provider" >&5
+$as_echo "$as_me: *** Configuring usnic provider" >&6;}
+ # Check whether --enable-usnic was given.
+if test "${enable_usnic+set}" = set; then :
+ enableval=$enable_usnic;
+else
+ enable_usnic=auto
+fi
+
+
+ # Save CPPFLAGS and LDFLAGS before they are modified by FI_CHECK_PREFIX_DIR.
+ # Provider's local macros could use the value if needed.
+ # Also save LIBS, as a matter of principle.
+ usnic_orig_CPPFLAGS=$CPPFLAGS
+ usnic_orig_LDFLAGS=$LDFLAGS
+ usnic_orig_LIBS=$LIBS
+
+ # Check the --enable-<usnic> value
+ usnic_dl=0
+ case $enable_usnic in #(
+ yes|no) :
+ ;; #(
+ dl) :
+ enable_usnic=yes usnic_dl=1 ;; #(
+ auto) :
+ ;; #(
+ *) :
+
+ # Check that the base directory exists
+ if test ! -d "$enable_usnic"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: supplied directory \"$enable_usnic\" does not exist" >&5
+$as_echo "$as_me: WARNING: supplied directory \"$enable_usnic\" does not exist" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/include exists
+ if test -d "$enable_usnic/include"; then :
+ usnic_PREFIX="$enable_usnic"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"include\" subdirectory in supplied \"$enable_usnic\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"include\" subdirectory in supplied \"$enable_usnic\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/lib or base/lib64 exists
+ if test -d "$enable_usnic/lib"; then :
+ usnic_LIBDIR="$enable_usnic/lib"
+else
+ if test -d "$enable_usnic/lib64"; then :
+ usnic_LIBDIR="$enable_usnic/lib64"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_usnic\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_usnic\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+fi
+
+ enable_usnic=yes
+ ;;
+esac
+
+ # Call the provider's CONFIGURE and CONDITIONALS macros
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ # Determine if we can support the usnic provider
+ usnic_happy=0
+ if test "x$enable_usnic" != "xno"; then :
+ ac_fn_c_check_header_mongrel "$LINENO" "infiniband/verbs.h" "ac_cv_header_infiniband_verbs_h" "$ac_includes_default"
+if test "x$ac_cv_header_infiniband_verbs_h" = x""yes; then :
+ usnic_happy=1
+fi
+
+
+ if test $usnic_happy -eq 1; then :
+
+
+# Check whether --with-libnl was given.
+if test "${with_libnl+set}" = set; then :
+ withval=$with_libnl;
+else
+ with_libnl=default
+fi
+
+
+ # The --with options carry two pieces of information: 1) do
+ # you want a specific version of libnl, and 2) where that
+ # version of libnl lives. For simplicity, let's separate
+ # those two pieces of information.
+
+ case "$with_libnl" in
+ no)
+ # Nope, don't want it
+ usnic_want_libnl=no
+ ;;
+ yes)
+ # Yes, definitely want it
+ usnic_want_libnl=yes
+ ;;
+ default)
+ # Default case -- try and see if we can find it
+ usnic_want_libnl=default
+ usnic_libnl_location=/usr
+ ;;
+ *)
+ # Yes, definitely want it -- at a specific location
+ usnic_want_libnl=yes
+ usnic_libnl_location="$with_libnl"
+ ;;
+ esac
+
+
+ # Default to a numeric value (this value gets AC_DEFINEd)
+ HAVE_LIBNL3=0
+
+ ###################################################
+ # NOTE: We *must* check for libnl3 before libnl.
+ ###################################################
+
+ if test "$usnic_want_libnl" != "no"; then :
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for libnl3" >&5
+$as_echo "$as_me: checking for libnl3" >&6;}
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for libnl3 prefix" >&5
+$as_echo_n "checking for libnl3 prefix... " >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $usnic_libnl_location" >&5
+$as_echo "$usnic_libnl_location" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $usnic_libnl_location/include/libnl3" >&5
+$as_echo_n "checking for $usnic_libnl_location/include/libnl3... " >&6; }
+ if test -d "$usnic_libnl_location/include/libnl3"; then :
+ usnic_libnl3_happy=1
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: found" >&5
+$as_echo "found" >&6; }
+else
+ usnic_libnl3_happy=0
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: not found" >&5
+$as_echo "not found" >&6; }
+fi
+
+ # Random note: netlink/version.h is only in libnl3 - it is not in libnl.
+ # Also, nl_socket_set_peer_groups is only in libnl3.
+ CPPFLAGS_save=$CPPFLAGS
+ usnic_tmp_CPPFLAGS="-I$usnic_libnl_location/include/libnl3"
+ CPPFLAGS="$usnic_tmp_CPPFLAGS $CPPFLAGS"
+ if test $usnic_libnl3_happy -eq 1; then :
+
+ fi_check_package_usnic_nl_save_CPPFLAGS="$CPPFLAGS"
+ fi_check_package_usnic_nl_save_LDFLAGS="$LDFLAGS"
+ fi_check_package_usnic_nl_save_LIBS="$LIBS"
+
+ fi_check_package_usnic_nl_orig_CPPFLAGS="$usnic_nl_CPPFLAGS"
+ fi_check_package_usnic_nl_orig_LDFLAGS="$usnic_nl_LDFLAGS"
+ fi_check_package_usnic_nl_orig_LIBS="$usnic_nl_LIBS"
+
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+
+ # There's unfortunately no way to get through the progression of
+ # header includes without killing off the cache variable and
+ # trying again...
+ unset ac_cv_header_netlink_version_h
+
+ fi_check_package_header_happy="no"
+ if test "$usnic_libnl_location" = "/usr" || test "$usnic_libnl_location" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for header without includes" >&5
+$as_echo "looking for header without includes" >&6; }
+ for ac_header in netlink/version.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "netlink/version.h" "ac_cv_header_netlink_version_h" "$ac_includes_default"
+if test "x$ac_cv_header_netlink_version_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_NETLINK_VERSION_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ # no go on the as is - reset the cache and try again
+ unset ac_cv_header_netlink_version_h
+fi
+fi
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ if test "$usnic_libnl_location" != ""; then :
+ usnic_nl_CPPFLAGS="$usnic_nl_CPPFLAGS -I$usnic_libnl_location/include"
+ CPPFLAGS="$CPPFLAGS -I$usnic_libnl_location/include"
+fi
+ for ac_header in netlink/version.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "netlink/version.h" "ac_cv_header_netlink_version_h" "$ac_includes_default"
+if test "x$ac_cv_header_netlink_version_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_NETLINK_VERSION_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "yes"; then :
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_nl_3_nl_socket_set_peer_groups
+ fi_check_package_lib_happy="no"
+ if test "" != ""; then :
+ # libdir was specified - search only there
+ usnic_nl_LDFLAGS="$usnic_nl_LDFLAGS -L"
+ LDFLAGS="$LDFLAGS -L"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_socket_set_peer_groups in -lnl-3" >&5
+$as_echo_n "checking for nl_socket_set_peer_groups in -lnl-3... " >&6; }
+if test "${ac_cv_lib_nl_3_nl_socket_set_peer_groups+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl-3 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_socket_set_peer_groups ();
+int
+main ()
+{
+return nl_socket_set_peer_groups ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=yes
+else
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&5
+$as_echo "$ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&6; }
+if test "x$ac_cv_lib_nl_3_nl_socket_set_peer_groups" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_3_nl_socket_set_peer_groups
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$usnic_libnl_location"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_socket_set_peer_groups in -lnl-3" >&5
+$as_echo_n "checking for nl_socket_set_peer_groups in -lnl-3... " >&6; }
+if test "${ac_cv_lib_nl_3_nl_socket_set_peer_groups+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl-3 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_socket_set_peer_groups ();
+int
+main ()
+{
+return nl_socket_set_peer_groups ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=yes
+else
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&5
+$as_echo "$ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&6; }
+if test "x$ac_cv_lib_nl_3_nl_socket_set_peer_groups" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_3_nl_socket_set_peer_groups
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ usnic_nl_LDFLAGS="$usnic_nl_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_socket_set_peer_groups in -lnl-3" >&5
+$as_echo_n "checking for nl_socket_set_peer_groups in -lnl-3... " >&6; }
+if test "${ac_cv_lib_nl_3_nl_socket_set_peer_groups+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl-3 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_socket_set_peer_groups ();
+int
+main ()
+{
+return nl_socket_set_peer_groups ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=yes
+else
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&5
+$as_echo "$ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&6; }
+if test "x$ac_cv_lib_nl_3_nl_socket_set_peer_groups" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_3_nl_socket_set_peer_groups
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ usnic_nl_LDFLAGS="$usnic_nl_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_socket_set_peer_groups in -lnl-3" >&5
+$as_echo_n "checking for nl_socket_set_peer_groups in -lnl-3... " >&6; }
+if test "${ac_cv_lib_nl_3_nl_socket_set_peer_groups+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl-3 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_socket_set_peer_groups ();
+int
+main ()
+{
+return nl_socket_set_peer_groups ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=yes
+else
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&5
+$as_echo "$ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&6; }
+if test "x$ac_cv_lib_nl_3_nl_socket_set_peer_groups" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_3_nl_socket_set_peer_groups
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ usnic_nl_LIBS="-lnl-3 "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+else
+ fi_check_package_happy="no"
+fi
+else
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_nl_3_nl_socket_set_peer_groups
+ fi_check_package_lib_happy="no"
+ if test "" != ""; then :
+ # libdir was specified - search only there
+ usnic_nl_LDFLAGS="$usnic_nl_LDFLAGS -L"
+ LDFLAGS="$LDFLAGS -L"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_socket_set_peer_groups in -lnl-3" >&5
+$as_echo_n "checking for nl_socket_set_peer_groups in -lnl-3... " >&6; }
+if test "${ac_cv_lib_nl_3_nl_socket_set_peer_groups+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl-3 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_socket_set_peer_groups ();
+int
+main ()
+{
+return nl_socket_set_peer_groups ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=yes
+else
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&5
+$as_echo "$ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&6; }
+if test "x$ac_cv_lib_nl_3_nl_socket_set_peer_groups" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_3_nl_socket_set_peer_groups
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$usnic_libnl_location"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_socket_set_peer_groups in -lnl-3" >&5
+$as_echo_n "checking for nl_socket_set_peer_groups in -lnl-3... " >&6; }
+if test "${ac_cv_lib_nl_3_nl_socket_set_peer_groups+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl-3 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_socket_set_peer_groups ();
+int
+main ()
+{
+return nl_socket_set_peer_groups ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=yes
+else
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&5
+$as_echo "$ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&6; }
+if test "x$ac_cv_lib_nl_3_nl_socket_set_peer_groups" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_3_nl_socket_set_peer_groups
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ usnic_nl_LDFLAGS="$usnic_nl_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_socket_set_peer_groups in -lnl-3" >&5
+$as_echo_n "checking for nl_socket_set_peer_groups in -lnl-3... " >&6; }
+if test "${ac_cv_lib_nl_3_nl_socket_set_peer_groups+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl-3 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_socket_set_peer_groups ();
+int
+main ()
+{
+return nl_socket_set_peer_groups ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=yes
+else
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&5
+$as_echo "$ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&6; }
+if test "x$ac_cv_lib_nl_3_nl_socket_set_peer_groups" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_3_nl_socket_set_peer_groups
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ usnic_nl_LDFLAGS="$usnic_nl_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_socket_set_peer_groups in -lnl-3" >&5
+$as_echo_n "checking for nl_socket_set_peer_groups in -lnl-3... " >&6; }
+if test "${ac_cv_lib_nl_3_nl_socket_set_peer_groups+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl-3 $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_socket_set_peer_groups ();
+int
+main ()
+{
+return nl_socket_set_peer_groups ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=yes
+else
+ ac_cv_lib_nl_3_nl_socket_set_peer_groups=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&5
+$as_echo "$ac_cv_lib_nl_3_nl_socket_set_peer_groups" >&6; }
+if test "x$ac_cv_lib_nl_3_nl_socket_set_peer_groups" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_3_nl_socket_set_peer_groups
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ usnic_nl_LIBS="-lnl-3 "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+fi
+ unset fi_check_package_header_happy
+
+
+
+ if test "$fi_check_package_happy" = "yes"; then :
+ usnic_libnl3_happy=1
+else
+ usnic_nl_CPPFLAGS="$fi_check_package_usnic_nl_orig_CPPFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ usnic_nl_LIBS="$fi_check_package_usnic_nl_orig_LIBS"
+ usnic_libnl3_happy=0
+fi
+
+
+
+
+
+ CPPFLAGS="$fi_check_package_usnic_nl_save_CPPFLAGS"
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ LIBS="$fi_check_package_usnic_nl_save_LIBS"
+
+
+ # Note that FI_CHECK_PACKAGE is going to add
+ # -I$dir/include into usnic_nl_CPPFLAGS. But because libnl3
+ # puts the headers in $dir/libnl3, we need to
+ # overwrite usnic_nl_CPPFLAGS with -I$dir/libnl3. We can do
+ # this unconditionally; we don't have to check for
+ # success (checking for success occurs below).
+ usnic_nl_CPPFLAGS=$usnic_tmp_CPPFLAGS
+fi
+
+ # If we found libnl-3, we *also* need libnl-route-3
+ LIBS_save=$LIBS
+ LDFLAGS_save=$LDFLAGS
+ if test "$usnic_nl_LDFLAGS" != ""; then :
+ LDFLAGS="$usnic_nl_LDFLAGS $LDFLAGS"
+fi
+ if test $usnic_libnl3_happy -eq 1; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing nl_rtgen_request" >&5
+$as_echo_n "checking for library containing nl_rtgen_request... " >&6; }
+if test "${ac_cv_search_nl_rtgen_request+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_rtgen_request ();
+int
+main ()
+{
+return nl_rtgen_request ();
+ ;
+ return 0;
+}
+_ACEOF
+for ac_lib in '' nl-route-3; do
+ if test -z "$ac_lib"; then
+ ac_res="none required"
+ else
+ ac_res=-l$ac_lib
+ LIBS="-l$ac_lib $ac_func_search_save_LIBS"
+ fi
+ if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_search_nl_rtgen_request=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext
+ if test "${ac_cv_search_nl_rtgen_request+set}" = set; then :
+ break
+fi
+done
+if test "${ac_cv_search_nl_rtgen_request+set}" = set; then :
+
+else
+ ac_cv_search_nl_rtgen_request=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_nl_rtgen_request" >&5
+$as_echo "$ac_cv_search_nl_rtgen_request" >&6; }
+ac_res=$ac_cv_search_nl_rtgen_request
+if test "$ac_res" != no; then :
+ test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+ usnic_libnl3_happy=1
+else
+ usnic_libnl3_happy=0
+fi
+
+fi
+ LIBS=$LIBS_save
+ LDFLAGS=$LDFLAGS_save
+
+ # Just because libnl* is evil, double check that the
+ # netlink/version.h we found was for libnl3. As far as we
+ # know, netlink/version.h only first appeared in version
+ # 3... but let's really be sure.
+ if test $usnic_libnl3_happy -eq 1; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking to ensure these really are libnl3 headers" >&5
+$as_echo_n "checking to ensure these really are libnl3 headers... " >&6; }
+ CPPFLAGS="$usnic_nl_CPPFLAGS $CPPFLAGS"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+#include <netlink/netlink.h>
+#include <netlink/version.h>
+#ifndef LIBNL_VER_MAJ
+#error "LIBNL_VER_MAJ not defined!"
+#endif
+/* to the best of our knowledge, version.h only exists in libnl3 */
+#if LIBNL_VER_MAJ != 3
+#error "LIBNL_VER_MAJ != 3, I am sad"
+#endif
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ usnic_libnl3_happy=0
+
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+ CPPFLAGS=$CPPFLAGS_save
+
+ # If we found everything
+ if test $usnic_libnl3_happy -eq 1; then :
+ usnic_nl_LIBS="-lnl-3 -lnl-route-3"
+ HAVE_LIBNL3=1
+else
+ usnic_nl_CPPFLAGS=
+ usnic_nl_LDFLAGS=
+ usnic_nl_LIBS=
+fi
+
+fi
+ if test "$usnic_want_libnl" != "no" &&
+ test "$usnic_nl_LIBS" = ""; then :
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for libnl" >&5
+$as_echo "$as_me: checking for libnl" >&6;}
+
+
+ fi_check_package_usnic_nl_save_CPPFLAGS="$CPPFLAGS"
+ fi_check_package_usnic_nl_save_LDFLAGS="$LDFLAGS"
+ fi_check_package_usnic_nl_save_LIBS="$LIBS"
+
+ fi_check_package_usnic_nl_orig_CPPFLAGS="$usnic_nl_CPPFLAGS"
+ fi_check_package_usnic_nl_orig_LDFLAGS="$usnic_nl_LDFLAGS"
+ fi_check_package_usnic_nl_orig_LIBS="$usnic_nl_LIBS"
+
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+
+ # There's unfortunately no way to get through the progression of
+ # header includes without killing off the cache variable and
+ # trying again...
+ unset ac_cv_header_netlink_netlink_h
+
+ fi_check_package_header_happy="no"
+ if test "$usnic_libnl_location" = "/usr" || test "$usnic_libnl_location" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for header without includes" >&5
+$as_echo "looking for header without includes" >&6; }
+ for ac_header in netlink/netlink.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "netlink/netlink.h" "ac_cv_header_netlink_netlink_h" "$ac_includes_default"
+if test "x$ac_cv_header_netlink_netlink_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_NETLINK_NETLINK_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ # no go on the as is - reset the cache and try again
+ unset ac_cv_header_netlink_netlink_h
+fi
+fi
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ if test "$usnic_libnl_location" != ""; then :
+ usnic_nl_CPPFLAGS="$usnic_nl_CPPFLAGS -I$usnic_libnl_location/include"
+ CPPFLAGS="$CPPFLAGS -I$usnic_libnl_location/include"
+fi
+ for ac_header in netlink/netlink.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "netlink/netlink.h" "ac_cv_header_netlink_netlink_h" "$ac_includes_default"
+if test "x$ac_cv_header_netlink_netlink_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_NETLINK_NETLINK_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "yes"; then :
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_nl_nl_connect
+ fi_check_package_lib_happy="no"
+ if test "" != ""; then :
+ # libdir was specified - search only there
+ usnic_nl_LDFLAGS="$usnic_nl_LDFLAGS -L"
+ LDFLAGS="$LDFLAGS -L"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_connect in -lnl" >&5
+$as_echo_n "checking for nl_connect in -lnl... " >&6; }
+if test "${ac_cv_lib_nl_nl_connect+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl -lm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_connect ();
+int
+main ()
+{
+return nl_connect ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_nl_connect=yes
+else
+ ac_cv_lib_nl_nl_connect=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_nl_connect" >&5
+$as_echo "$ac_cv_lib_nl_nl_connect" >&6; }
+if test "x$ac_cv_lib_nl_nl_connect" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_nl_connect
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$usnic_libnl_location"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_connect in -lnl" >&5
+$as_echo_n "checking for nl_connect in -lnl... " >&6; }
+if test "${ac_cv_lib_nl_nl_connect+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl -lm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_connect ();
+int
+main ()
+{
+return nl_connect ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_nl_connect=yes
+else
+ ac_cv_lib_nl_nl_connect=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_nl_connect" >&5
+$as_echo "$ac_cv_lib_nl_nl_connect" >&6; }
+if test "x$ac_cv_lib_nl_nl_connect" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_nl_connect
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ usnic_nl_LDFLAGS="$usnic_nl_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_connect in -lnl" >&5
+$as_echo_n "checking for nl_connect in -lnl... " >&6; }
+if test "${ac_cv_lib_nl_nl_connect+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl -lm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_connect ();
+int
+main ()
+{
+return nl_connect ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_nl_connect=yes
+else
+ ac_cv_lib_nl_nl_connect=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_nl_connect" >&5
+$as_echo "$ac_cv_lib_nl_nl_connect" >&6; }
+if test "x$ac_cv_lib_nl_nl_connect" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_nl_connect
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ usnic_nl_LDFLAGS="$usnic_nl_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_connect in -lnl" >&5
+$as_echo_n "checking for nl_connect in -lnl... " >&6; }
+if test "${ac_cv_lib_nl_nl_connect+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl -lm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_connect ();
+int
+main ()
+{
+return nl_connect ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_nl_connect=yes
+else
+ ac_cv_lib_nl_nl_connect=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_nl_connect" >&5
+$as_echo "$ac_cv_lib_nl_nl_connect" >&6; }
+if test "x$ac_cv_lib_nl_nl_connect" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_nl_connect
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ usnic_nl_LIBS="-lnl -lm"
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+else
+ fi_check_package_happy="no"
+fi
+else
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_nl_nl_connect
+ fi_check_package_lib_happy="no"
+ if test "" != ""; then :
+ # libdir was specified - search only there
+ usnic_nl_LDFLAGS="$usnic_nl_LDFLAGS -L"
+ LDFLAGS="$LDFLAGS -L"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_connect in -lnl" >&5
+$as_echo_n "checking for nl_connect in -lnl... " >&6; }
+if test "${ac_cv_lib_nl_nl_connect+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl -lm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_connect ();
+int
+main ()
+{
+return nl_connect ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_nl_connect=yes
+else
+ ac_cv_lib_nl_nl_connect=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_nl_connect" >&5
+$as_echo "$ac_cv_lib_nl_nl_connect" >&6; }
+if test "x$ac_cv_lib_nl_nl_connect" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_nl_connect
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$usnic_libnl_location"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_connect in -lnl" >&5
+$as_echo_n "checking for nl_connect in -lnl... " >&6; }
+if test "${ac_cv_lib_nl_nl_connect+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl -lm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_connect ();
+int
+main ()
+{
+return nl_connect ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_nl_connect=yes
+else
+ ac_cv_lib_nl_nl_connect=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_nl_connect" >&5
+$as_echo "$ac_cv_lib_nl_nl_connect" >&6; }
+if test "x$ac_cv_lib_nl_nl_connect" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_nl_connect
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ usnic_nl_LDFLAGS="$usnic_nl_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_connect in -lnl" >&5
+$as_echo_n "checking for nl_connect in -lnl... " >&6; }
+if test "${ac_cv_lib_nl_nl_connect+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl -lm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_connect ();
+int
+main ()
+{
+return nl_connect ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_nl_connect=yes
+else
+ ac_cv_lib_nl_nl_connect=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_nl_connect" >&5
+$as_echo "$ac_cv_lib_nl_nl_connect" >&6; }
+if test "x$ac_cv_lib_nl_nl_connect" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_nl_connect
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ usnic_nl_LDFLAGS="$usnic_nl_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nl_connect in -lnl" >&5
+$as_echo_n "checking for nl_connect in -lnl... " >&6; }
+if test "${ac_cv_lib_nl_nl_connect+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lnl -lm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char nl_connect ();
+int
+main ()
+{
+return nl_connect ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_nl_nl_connect=yes
+else
+ ac_cv_lib_nl_nl_connect=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nl_nl_connect" >&5
+$as_echo "$ac_cv_lib_nl_nl_connect" >&6; }
+if test "x$ac_cv_lib_nl_nl_connect" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ unset ac_cv_lib_nl_nl_connect
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ usnic_nl_LIBS="-lnl -lm"
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+fi
+ unset fi_check_package_header_happy
+
+
+
+ if test "$fi_check_package_happy" = "yes"; then :
+ usnic_libnl_happy=1
+else
+ usnic_nl_CPPFLAGS="$fi_check_package_usnic_nl_orig_CPPFLAGS"
+ usnic_nl_LDFLAGS="$fi_check_package_usnic_nl_orig_LDFLAGS"
+ usnic_nl_LIBS="$fi_check_package_usnic_nl_orig_LIBS"
+ usnic_libnl_happy=0
+fi
+
+
+
+
+
+ CPPFLAGS="$fi_check_package_usnic_nl_save_CPPFLAGS"
+ LDFLAGS="$fi_check_package_usnic_nl_save_LDFLAGS"
+ LIBS="$fi_check_package_usnic_nl_save_LIBS"
+
+
+ if test $usnic_libnl_happy -eq 1; then :
+ usnic_nl_LIBS="-lnl -lm"
+ HAVE_LIBNL3=0
+fi
+
+fi
+
+ if test "$usnic_want_libnl" = "yes" &&
+ test "$usnic_nl_LIBS" = ""; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --with-libnl specified, but not found" >&5
+$as_echo "$as_me: WARNING: --with-libnl specified, but not found" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+fi
+
+ # Final result
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBNL3 $HAVE_LIBNL3
+_ACEOF
+
+
+ usnic_CPPFLAGS=$usnic_nl_CPPFLAGS
+ usnic_LDFLAGS=$usnic_nl_LDFLAGS
+ usnic_LIBS=$usnic_nl_LIBS
+
+ # If the verbs or usnic providers are being built as a DL,
+ # then we need to add libibverbs to usnic_LIBS. We can tell
+ # if verbs/usnic are being built as DL because fi_provider.m4
+ # will set $PROVIDER_dl to 1. Also, per note in configure.ac,
+ # the verbs provider *must* be configured before the usnic
+ # provider explicitly for this case: so that $verbs_dl will be
+ # (potentially) set by the time we get here.
+
+ # NOTE: this decision whether to -libverbs or not used to be
+ # handled in Makefile.am via an AM_CONDITIONAL. However, to
+ # properly support pkg-config, we have to make this decision
+ # here/now and AC SUBST the final result into usnic_LIBS.
+ if test "$verbs_dl" = "1" || test "$usnic_dl" = "1"; then :
+ usnic_LIBS="$usnic_LIBS -libverbs"
+fi
+
+
+
+
+
+ if test "$usnic_nl_LIBS" = ""; then :
+ usnic_happy=0
+fi
+
+fi
+
+fi
+
+
+
+
+
+
+
+ # See if the provider configured successfully
+ if test $usnic_happy -eq 1; then :
+ PROVIDERS_TO_BUILD="$PROVIDERS_TO_BUILD usnic"
+ PROVIDERS_COUNT=$((PROVIDERS_COUNT+1))
+
+ FI_PC_CFLAGS="$FI_PC_CFLAGS $usnic_CFLAGS $usnic_CPPFLAGS"
+ FI_PC_PRIVATE_LIBS="$FI_PC_PRIVATE_LIBS $usnic_LDFLAGS $usnic_LIBS"
+
+ if test $usnic_dl -eq 1; then :
+ PROVIDERS_DL="prov/usnic/libusnic.la $PROVIDERS_DL"
+ if test x"$enable_static" = x"yes" &&
+ test x"$enable_shared" = x"no"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: usnic provider was selected to be built as DL" >&5
+$as_echo "$as_me: WARNING: usnic provider was selected to be built as DL" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: but libfabric is being built as static-only" >&5
+$as_echo "$as_me: WARNING: but libfabric is being built as static-only" >&2;}
+ as_fn_error $? "This is an impossible situation. Cannot continue." "$LINENO" 5
+fi
+ { $as_echo "$as_me:${as_lineno-$LINENO}: usnic provider: build as plugin" >&5
+$as_echo "$as_me: usnic provider: build as plugin" >&6;}
+
+else
+ PROVIDERS_STATIC="prov/usnic/libusnic.la $PROVIDERS_STATIC"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: usnic provider: include in libfabric" >&5
+$as_echo "$as_me: usnic provider: include in libfabric" >&6;}
+fi
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: usnic provider: disabled" >&5
+$as_echo "$as_me: usnic provider: disabled" >&6;}
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_USNIC $usnic_happy
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_USNIC_DL $usnic_dl
+_ACEOF
+
+
+ # Set AM conditionals for HAVE_<provider> and HAVE_<provider>_DL
+ # as well as AC defines
+ if test $usnic_happy -eq 1; then
+ HAVE_USNIC_TRUE=
+ HAVE_USNIC_FALSE='#'
+else
+ HAVE_USNIC_TRUE='#'
+ HAVE_USNIC_FALSE=
+fi
+
+ if test $usnic_dl -eq 1; then
+ HAVE_USNIC_DL_TRUE=
+ HAVE_USNIC_DL_FALSE='#'
+else
+ HAVE_USNIC_DL_TRUE='#'
+ HAVE_USNIC_DL_FALSE=
+fi
+
+
+ # If this provider was specifically requested but we can't
+ # build it, error.
+ if test "$enable_usnic $usnic_happy" = "yes 0"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: usnic provider was requested, but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: usnic provider was requested, but cannot be compiled" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+ # If this provider was requested for direct build, ensure that
+ # provider's fi_direct.h exists in tree. Error otherwise.
+ if test x"$enable_direct" = x"usnic"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $srcdir/prov/usnic/include/rdma/fi_direct.h" >&5
+$as_echo_n "checking for $srcdir/prov/usnic/include/rdma/fi_direct.h... " >&6; }
+ if test -f "$srcdir/prov/usnic/include/rdma/fi_direct.h"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ as_fn_error $? "usnic provider was requested as direct, but is missing fi_direct.h" "$LINENO" 5
+
+fi
+fi
+
+ # Restore CPPFLAGS/LDFLAGS/LIBS
+ CPPFLAGS=$usnic_orig_CPPFLAGS
+ unset usnic_orig_CPPFLAGS
+ LDFLAGS=$usnic_orig_LDFLAGS
+ unset usnic_orig_LDFLAGS
+ LIBS=$usnic_orig_LIBS
+ unset usnic_orig_LIBS
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: *** Configuring mxm provider" >&5
+$as_echo "$as_me: *** Configuring mxm provider" >&6;}
+ # Check whether --enable-mxm was given.
+if test "${enable_mxm+set}" = set; then :
+ enableval=$enable_mxm;
+else
+ enable_mxm=auto
+fi
+
+
+ # Save CPPFLAGS and LDFLAGS before they are modified by FI_CHECK_PREFIX_DIR.
+ # Provider's local macros could use the value if needed.
+ # Also save LIBS, as a matter of principle.
+ mxm_orig_CPPFLAGS=$CPPFLAGS
+ mxm_orig_LDFLAGS=$LDFLAGS
+ mxm_orig_LIBS=$LIBS
+
+ # Check the --enable-<mxm> value
+ mxm_dl=0
+ case $enable_mxm in #(
+ yes|no) :
+ ;; #(
+ dl) :
+ enable_mxm=yes mxm_dl=1 ;; #(
+ auto) :
+ ;; #(
+ *) :
+
+ # Check that the base directory exists
+ if test ! -d "$enable_mxm"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: supplied directory \"$enable_mxm\" does not exist" >&5
+$as_echo "$as_me: WARNING: supplied directory \"$enable_mxm\" does not exist" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/include exists
+ if test -d "$enable_mxm/include"; then :
+ mxm_PREFIX="$enable_mxm"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"include\" subdirectory in supplied \"$enable_mxm\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"include\" subdirectory in supplied \"$enable_mxm\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/lib or base/lib64 exists
+ if test -d "$enable_mxm/lib"; then :
+ mxm_LIBDIR="$enable_mxm/lib"
+else
+ if test -d "$enable_mxm/lib64"; then :
+ mxm_LIBDIR="$enable_mxm/lib64"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_mxm\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_mxm\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+fi
+
+ enable_mxm=yes
+ ;;
+esac
+
+ # Call the provider's CONFIGURE and CONDITIONALS macros
+
+
+
+
+
+
+ # Determine if we can support the mxm provider
+ mxm_happy=0
+ if test x"$enable_mxm" != x"no"; then :
+
+ fi_check_package_mxm_save_CPPFLAGS="$CPPFLAGS"
+ fi_check_package_mxm_save_LDFLAGS="$LDFLAGS"
+ fi_check_package_mxm_save_LIBS="$LIBS"
+
+ fi_check_package_mxm_orig_CPPFLAGS="$mxm_CPPFLAGS"
+ fi_check_package_mxm_orig_LDFLAGS="$mxm_LDFLAGS"
+ fi_check_package_mxm_orig_LIBS="$mxm_LIBS"
+
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+
+ # There's unfortunately no way to get through the progression of
+ # header includes without killing off the cache variable and
+ # trying again...
+ unset ac_cv_header_mxm_api_mxm_api_h
+
+ fi_check_package_header_happy="no"
+ if test "$mxm_PREFIX" = "/usr" || test "$mxm_PREFIX" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for header without includes" >&5
+$as_echo "looking for header without includes" >&6; }
+ for ac_header in mxm/api/mxm_api.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "mxm/api/mxm_api.h" "ac_cv_header_mxm_api_mxm_api_h" "$ac_includes_default"
+if test "x$ac_cv_header_mxm_api_mxm_api_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_MXM_API_MXM_API_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ # no go on the as is - reset the cache and try again
+ unset ac_cv_header_mxm_api_mxm_api_h
+fi
+fi
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ if test "$mxm_PREFIX" != ""; then :
+ mxm_CPPFLAGS="$mxm_CPPFLAGS -I$mxm_PREFIX/include"
+ CPPFLAGS="$CPPFLAGS -I$mxm_PREFIX/include"
+fi
+ for ac_header in mxm/api/mxm_api.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "mxm/api/mxm_api.h" "ac_cv_header_mxm_api_mxm_api_h" "$ac_includes_default"
+if test "x$ac_cv_header_mxm_api_mxm_api_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_MXM_API_MXM_API_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "yes"; then :
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_mxm_mxm_get_version
+ fi_check_package_lib_happy="no"
+ if test "$mxm_LIBDIR" != ""; then :
+ # libdir was specified - search only there
+ mxm_LDFLAGS="$mxm_LDFLAGS -L$mxm_LIBDIR"
+ LDFLAGS="$LDFLAGS -L$mxm_LIBDIR"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mxm_get_version in -lmxm" >&5
+$as_echo_n "checking for mxm_get_version in -lmxm... " >&6; }
+if test "${ac_cv_lib_mxm_mxm_get_version+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lmxm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char mxm_get_version ();
+int
+main ()
+{
+return mxm_get_version ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_mxm_mxm_get_version=yes
+else
+ ac_cv_lib_mxm_mxm_get_version=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mxm_mxm_get_version" >&5
+$as_echo "$ac_cv_lib_mxm_mxm_get_version" >&6; }
+if test "x$ac_cv_lib_mxm_mxm_get_version" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_mxm_save_LDFLAGS"
+ mxm_LDFLAGS="$fi_check_package_mxm_orig_LDFLAGS"
+ unset ac_cv_lib_mxm_mxm_get_version
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$mxm_PREFIX"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mxm_get_version in -lmxm" >&5
+$as_echo_n "checking for mxm_get_version in -lmxm... " >&6; }
+if test "${ac_cv_lib_mxm_mxm_get_version+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lmxm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char mxm_get_version ();
+int
+main ()
+{
+return mxm_get_version ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_mxm_mxm_get_version=yes
+else
+ ac_cv_lib_mxm_mxm_get_version=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mxm_mxm_get_version" >&5
+$as_echo "$ac_cv_lib_mxm_mxm_get_version" >&6; }
+if test "x$ac_cv_lib_mxm_mxm_get_version" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_mxm_save_LDFLAGS"
+ mxm_LDFLAGS="$fi_check_package_mxm_orig_LDFLAGS"
+ unset ac_cv_lib_mxm_mxm_get_version
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ mxm_LDFLAGS="$mxm_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mxm_get_version in -lmxm" >&5
+$as_echo_n "checking for mxm_get_version in -lmxm... " >&6; }
+if test "${ac_cv_lib_mxm_mxm_get_version+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lmxm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char mxm_get_version ();
+int
+main ()
+{
+return mxm_get_version ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_mxm_mxm_get_version=yes
+else
+ ac_cv_lib_mxm_mxm_get_version=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mxm_mxm_get_version" >&5
+$as_echo "$ac_cv_lib_mxm_mxm_get_version" >&6; }
+if test "x$ac_cv_lib_mxm_mxm_get_version" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_mxm_save_LDFLAGS"
+ mxm_LDFLAGS="$fi_check_package_mxm_orig_LDFLAGS"
+ unset ac_cv_lib_mxm_mxm_get_version
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ mxm_LDFLAGS="$mxm_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mxm_get_version in -lmxm" >&5
+$as_echo_n "checking for mxm_get_version in -lmxm... " >&6; }
+if test "${ac_cv_lib_mxm_mxm_get_version+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lmxm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char mxm_get_version ();
+int
+main ()
+{
+return mxm_get_version ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_mxm_mxm_get_version=yes
+else
+ ac_cv_lib_mxm_mxm_get_version=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mxm_mxm_get_version" >&5
+$as_echo "$ac_cv_lib_mxm_mxm_get_version" >&6; }
+if test "x$ac_cv_lib_mxm_mxm_get_version" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_mxm_save_LDFLAGS"
+ mxm_LDFLAGS="$fi_check_package_mxm_orig_LDFLAGS"
+ unset ac_cv_lib_mxm_mxm_get_version
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ mxm_LIBS="-lmxm "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+else
+ fi_check_package_happy="no"
+fi
+else
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_mxm_mxm_get_version
+ fi_check_package_lib_happy="no"
+ if test "$mxm_LIBDIR" != ""; then :
+ # libdir was specified - search only there
+ mxm_LDFLAGS="$mxm_LDFLAGS -L$mxm_LIBDIR"
+ LDFLAGS="$LDFLAGS -L$mxm_LIBDIR"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mxm_get_version in -lmxm" >&5
+$as_echo_n "checking for mxm_get_version in -lmxm... " >&6; }
+if test "${ac_cv_lib_mxm_mxm_get_version+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lmxm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char mxm_get_version ();
+int
+main ()
+{
+return mxm_get_version ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_mxm_mxm_get_version=yes
+else
+ ac_cv_lib_mxm_mxm_get_version=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mxm_mxm_get_version" >&5
+$as_echo "$ac_cv_lib_mxm_mxm_get_version" >&6; }
+if test "x$ac_cv_lib_mxm_mxm_get_version" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_mxm_save_LDFLAGS"
+ mxm_LDFLAGS="$fi_check_package_mxm_orig_LDFLAGS"
+ unset ac_cv_lib_mxm_mxm_get_version
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir="$mxm_PREFIX"
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mxm_get_version in -lmxm" >&5
+$as_echo_n "checking for mxm_get_version in -lmxm... " >&6; }
+if test "${ac_cv_lib_mxm_mxm_get_version+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lmxm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char mxm_get_version ();
+int
+main ()
+{
+return mxm_get_version ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_mxm_mxm_get_version=yes
+else
+ ac_cv_lib_mxm_mxm_get_version=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mxm_mxm_get_version" >&5
+$as_echo "$ac_cv_lib_mxm_mxm_get_version" >&6; }
+if test "x$ac_cv_lib_mxm_mxm_get_version" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_mxm_save_LDFLAGS"
+ mxm_LDFLAGS="$fi_check_package_mxm_orig_LDFLAGS"
+ unset ac_cv_lib_mxm_mxm_get_version
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ mxm_LDFLAGS="$mxm_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mxm_get_version in -lmxm" >&5
+$as_echo_n "checking for mxm_get_version in -lmxm... " >&6; }
+if test "${ac_cv_lib_mxm_mxm_get_version+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lmxm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char mxm_get_version ();
+int
+main ()
+{
+return mxm_get_version ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_mxm_mxm_get_version=yes
+else
+ ac_cv_lib_mxm_mxm_get_version=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mxm_mxm_get_version" >&5
+$as_echo "$ac_cv_lib_mxm_mxm_get_version" >&6; }
+if test "x$ac_cv_lib_mxm_mxm_get_version" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_mxm_save_LDFLAGS"
+ mxm_LDFLAGS="$fi_check_package_mxm_orig_LDFLAGS"
+ unset ac_cv_lib_mxm_mxm_get_version
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ mxm_LDFLAGS="$mxm_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mxm_get_version in -lmxm" >&5
+$as_echo_n "checking for mxm_get_version in -lmxm... " >&6; }
+if test "${ac_cv_lib_mxm_mxm_get_version+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lmxm $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char mxm_get_version ();
+int
+main ()
+{
+return mxm_get_version ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_mxm_mxm_get_version=yes
+else
+ ac_cv_lib_mxm_mxm_get_version=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mxm_mxm_get_version" >&5
+$as_echo "$ac_cv_lib_mxm_mxm_get_version" >&6; }
+if test "x$ac_cv_lib_mxm_mxm_get_version" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_mxm_save_LDFLAGS"
+ mxm_LDFLAGS="$fi_check_package_mxm_orig_LDFLAGS"
+ unset ac_cv_lib_mxm_mxm_get_version
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ mxm_LIBS="-lmxm "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+fi
+ unset fi_check_package_header_happy
+
+
+
+ if test "$fi_check_package_happy" = "yes"; then :
+ mxm_happy=1
+else
+ mxm_CPPFLAGS="$fi_check_package_mxm_orig_CPPFLAGS"
+ mxm_LDFLAGS="$fi_check_package_mxm_orig_LDFLAGS"
+ mxm_LIBS="$fi_check_package_mxm_orig_LIBS"
+ mxm_happy=0
+fi
+
+
+
+
+
+ CPPFLAGS="$fi_check_package_mxm_save_CPPFLAGS"
+ LDFLAGS="$fi_check_package_mxm_save_LDFLAGS"
+ LIBS="$fi_check_package_mxm_save_LIBS"
+
+
+fi
+ if test $mxm_happy -eq 1; then :
+ mxm_happy=1
+else
+ mxm_happy=0
+fi
+
+
+
+
+
+
+
+ # See if the provider configured successfully
+ if test $mxm_happy -eq 1; then :
+ PROVIDERS_TO_BUILD="$PROVIDERS_TO_BUILD mxm"
+ PROVIDERS_COUNT=$((PROVIDERS_COUNT+1))
+
+ FI_PC_CFLAGS="$FI_PC_CFLAGS $mxm_CFLAGS $mxm_CPPFLAGS"
+ FI_PC_PRIVATE_LIBS="$FI_PC_PRIVATE_LIBS $mxm_LDFLAGS $mxm_LIBS"
+
+ if test $mxm_dl -eq 1; then :
+ PROVIDERS_DL="prov/mxm/libmxm.la $PROVIDERS_DL"
+ if test x"$enable_static" = x"yes" &&
+ test x"$enable_shared" = x"no"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: mxm provider was selected to be built as DL" >&5
+$as_echo "$as_me: WARNING: mxm provider was selected to be built as DL" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: but libfabric is being built as static-only" >&5
+$as_echo "$as_me: WARNING: but libfabric is being built as static-only" >&2;}
+ as_fn_error $? "This is an impossible situation. Cannot continue." "$LINENO" 5
+fi
+ { $as_echo "$as_me:${as_lineno-$LINENO}: mxm provider: build as plugin" >&5
+$as_echo "$as_me: mxm provider: build as plugin" >&6;}
+
+else
+ PROVIDERS_STATIC="prov/mxm/libmxm.la $PROVIDERS_STATIC"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: mxm provider: include in libfabric" >&5
+$as_echo "$as_me: mxm provider: include in libfabric" >&6;}
+fi
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: mxm provider: disabled" >&5
+$as_echo "$as_me: mxm provider: disabled" >&6;}
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_MXM $mxm_happy
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_MXM_DL $mxm_dl
+_ACEOF
+
+
+ # Set AM conditionals for HAVE_<provider> and HAVE_<provider>_DL
+ # as well as AC defines
+ if test $mxm_happy -eq 1; then
+ HAVE_MXM_TRUE=
+ HAVE_MXM_FALSE='#'
+else
+ HAVE_MXM_TRUE='#'
+ HAVE_MXM_FALSE=
+fi
+
+ if test $mxm_dl -eq 1; then
+ HAVE_MXM_DL_TRUE=
+ HAVE_MXM_DL_FALSE='#'
+else
+ HAVE_MXM_DL_TRUE='#'
+ HAVE_MXM_DL_FALSE=
+fi
+
+
+ # If this provider was specifically requested but we can't
+ # build it, error.
+ if test "$enable_mxm $mxm_happy" = "yes 0"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: mxm provider was requested, but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: mxm provider was requested, but cannot be compiled" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+ # If this provider was requested for direct build, ensure that
+ # provider's fi_direct.h exists in tree. Error otherwise.
+ if test x"$enable_direct" = x"mxm"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $srcdir/prov/mxm/include/rdma/fi_direct.h" >&5
+$as_echo_n "checking for $srcdir/prov/mxm/include/rdma/fi_direct.h... " >&6; }
+ if test -f "$srcdir/prov/mxm/include/rdma/fi_direct.h"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ as_fn_error $? "mxm provider was requested as direct, but is missing fi_direct.h" "$LINENO" 5
+
+fi
+fi
+
+ # Restore CPPFLAGS/LDFLAGS/LIBS
+ CPPFLAGS=$mxm_orig_CPPFLAGS
+ unset mxm_orig_CPPFLAGS
+ LDFLAGS=$mxm_orig_LDFLAGS
+ unset mxm_orig_LDFLAGS
+ LIBS=$mxm_orig_LIBS
+ unset mxm_orig_LIBS
+
+
+
+
+
+
+
+
+if test "x$ac_cv_env_FI_PKG_CONFIG_set" != "xset"; then
+ if test -n "$ac_tool_prefix"; then
+ # Extract the first word of "${ac_tool_prefix}pkg-config", so it can be a program name with args.
+set dummy ${ac_tool_prefix}pkg-config; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_path_FI_PKG_CONFIG+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ case $FI_PKG_CONFIG in
+ [\\/]* | ?:[\\/]*)
+ ac_cv_path_FI_PKG_CONFIG="$FI_PKG_CONFIG" # Let the user override the test with a path.
+ ;;
+ *)
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_path_FI_PKG_CONFIG="$as_dir/$ac_word$ac_exec_ext"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+ ;;
+esac
+fi
+FI_PKG_CONFIG=$ac_cv_path_FI_PKG_CONFIG
+if test -n "$FI_PKG_CONFIG"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FI_PKG_CONFIG" >&5
+$as_echo "$FI_PKG_CONFIG" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_path_FI_PKG_CONFIG"; then
+ ac_pt_FI_PKG_CONFIG=$FI_PKG_CONFIG
+ # Extract the first word of "pkg-config", so it can be a program name with args.
+set dummy pkg-config; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_path_ac_pt_FI_PKG_CONFIG+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ case $ac_pt_FI_PKG_CONFIG in
+ [\\/]* | ?:[\\/]*)
+ ac_cv_path_ac_pt_FI_PKG_CONFIG="$ac_pt_FI_PKG_CONFIG" # Let the user override the test with a path.
+ ;;
+ *)
+ as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_path_ac_pt_FI_PKG_CONFIG="$as_dir/$ac_word$ac_exec_ext"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+ ;;
+esac
+fi
+ac_pt_FI_PKG_CONFIG=$ac_cv_path_ac_pt_FI_PKG_CONFIG
+if test -n "$ac_pt_FI_PKG_CONFIG"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_FI_PKG_CONFIG" >&5
+$as_echo "$ac_pt_FI_PKG_CONFIG" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+ if test "x$ac_pt_FI_PKG_CONFIG" = x; then
+ FI_PKG_CONFIG=""
+ else
+ case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+ FI_PKG_CONFIG=$ac_pt_FI_PKG_CONFIG
+ fi
+else
+ FI_PKG_CONFIG="$ac_cv_path_FI_PKG_CONFIG"
+fi
+
+fi
+if test -n "$FI_PKG_CONFIG"; then
+ _pkg_min_version=0.9.0
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking pkg-config is at least version $_pkg_min_version" >&5
+$as_echo_n "checking pkg-config is at least version $_pkg_min_version... " >&6; }
+ if $FI_PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ FI_PKG_CONFIG=""
+ fi
+fi
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: *** Configuring gni provider" >&5
+$as_echo "$as_me: *** Configuring gni provider" >&6;}
+ # Check whether --enable-gni was given.
+if test "${enable_gni+set}" = set; then :
+ enableval=$enable_gni;
+else
+ enable_gni=auto
+fi
+
+
+ # Save CPPFLAGS and LDFLAGS before they are modified by FI_CHECK_PREFIX_DIR.
+ # Provider's local macros could use the value if needed.
+ # Also save LIBS, as a matter of principle.
+ gni_orig_CPPFLAGS=$CPPFLAGS
+ gni_orig_LDFLAGS=$LDFLAGS
+ gni_orig_LIBS=$LIBS
+
+ # Check the --enable-<gni> value
+ gni_dl=0
+ case $enable_gni in #(
+ yes|no) :
+ ;; #(
+ dl) :
+ enable_gni=yes gni_dl=1 ;; #(
+ auto) :
+ ;; #(
+ *) :
+
+ # Check that the base directory exists
+ if test ! -d "$enable_gni"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: supplied directory \"$enable_gni\" does not exist" >&5
+$as_echo "$as_me: WARNING: supplied directory \"$enable_gni\" does not exist" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/include exists
+ if test -d "$enable_gni/include"; then :
+ gni_PREFIX="$enable_gni"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"include\" subdirectory in supplied \"$enable_gni\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"include\" subdirectory in supplied \"$enable_gni\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/lib or base/lib64 exists
+ if test -d "$enable_gni/lib"; then :
+ gni_LIBDIR="$enable_gni/lib"
+else
+ if test -d "$enable_gni/lib64"; then :
+ gni_LIBDIR="$enable_gni/lib64"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_gni\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_gni\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+fi
+
+ enable_gni=yes
+ ;;
+esac
+
+ # Call the provider's CONFIGURE and CONDITIONALS macros
+
+
+# pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*-
+# serial 1 (pkg-config-0.24)
+#
+# Copyright © 2004 Scott James Remnant <scott at netsplit.com>.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# FI_PKG_PROG_PKG_CONFIG([MIN-VERSION])
+# ----------------------------------
+# FI_PKG_PROG_PKG_CONFIG
+
+# FI_PKG_CHECK_EXISTS(MODULES, [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])
+#
+# Check to see whether a particular set of modules exists. Similar
+# to FI_PKG_CHECK_MODULES(), but does not set variables or print errors.
+#
+# Please remember that m4 expands AC_REQUIRE([FI_PKG_PROG_PKG_CONFIG])
+# only at the first occurence in configure.ac, so if the first place
+# it's called might be skipped (such as if it is within an "if", you
+# have to call FI_PKG_CHECK_EXISTS manually
+# --------------------------------------------------------------
+
+
+# _FI_PKG_CONFIG([VARIABLE], [COMMAND], [MODULES])
+# ---------------------------------------------
+# _FI_PKG_CONFIG
+
+# _PKG_SHORT_ERRORS_SUPPORTED
+# -----------------------------
+# _PKG_SHORT_ERRORS_SUPPORTED
+
+
+# FI_PKG_CHECK_MODULES(VARIABLE-PREFIX, MODULES, [ACTION-IF-FOUND],
+# [ACTION-IF-NOT-FOUND])
+#
+#
+# Note that if there is a possibility the first call to
+# FI_PKG_CHECK_MODULES might not happen, you should be sure to include an
+# explicit call to FI_PKG_PROG_PKG_CONFIG in your configure.ac
+#
+#
+# --------------------------------------------------------------
+# FI_PKG_CHECK_MODULES
+
+
+# PKG_INSTALLDIR(DIRECTORY)
+# -------------------------
+# Substitutes the variable pkgconfigdir as the location where a module
+# should install pkg-config .pc files. By default the directory is
+# $libdir/pkgconfig, but the default can be changed by passing
+# DIRECTORY. The user can override through the --with-pkgconfigdir
+# parameter.
+
+
+# PKG_NOARCH_INSTALLDIR(DIRECTORY)
+# -------------------------
+# Substitutes the variable noarch_pkgconfigdir as the location where a
+# module should install arch-independent pkg-config .pc files. By
+# default the directory is $datadir/pkgconfig, but the default can be
+# changed by passing DIRECTORY. The user can override through the
+# --with-noarch-pkgconfigdir parameter.
+
+
+# FI_PKG_CHECK_VAR(VARIABLE, MODULE, CONFIG-VARIABLE,
+# [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])
+# -------------------------------------------
+# Retrieves the value of the pkg-config variable for the given module.
+# FI_PKG_CHECK_VAR
+
+###########################################################################
+###########################################################################
+###########################################################################
+# libfabric project extensions to pkg.m4
+###########################################################################
+###########################################################################
+###########################################################################
+
+# _FI_PKG_CONFIG2([VARIABLE], [COMMAND1], [COMMAND2], [MODULES])
+# ---------------------------------------------
+# _FI_PKG_CONFIG2
+
+# FI_PKG_CHECK_MODULES_STATIC(VARIABLE-PREFIX, MODULES, [ACTION-IF-FOUND],
+# [ACTION-IF-NOT-FOUND])
+#
+#
+# Variation of FI_PGK_CHECK_MODULES which also defines $1_LIBS_STATIC
+# by using the pkg-config --libs --static pkg-config option.
+#
+# --------------------------------------------------------------
+# FI_PKG_CHECK_MODULES_STATIC
+
+
+
+
+
+
+ # Determine if we can support the gni provider
+ # have to pull in pkg.m4 manually
+ ugni_lib_happy=0
+ udreg_lib_happy=0
+ gni_header_happy=0
+ alps_lli_happy=0
+ alps_util_happy=0
+ have_criterion=false
+ criterion_tests_present=true
+ gni_CPPFLAGS=
+ gni_LDFLAGS=
+ gnitest_CPPFLAGS=
+ gnitest_LDFLAGS=
+ gnitest_LIBS=
+
+
+ # Check whether --enable-xpmem was given.
+if test "${enable_xpmem+set}" = set; then :
+ enableval=$enable_xpmem;
+fi
+
+
+ if test x"$enable_gni" != x"no"; then :
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for CRAY_GNI_HEADERS" >&5
+$as_echo_n "checking for CRAY_GNI_HEADERS... " >&6; }
+
+if test -n "$CRAY_GNI_HEADERS_CFLAGS"; then
+ pkg_cv_CRAY_GNI_HEADERS_CFLAGS="$CRAY_GNI_HEADERS_CFLAGS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-gni-headers\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-gni-headers") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_GNI_HEADERS_CFLAGS=`$FI_PKG_CONFIG --cflags "cray-gni-headers" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$CRAY_GNI_HEADERS_LIBS"; then
+ pkg_cv_CRAY_GNI_HEADERS_LIBS="$CRAY_GNI_HEADERS_LIBS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-gni-headers\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-gni-headers") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_GNI_HEADERS_LIBS=`$FI_PKG_CONFIG --libs "cray-gni-headers" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $FI_PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ CRAY_GNI_HEADERS_PKG_ERRORS=`$FI_PKG_CONFIG --short-errors --print-errors --cflags --libs "cray-gni-headers" 2>&1`
+ else
+ CRAY_GNI_HEADERS_PKG_ERRORS=`$FI_PKG_CONFIG --print-errors --cflags --libs "cray-gni-headers" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$CRAY_GNI_HEADERS_PKG_ERRORS" >&5
+
+ gni_header_happy=0
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ gni_header_happy=0
+else
+ CRAY_GNI_HEADERS_CFLAGS=$pkg_cv_CRAY_GNI_HEADERS_CFLAGS
+ CRAY_GNI_HEADERS_LIBS=$pkg_cv_CRAY_GNI_HEADERS_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ gni_header_happy=1
+ gni_CPPFLAGS="$CRAY_GNI_HEADERS_CFLAGS $gni_CPPFLAGS"
+ gni_LDFLAGS="$CRAY_GNI_HEADER_LIBS $gni_LDFLAGS"
+
+fi
+
+fi
+
+ if test "$gni_header_happy" -eq 1; then :
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for CRAY_UGNI" >&5
+$as_echo_n "checking for CRAY_UGNI... " >&6; }
+
+if test -n "$CRAY_UGNI_CFLAGS"; then
+ pkg_cv_CRAY_UGNI_CFLAGS="$CRAY_UGNI_CFLAGS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-ugni\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-ugni") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_UGNI_CFLAGS=`$FI_PKG_CONFIG --cflags "cray-ugni" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$CRAY_UGNI_LIBS"; then
+ pkg_cv_CRAY_UGNI_LIBS="$CRAY_UGNI_LIBS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-ugni\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-ugni") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_UGNI_LIBS=`$FI_PKG_CONFIG --libs "cray-ugni" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $FI_PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ CRAY_UGNI_PKG_ERRORS=`$FI_PKG_CONFIG --short-errors --print-errors --cflags --libs "cray-ugni" 2>&1`
+ else
+ CRAY_UGNI_PKG_ERRORS=`$FI_PKG_CONFIG --print-errors --cflags --libs "cray-ugni" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$CRAY_UGNI_PKG_ERRORS" >&5
+
+ ugni_lib_happy=0
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ ugni_lib_happy=0
+else
+ CRAY_UGNI_CFLAGS=$pkg_cv_CRAY_UGNI_CFLAGS
+ CRAY_UGNI_LIBS=$pkg_cv_CRAY_UGNI_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ ugni_lib_happy=1
+ gni_CPPFLAGS=$CRAY_UGNI_CFLAGS
+ gni_LDFLAGS=$CRAY_UGNI_LIBS
+
+fi
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for CRAY_ALPS_LLI" >&5
+$as_echo_n "checking for CRAY_ALPS_LLI... " >&6; }
+
+if test -n "$CRAY_ALPS_LLI_CFLAGS"; then
+ pkg_cv_CRAY_ALPS_LLI_CFLAGS="$CRAY_ALPS_LLI_CFLAGS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-alpslli\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-alpslli") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_ALPS_LLI_CFLAGS=`$FI_PKG_CONFIG --cflags "cray-alpslli" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$CRAY_ALPS_LLI_LIBS"; then
+ pkg_cv_CRAY_ALPS_LLI_LIBS="$CRAY_ALPS_LLI_LIBS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-alpslli\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-alpslli") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_ALPS_LLI_LIBS=`$FI_PKG_CONFIG --libs "cray-alpslli" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$CRAY_ALPS_LLI_STATIC_LIBS"; then
+ pkg_cv_CRAY_ALPS_LLI_STATIC_LIBS="$CRAY_ALPS_LLI_STATIC_LIBS"
+elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-alpslli\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-alpslli") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_ALPS_LLI_STATIC_LIBS=`$FI_PKG_CONFIG --libs --static "cray-alpslli" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $FI_PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ CRAY_ALPS_LLI_PKG_ERRORS=`$FI_PKG_CONFIG --short-errors --print-errors --cflags --libs "cray-alpslli" 2>&1`
+ else
+ CRAY_ALPS_LLI_PKG_ERRORS=`$FI_PKG_CONFIG --print-errors --cflags --libs "cray-alpslli" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$CRAY_ALPS_LLI_PKG_ERRORS" >&5
+
+ alps_lli_happy=0
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ alps_lli_happy=0
+else
+ CRAY_ALPS_LLI_CFLAGS=$pkg_cv_CRAY_ALPS_LLI_CFLAGS
+ CRAY_ALPS_LLI_LIBS=$pkg_cv_CRAY_ALPS_LLI_LIBS
+ CRAY_ALPS_LLI_STATIC_LIBS=$pkg_cv_CRAY_ALPS_LLI_STATIC_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ alps_lli_happy=1
+ gni_CPPFLAGS="$CRAY_ALPS_LLI_CFLAGS $gni_CPPFLAGS"
+ gni_LDFLAGS="$CRAY_ALPS_LLI_LIBS $gni_LDFLAGS"
+
+fi
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for CRAY_ALPS_UTIL" >&5
+$as_echo_n "checking for CRAY_ALPS_UTIL... " >&6; }
+
+if test -n "$CRAY_ALPS_UTIL_CFLAGS"; then
+ pkg_cv_CRAY_ALPS_UTIL_CFLAGS="$CRAY_ALPS_UTIL_CFLAGS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-alpsutil\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-alpsutil") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_ALPS_UTIL_CFLAGS=`$FI_PKG_CONFIG --cflags "cray-alpsutil" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$CRAY_ALPS_UTIL_LIBS"; then
+ pkg_cv_CRAY_ALPS_UTIL_LIBS="$CRAY_ALPS_UTIL_LIBS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-alpsutil\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-alpsutil") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_ALPS_UTIL_LIBS=`$FI_PKG_CONFIG --libs "cray-alpsutil" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $FI_PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ CRAY_ALPS_UTIL_PKG_ERRORS=`$FI_PKG_CONFIG --short-errors --print-errors --cflags --libs "cray-alpsutil" 2>&1`
+ else
+ CRAY_ALPS_UTIL_PKG_ERRORS=`$FI_PKG_CONFIG --print-errors --cflags --libs "cray-alpsutil" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$CRAY_ALPS_UTIL_PKG_ERRORS" >&5
+
+ alps_util_happy=0
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ alps_util_happy=0
+else
+ CRAY_ALPS_UTIL_CFLAGS=$pkg_cv_CRAY_ALPS_UTIL_CFLAGS
+ CRAY_ALPS_UTIL_LIBS=$pkg_cv_CRAY_ALPS_UTIL_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ alps_util_happy=1
+ gni_CPPFLAGS="$CRAY_ALPS_UTIL_CFLAGS $gni_CPPFLAGS"
+ gni_LDFLAGS="$CRAY_ALPS_UTIL_LIBS $gni_LDFLAGS"
+
+fi
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for CRAY_UDREG" >&5
+$as_echo_n "checking for CRAY_UDREG... " >&6; }
+
+if test -n "$CRAY_UDREG_CFLAGS"; then
+ pkg_cv_CRAY_UDREG_CFLAGS="$CRAY_UDREG_CFLAGS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-udreg\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-udreg") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_UDREG_CFLAGS=`$FI_PKG_CONFIG --cflags "cray-udreg" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$CRAY_UDREG_LIBS"; then
+ pkg_cv_CRAY_UDREG_LIBS="$CRAY_UDREG_LIBS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-udreg\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-udreg") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_UDREG_LIBS=`$FI_PKG_CONFIG --libs "cray-udreg" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $FI_PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ CRAY_UDREG_PKG_ERRORS=`$FI_PKG_CONFIG --short-errors --print-errors --cflags --libs "cray-udreg" 2>&1`
+ else
+ CRAY_UDREG_PKG_ERRORS=`$FI_PKG_CONFIG --print-errors --cflags --libs "cray-udreg" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$CRAY_UDREG_PKG_ERRORS" >&5
+
+ udreg_lib_happy=0
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ udreg_lib_happy=0
+else
+ CRAY_UDREG_CFLAGS=$pkg_cv_CRAY_UDREG_CFLAGS
+ CRAY_UDREG_LIBS=$pkg_cv_CRAY_UDREG_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ udreg_lib_happy=1
+ gni_CPPFLAGS="-DHAVE_UDREG $CRAY_UDREG_CFLAGS $gni_CPPFLAGS"
+ gni_LDFLAGS="$CRAY_UDREG_LIBS $gni_LDFLAGS"
+
+fi
+ if test x"$enable_xpmem" != x"no"; then :
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for CRAY_XPMEM" >&5
+$as_echo_n "checking for CRAY_XPMEM... " >&6; }
+
+if test -n "$CRAY_XPMEM_CFLAGS"; then
+ pkg_cv_CRAY_XPMEM_CFLAGS="$CRAY_XPMEM_CFLAGS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-xpmem\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-xpmem") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_XPMEM_CFLAGS=`$FI_PKG_CONFIG --cflags "cray-xpmem" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$CRAY_XPMEM_LIBS"; then
+ pkg_cv_CRAY_XPMEM_LIBS="$CRAY_XPMEM_LIBS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-xpmem\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-xpmem") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_XPMEM_LIBS=`$FI_PKG_CONFIG --libs "cray-xpmem" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $FI_PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ CRAY_XPMEM_PKG_ERRORS=`$FI_PKG_CONFIG --short-errors --print-errors --cflags --libs "cray-xpmem" 2>&1`
+ else
+ CRAY_XPMEM_PKG_ERRORS=`$FI_PKG_CONFIG --print-errors --cflags --libs "cray-xpmem" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$CRAY_XPMEM_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (cray-xpmem) were not met:
+
+$CRAY_XPMEM_PKG_ERRORS
+
+Consider adjusting the FI_PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables CRAY_XPMEM_CFLAGS
+and CRAY_XPMEM_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the FI_PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables CRAY_XPMEM_CFLAGS
+and CRAY_XPMEM_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/>.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ CRAY_XPMEM_CFLAGS=$pkg_cv_CRAY_XPMEM_CFLAGS
+ CRAY_XPMEM_LIBS=$pkg_cv_CRAY_XPMEM_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_XPMEM 1
+_ACEOF
+
+ gni_CPPFLAGS="$CRAY_XPMEM_CFLAGS $gni_CPPFLAGS"
+ gni_LDFLAGS="$CRAY_XPMEM_LIBS $gni_LDFLAGS"
+
+fi
+
+else
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_XPMEM 0
+_ACEOF
+
+
+fi
+
+ gni_path_to_gni_pub=${CRAY_GNI_HEADERS_CFLAGS:2}
+ gni_path_to_gni_pub=${gni_path_to_gni_pub%?}/gni_pub.h
+ gni_path_to_gni_pub=${CRAY_GNI_HEADERS_CFLAGS:2}
+ gni_path_to_gni_pub=${gni_path_to_gni_pub%?}/gni_pub.h
+
+ ac_fn_c_check_type "$LINENO" "gni_ct_cqw_post_descriptor_t" "ac_cv_type_gni_ct_cqw_post_descriptor_t" "#include \"$gni_path_to_gni_pub\"
+"
+if test "x$ac_cv_type_gni_ct_cqw_post_descriptor_t" = x""yes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_GNI_CT_CQW_POST_DESCRIPTOR_T 1
+_ACEOF
+
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: GNI provider requires CLE 5.2.UP04 or higher. Disabling gni provider." >&5
+$as_echo "$as_me: WARNING: GNI provider requires CLE 5.2.UP04 or higher. Disabling gni provider." >&2;}
+ gni_header_happy=0
+
+fi
+
+
+ if test -d $srcdir/prov/gni/test; then :
+
+# Check whether --with-criterion was given.
+if test "${with_criterion+set}" = set; then :
+ withval=$with_criterion;
+fi
+
+else
+ criterion_tests_present=false
+fi
+
+ if test "$with_criterion" != "" && test "$with_criterion" != "no"; then
+ if test "$enable_direct" != "" && test "$enable_direct" != "no"; then
+ gnitest_CPPFLAGS="-I$srcdir/prov/gni/include"
+ fi
+
+ if test "$criterion_tests_present" = "true"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking criterion path" >&5
+$as_echo_n "checking criterion path... " >&6; }
+ if test -d "$with_criterion"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ gnitest_CPPFLAGS="-I$with_criterion/include -DHAVE_UDREG $CRAY_UDREG_CFLAGS $gnitest_CPPFLAGS"
+ gnitest_LIBS="-lcriterion -ludreg $gnitest_LIBS"
+
+ if test -d "$with_criterion/lib"; then
+ gnitest_LDFLAGS="$CRAY_ALPS_LLI_STATIC_LIBS -L$with_criterion/lib -Wl,-rpath=$with_criterion/lib $gnitest_LDFLAGS"
+ have_criterion=true
+ elif test -d "$with_criterion/lib64"; then
+ gnitest_LDFLAGS="$CRAY_ALPS_LLI_STATIC_LIBS -L$with_criterion/lib64 -Wl,-rpath=$with_criterion/lib64 $gnitest_LDFLAGS"
+ have_criterion=true
+ else
+ have_criterion=false
+ fi
+
+ gnitest_LDFLAGS="$CRAY_UDREG_LIBS $gnitest_LDFLAGS"
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for CRAY_PMI" >&5
+$as_echo_n "checking for CRAY_PMI... " >&6; }
+
+if test -n "$CRAY_PMI_CFLAGS"; then
+ pkg_cv_CRAY_PMI_CFLAGS="$CRAY_PMI_CFLAGS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-pmi\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-pmi") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_PMI_CFLAGS=`$FI_PKG_CONFIG --cflags "cray-pmi" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$CRAY_PMI_LIBS"; then
+ pkg_cv_CRAY_PMI_LIBS="$CRAY_PMI_LIBS"
+ elif test -n "$FI_PKG_CONFIG"; then
+ if test -n "$FI_PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$FI_PKG_CONFIG --exists --print-errors \"cray-pmi\""; } >&5
+ ($FI_PKG_CONFIG --exists --print-errors "cray-pmi") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_CRAY_PMI_LIBS=`$FI_PKG_CONFIG --libs "cray-pmi" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $FI_PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ CRAY_PMI_PKG_ERRORS=`$FI_PKG_CONFIG --short-errors --print-errors --cflags --libs "cray-pmi" 2>&1`
+ else
+ CRAY_PMI_PKG_ERRORS=`$FI_PKG_CONFIG --print-errors --cflags --libs "cray-pmi" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$CRAY_PMI_PKG_ERRORS" >&5
+
+ have_criterion=false
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ have_criterion=false
+else
+ CRAY_PMI_CFLAGS=$pkg_cv_CRAY_PMI_CFLAGS
+ CRAY_PMI_LIBS=$pkg_cv_CRAY_PMI_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ as_fn_error $? "criterion requested but invalid path given" "$LINENO" 5
+ fi
+else
+ as_fn_error $? "criterion requested tests not available" "$LINENO" 5
+fi
+ fi
+
+ ac_fn_c_check_decl "$LINENO" "HAVE_ATOMICS" "ac_cv_have_decl_HAVE_ATOMICS" "$ac_includes_default"
+if test "x$ac_cv_have_decl_HAVE_ATOMICS" = x""yes; then :
+
+else
+ cc_version=`$CC --version | head -n1`
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: \"$cc_version\" doesn't support native atomics. Disabling GNI provider." >&5
+$as_echo "$as_me: WARNING: \"$cc_version\" doesn't support native atomics. Disabling GNI provider." >&2;}
+ ugni_lib_happy=0
+fi
+
+
+
+fi
+
+
+# Check whether --with-kdreg was given.
+if test "${with_kdreg+set}" = set; then :
+ withval=$with_kdreg;
+fi
+
+
+ if test "$with_kdreg" != "" && test "$with_kdreg" != "no"; then
+ gni_CPPFLAGS="-I$with_kdreg/include -DHAVE_KDREG $gni_CPPFLAGS"
+ fi
+
+
+ if test "x$have_criterion" = "xtrue"; then
+ HAVE_CRITERION_TRUE=
+ HAVE_CRITERION_FALSE='#'
+else
+ HAVE_CRITERION_TRUE='#'
+ HAVE_CRITERION_FALSE=
+fi
+
+
+
+
+
+
+
+
+ if test $gni_header_happy -eq 1 -a $ugni_lib_happy -eq 1 \
+ -a $alps_lli_happy -eq 1 -a $alps_util_happy -eq 1 \
+ -a $udreg_lib_happy -eq 1; then :
+ gni_happy=1
+else
+ gni_happy=0
+fi
+
+
+
+
+
+
+
+ # See if the provider configured successfully
+ if test $gni_happy -eq 1; then :
+ PROVIDERS_TO_BUILD="$PROVIDERS_TO_BUILD gni"
+ PROVIDERS_COUNT=$((PROVIDERS_COUNT+1))
+
+ FI_PC_CFLAGS="$FI_PC_CFLAGS $gni_CFLAGS $gni_CPPFLAGS"
+ FI_PC_PRIVATE_LIBS="$FI_PC_PRIVATE_LIBS $gni_LDFLAGS $gni_LIBS"
+
+ if test $gni_dl -eq 1; then :
+ PROVIDERS_DL="prov/gni/libgni.la $PROVIDERS_DL"
+ if test x"$enable_static" = x"yes" &&
+ test x"$enable_shared" = x"no"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: gni provider was selected to be built as DL" >&5
+$as_echo "$as_me: WARNING: gni provider was selected to be built as DL" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: but libfabric is being built as static-only" >&5
+$as_echo "$as_me: WARNING: but libfabric is being built as static-only" >&2;}
+ as_fn_error $? "This is an impossible situation. Cannot continue." "$LINENO" 5
+fi
+ { $as_echo "$as_me:${as_lineno-$LINENO}: gni provider: build as plugin" >&5
+$as_echo "$as_me: gni provider: build as plugin" >&6;}
+
+else
+ PROVIDERS_STATIC="prov/gni/libgni.la $PROVIDERS_STATIC"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: gni provider: include in libfabric" >&5
+$as_echo "$as_me: gni provider: include in libfabric" >&6;}
+fi
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: gni provider: disabled" >&5
+$as_echo "$as_me: gni provider: disabled" >&6;}
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_GNI $gni_happy
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_GNI_DL $gni_dl
+_ACEOF
+
+
+ # Set AM conditionals for HAVE_<provider> and HAVE_<provider>_DL
+ # as well as AC defines
+ if test $gni_happy -eq 1; then
+ HAVE_GNI_TRUE=
+ HAVE_GNI_FALSE='#'
+else
+ HAVE_GNI_TRUE='#'
+ HAVE_GNI_FALSE=
+fi
+
+ if test $gni_dl -eq 1; then
+ HAVE_GNI_DL_TRUE=
+ HAVE_GNI_DL_FALSE='#'
+else
+ HAVE_GNI_DL_TRUE='#'
+ HAVE_GNI_DL_FALSE=
+fi
+
+
+ # If this provider was specifically requested but we can't
+ # build it, error.
+ if test "$enable_gni $gni_happy" = "yes 0"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: gni provider was requested, but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: gni provider was requested, but cannot be compiled" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+ # If this provider was requested for direct build, ensure that
+ # provider's fi_direct.h exists in tree. Error otherwise.
+ if test x"$enable_direct" = x"gni"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $srcdir/prov/gni/include/rdma/fi_direct.h" >&5
+$as_echo_n "checking for $srcdir/prov/gni/include/rdma/fi_direct.h... " >&6; }
+ if test -f "$srcdir/prov/gni/include/rdma/fi_direct.h"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ as_fn_error $? "gni provider was requested as direct, but is missing fi_direct.h" "$LINENO" 5
+
+fi
+fi
+
+ # Restore CPPFLAGS/LDFLAGS/LIBS
+ CPPFLAGS=$gni_orig_CPPFLAGS
+ unset gni_orig_CPPFLAGS
+ LDFLAGS=$gni_orig_LDFLAGS
+ unset gni_orig_LDFLAGS
+ LIBS=$gni_orig_LIBS
+ unset gni_orig_LIBS
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: *** Configuring udp provider" >&5
+$as_echo "$as_me: *** Configuring udp provider" >&6;}
+ # Check whether --enable-udp was given.
+if test "${enable_udp+set}" = set; then :
+ enableval=$enable_udp;
+else
+ enable_udp=auto
+fi
+
+
+ # Save CPPFLAGS and LDFLAGS before they are modified by FI_CHECK_PREFIX_DIR.
+ # Provider's local macros could use the value if needed.
+ # Also save LIBS, as a matter of principle.
+ udp_orig_CPPFLAGS=$CPPFLAGS
+ udp_orig_LDFLAGS=$LDFLAGS
+ udp_orig_LIBS=$LIBS
+
+ # Check the --enable-<udp> value
+ udp_dl=0
+ case $enable_udp in #(
+ yes|no) :
+ ;; #(
+ dl) :
+ enable_udp=yes udp_dl=1 ;; #(
+ auto) :
+ ;; #(
+ *) :
+
+ # Check that the base directory exists
+ if test ! -d "$enable_udp"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: supplied directory \"$enable_udp\" does not exist" >&5
+$as_echo "$as_me: WARNING: supplied directory \"$enable_udp\" does not exist" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/include exists
+ if test -d "$enable_udp/include"; then :
+ udp_PREFIX="$enable_udp"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"include\" subdirectory in supplied \"$enable_udp\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"include\" subdirectory in supplied \"$enable_udp\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/lib or base/lib64 exists
+ if test -d "$enable_udp/lib"; then :
+ udp_LIBDIR="$enable_udp/lib"
+else
+ if test -d "$enable_udp/lib64"; then :
+ udp_LIBDIR="$enable_udp/lib64"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_udp\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_udp\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+fi
+
+ enable_udp=yes
+ ;;
+esac
+
+ # Call the provider's CONFIGURE and CONDITIONALS macros
+
+
+
+
+
+ # Determine if we can support the udp provider
+ udp_h_happy=0
+ udp_shm_happy=0
+ if test x"$enable_udp" != x"no"; then :
+ ac_fn_c_check_header_mongrel "$LINENO" "sys/socket.h" "ac_cv_header_sys_socket_h" "$ac_includes_default"
+if test "x$ac_cv_header_sys_socket_h" = x""yes; then :
+ udp_h_happy=1
+else
+ udp_h_happy=0
+fi
+
+
+
+
+ # check if shm_open is already present
+ ac_fn_c_check_func "$LINENO" "shm_open" "ac_cv_func_shm_open"
+if test "x$ac_cv_func_shm_open" = x""yes; then :
+ udp_shm_happy=1
+else
+ udp_shm_happy=0
+fi
+
+
+ # look for shm_open in librt if not already present
+ if test $udp_shm_happy -eq 0; then :
+
+ fi_check_package_udp_shm_save_CPPFLAGS="$CPPFLAGS"
+ fi_check_package_udp_shm_save_LDFLAGS="$LDFLAGS"
+ fi_check_package_udp_shm_save_LIBS="$LIBS"
+
+ fi_check_package_udp_shm_orig_CPPFLAGS="$udp_shm_CPPFLAGS"
+ fi_check_package_udp_shm_orig_LDFLAGS="$udp_shm_LDFLAGS"
+ fi_check_package_udp_shm_orig_LIBS="$udp_shm_LIBS"
+
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+
+ # There's unfortunately no way to get through the progression of
+ # header includes without killing off the cache variable and
+ # trying again...
+ unset ac_cv_header_sys_mman_h
+
+ fi_check_package_header_happy="no"
+ if test "" = "/usr" || test "" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for header without includes" >&5
+$as_echo "looking for header without includes" >&6; }
+ for ac_header in sys/mman.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "sys/mman.h" "ac_cv_header_sys_mman_h" "$ac_includes_default"
+if test "x$ac_cv_header_sys_mman_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_SYS_MMAN_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ # no go on the as is - reset the cache and try again
+ unset ac_cv_header_sys_mman_h
+fi
+fi
+
+ if test "$fi_check_package_header_happy" = "no"; then :
+ if test "" != ""; then :
+ udp_shm_CPPFLAGS="$udp_shm_CPPFLAGS -I/include"
+ CPPFLAGS="$CPPFLAGS -I/include"
+fi
+ for ac_header in sys/mman.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "sys/mman.h" "ac_cv_header_sys_mman_h" "$ac_includes_default"
+if test "x$ac_cv_header_sys_mman_h" = x""yes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_SYS_MMAN_H 1
+_ACEOF
+ fi_check_package_header_happy="yes"
+fi
+
+done
+
+ if test "$fi_check_package_header_happy" = "yes"; then :
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_rt_shm_open
+ fi_check_package_lib_happy="no"
+ if test "" != ""; then :
+ # libdir was specified - search only there
+ udp_shm_LDFLAGS="$udp_shm_LDFLAGS -L"
+ LDFLAGS="$LDFLAGS -L"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_udp_shm_save_LDFLAGS"
+ udp_shm_LDFLAGS="$fi_check_package_udp_shm_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir=""
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_udp_shm_save_LDFLAGS"
+ udp_shm_LDFLAGS="$fi_check_package_udp_shm_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ udp_shm_LDFLAGS="$udp_shm_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_udp_shm_save_LDFLAGS"
+ udp_shm_LDFLAGS="$fi_check_package_udp_shm_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ udp_shm_LDFLAGS="$udp_shm_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_udp_shm_save_LDFLAGS"
+ udp_shm_LDFLAGS="$fi_check_package_udp_shm_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ udp_shm_LIBS="-lrt "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+else
+ fi_check_package_happy="no"
+fi
+else
+
+ # This is stolen from autoconf to peek under the covers to get the
+ # cache variable for the library check.
+
+ # See comment above
+ unset ac_cv_lib_rt_shm_open
+ fi_check_package_lib_happy="no"
+ if test "" != ""; then :
+ # libdir was specified - search only there
+ udp_shm_LDFLAGS="$udp_shm_LDFLAGS -L"
+ LDFLAGS="$LDFLAGS -L"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ LDFLAGS="$fi_check_package_udp_shm_save_LDFLAGS"
+ udp_shm_LDFLAGS="$fi_check_package_udp_shm_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+else
+ # libdir was not specified - go through search path
+ fi_check_package_libdir=""
+ if test "$fi_check_package_libdir" = "" || \
+ test "$fi_check_package_libdir" = "/usr" || \
+ test "$fi_check_package_libdir" = "/usr/local"; then :
+ # try as is...
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library without search path" >&5
+$as_echo "looking for library without search path" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_udp_shm_save_LDFLAGS"
+ udp_shm_LDFLAGS="$fi_check_package_udp_shm_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ udp_shm_LDFLAGS="$udp_shm_LDFLAGS -L$fi_check_package_libdir/lib"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib" >&5
+$as_echo "looking for library in lib" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_udp_shm_save_LDFLAGS"
+ udp_shm_LDFLAGS="$fi_check_package_udp_shm_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ if test "$fi_check_package_libdir" != ""; then :
+ udp_shm_LDFLAGS="$udp_shm_LDFLAGS -L$fi_check_package_libdir/lib64"
+ LDFLAGS="$LDFLAGS -L$fi_check_package_libdir/lib64"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: looking for library in lib64" >&5
+$as_echo "looking for library in lib64" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shm_open in -lrt" >&5
+$as_echo_n "checking for shm_open in -lrt... " >&6; }
+if test "${ac_cv_lib_rt_shm_open+set}" = set; then :
+ $as_echo_n "(cached) " >&6
+else
+ ac_check_lib_save_LIBS=$LIBS
+LIBS="-lrt $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char shm_open ();
+int
+main ()
+{
+return shm_open ();
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_lib_rt_shm_open=yes
+else
+ ac_cv_lib_rt_shm_open=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rt_shm_open" >&5
+$as_echo "$ac_cv_lib_rt_shm_open" >&6; }
+if test "x$ac_cv_lib_rt_shm_open" = x""yes; then :
+ fi_check_package_lib_happy="yes"
+else
+ fi_check_package_lib_happy="no"
+fi
+
+ if test "$fi_check_package_lib_happy" = "no"; then :
+ # no go on the as is.. see what happens later...
+ LDFLAGS="$fi_check_package_udp_shm_save_LDFLAGS"
+ udp_shm_LDFLAGS="$fi_check_package_udp_shm_orig_LDFLAGS"
+ unset ac_cv_lib_rt_shm_open
+fi
+fi
+fi
+fi
+
+ if test "$fi_check_package_lib_happy" = "yes"; then :
+ udp_shm_LIBS="-lrt "
+ fi_check_package_happy="yes"
+else
+ fi_check_package_happy="no"
+fi
+
+
+fi
+ unset fi_check_package_header_happy
+
+
+
+ if test "$fi_check_package_happy" = "yes"; then :
+ udp_shm_happy=1
+else
+ udp_shm_CPPFLAGS="$fi_check_package_udp_shm_orig_CPPFLAGS"
+ udp_shm_LDFLAGS="$fi_check_package_udp_shm_orig_LDFLAGS"
+ udp_shm_LIBS="$fi_check_package_udp_shm_orig_LIBS"
+ udp_shm_happy=0
+fi
+
+
+
+
+
+ CPPFLAGS="$fi_check_package_udp_shm_save_CPPFLAGS"
+ LDFLAGS="$fi_check_package_udp_shm_save_LDFLAGS"
+ LIBS="$fi_check_package_udp_shm_save_LIBS"
+
+fi
+
+fi
+
+ if test $udp_h_happy -eq 1 && \
+ test $udp_shm_happy -eq 1; then :
+ udp_happy=1
+else
+ udp_happy=0
+fi
+
+
+
+
+
+
+
+ # See if the provider configured successfully
+ if test $udp_happy -eq 1; then :
+ PROVIDERS_TO_BUILD="$PROVIDERS_TO_BUILD udp"
+ PROVIDERS_COUNT=$((PROVIDERS_COUNT+1))
+
+ FI_PC_CFLAGS="$FI_PC_CFLAGS $udp_CFLAGS $udp_CPPFLAGS"
+ FI_PC_PRIVATE_LIBS="$FI_PC_PRIVATE_LIBS $udp_LDFLAGS $udp_LIBS"
+
+ if test $udp_dl -eq 1; then :
+ PROVIDERS_DL="prov/udp/libudp.la $PROVIDERS_DL"
+ if test x"$enable_static" = x"yes" &&
+ test x"$enable_shared" = x"no"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: udp provider was selected to be built as DL" >&5
+$as_echo "$as_me: WARNING: udp provider was selected to be built as DL" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: but libfabric is being built as static-only" >&5
+$as_echo "$as_me: WARNING: but libfabric is being built as static-only" >&2;}
+ as_fn_error $? "This is an impossible situation. Cannot continue." "$LINENO" 5
+fi
+ { $as_echo "$as_me:${as_lineno-$LINENO}: udp provider: build as plugin" >&5
+$as_echo "$as_me: udp provider: build as plugin" >&6;}
+
+else
+ PROVIDERS_STATIC="prov/udp/libudp.la $PROVIDERS_STATIC"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: udp provider: include in libfabric" >&5
+$as_echo "$as_me: udp provider: include in libfabric" >&6;}
+fi
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: udp provider: disabled" >&5
+$as_echo "$as_me: udp provider: disabled" >&6;}
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_UDP $udp_happy
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_UDP_DL $udp_dl
+_ACEOF
+
+
+ # Set AM conditionals for HAVE_<provider> and HAVE_<provider>_DL
+ # as well as AC defines
+ if test $udp_happy -eq 1; then
+ HAVE_UDP_TRUE=
+ HAVE_UDP_FALSE='#'
+else
+ HAVE_UDP_TRUE='#'
+ HAVE_UDP_FALSE=
+fi
+
+ if test $udp_dl -eq 1; then
+ HAVE_UDP_DL_TRUE=
+ HAVE_UDP_DL_FALSE='#'
+else
+ HAVE_UDP_DL_TRUE='#'
+ HAVE_UDP_DL_FALSE=
+fi
+
+
+ # If this provider was specifically requested but we can't
+ # build it, error.
+ if test "$enable_udp $udp_happy" = "yes 0"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: udp provider was requested, but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: udp provider was requested, but cannot be compiled" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+ # If this provider was requested for direct build, ensure that
+ # provider's fi_direct.h exists in tree. Error otherwise.
+ if test x"$enable_direct" = x"udp"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $srcdir/prov/udp/include/rdma/fi_direct.h" >&5
+$as_echo_n "checking for $srcdir/prov/udp/include/rdma/fi_direct.h... " >&6; }
+ if test -f "$srcdir/prov/udp/include/rdma/fi_direct.h"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ as_fn_error $? "udp provider was requested as direct, but is missing fi_direct.h" "$LINENO" 5
+
+fi
+fi
+
+ # Restore CPPFLAGS/LDFLAGS/LIBS
+ CPPFLAGS=$udp_orig_CPPFLAGS
+ unset udp_orig_CPPFLAGS
+ LDFLAGS=$udp_orig_LDFLAGS
+ unset udp_orig_LDFLAGS
+ LIBS=$udp_orig_LIBS
+ unset udp_orig_LIBS
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: *** Configuring rxm provider" >&5
+$as_echo "$as_me: *** Configuring rxm provider" >&6;}
+ # Check whether --enable-rxm was given.
+if test "${enable_rxm+set}" = set; then :
+ enableval=$enable_rxm;
+else
+ enable_rxm=auto
+fi
+
+
+ # Save CPPFLAGS and LDFLAGS before they are modified by FI_CHECK_PREFIX_DIR.
+ # Provider's local macros could use the value if needed.
+ # Also save LIBS, as a matter of principle.
+ rxm_orig_CPPFLAGS=$CPPFLAGS
+ rxm_orig_LDFLAGS=$LDFLAGS
+ rxm_orig_LIBS=$LIBS
+
+ # Check the --enable-<rxm> value
+ rxm_dl=0
+ case $enable_rxm in #(
+ yes|no) :
+ ;; #(
+ dl) :
+ enable_rxm=yes rxm_dl=1 ;; #(
+ auto) :
+ ;; #(
+ *) :
+
+ # Check that the base directory exists
+ if test ! -d "$enable_rxm"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: supplied directory \"$enable_rxm\" does not exist" >&5
+$as_echo "$as_me: WARNING: supplied directory \"$enable_rxm\" does not exist" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/include exists
+ if test -d "$enable_rxm/include"; then :
+ rxm_PREFIX="$enable_rxm"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"include\" subdirectory in supplied \"$enable_rxm\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"include\" subdirectory in supplied \"$enable_rxm\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/lib or base/lib64 exists
+ if test -d "$enable_rxm/lib"; then :
+ rxm_LIBDIR="$enable_rxm/lib"
+else
+ if test -d "$enable_rxm/lib64"; then :
+ rxm_LIBDIR="$enable_rxm/lib64"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_rxm\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_rxm\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+fi
+
+ enable_rxm=yes
+ ;;
+esac
+
+ # Call the provider's CONFIGURE and CONDITIONALS macros
+
+
+
+
+
+
+ # Determine if we can support the rxm provider
+ rxm_h_happy=0
+ if test x"$enable_rxm" != x"no"; then :
+ rxm_h_happy=1
+fi
+ if test $rxm_h_happy -eq 1; then :
+ rxm_happy=1
+else
+ rxm_happy=0
+fi
+
+
+
+
+
+
+
+ # See if the provider configured successfully
+ if test $rxm_happy -eq 1; then :
+ PROVIDERS_TO_BUILD="$PROVIDERS_TO_BUILD rxm"
+ PROVIDERS_COUNT=$((PROVIDERS_COUNT+1))
+
+ FI_PC_CFLAGS="$FI_PC_CFLAGS $rxm_CFLAGS $rxm_CPPFLAGS"
+ FI_PC_PRIVATE_LIBS="$FI_PC_PRIVATE_LIBS $rxm_LDFLAGS $rxm_LIBS"
+
+ if test $rxm_dl -eq 1; then :
+ PROVIDERS_DL="prov/rxm/librxm.la $PROVIDERS_DL"
+ if test x"$enable_static" = x"yes" &&
+ test x"$enable_shared" = x"no"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: rxm provider was selected to be built as DL" >&5
+$as_echo "$as_me: WARNING: rxm provider was selected to be built as DL" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: but libfabric is being built as static-only" >&5
+$as_echo "$as_me: WARNING: but libfabric is being built as static-only" >&2;}
+ as_fn_error $? "This is an impossible situation. Cannot continue." "$LINENO" 5
+fi
+ { $as_echo "$as_me:${as_lineno-$LINENO}: rxm provider: build as plugin" >&5
+$as_echo "$as_me: rxm provider: build as plugin" >&6;}
+
+else
+ PROVIDERS_STATIC="prov/rxm/librxm.la $PROVIDERS_STATIC"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: rxm provider: include in libfabric" >&5
+$as_echo "$as_me: rxm provider: include in libfabric" >&6;}
+fi
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: rxm provider: disabled" >&5
+$as_echo "$as_me: rxm provider: disabled" >&6;}
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_RXM $rxm_happy
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_RXM_DL $rxm_dl
+_ACEOF
+
+
+ # Set AM conditionals for HAVE_<provider> and HAVE_<provider>_DL
+ # as well as AC defines
+ if test $rxm_happy -eq 1; then
+ HAVE_RXM_TRUE=
+ HAVE_RXM_FALSE='#'
+else
+ HAVE_RXM_TRUE='#'
+ HAVE_RXM_FALSE=
+fi
+
+ if test $rxm_dl -eq 1; then
+ HAVE_RXM_DL_TRUE=
+ HAVE_RXM_DL_FALSE='#'
+else
+ HAVE_RXM_DL_TRUE='#'
+ HAVE_RXM_DL_FALSE=
+fi
+
+
+ # If this provider was specifically requested but we can't
+ # build it, error.
+ if test "$enable_rxm $rxm_happy" = "yes 0"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: rxm provider was requested, but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: rxm provider was requested, but cannot be compiled" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+ # If this provider was requested for direct build, ensure that
+ # provider's fi_direct.h exists in tree. Error otherwise.
+ if test x"$enable_direct" = x"rxm"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $srcdir/prov/rxm/include/rdma/fi_direct.h" >&5
+$as_echo_n "checking for $srcdir/prov/rxm/include/rdma/fi_direct.h... " >&6; }
+ if test -f "$srcdir/prov/rxm/include/rdma/fi_direct.h"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ as_fn_error $? "rxm provider was requested as direct, but is missing fi_direct.h" "$LINENO" 5
+
+fi
+fi
+
+ # Restore CPPFLAGS/LDFLAGS/LIBS
+ CPPFLAGS=$rxm_orig_CPPFLAGS
+ unset rxm_orig_CPPFLAGS
+ LDFLAGS=$rxm_orig_LDFLAGS
+ unset rxm_orig_LDFLAGS
+ LIBS=$rxm_orig_LIBS
+ unset rxm_orig_LIBS
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: *** Configuring rxd provider" >&5
+$as_echo "$as_me: *** Configuring rxd provider" >&6;}
+ # Check whether --enable-rxd was given.
+if test "${enable_rxd+set}" = set; then :
+ enableval=$enable_rxd;
+else
+ enable_rxd=auto
+fi
+
+
+ # Save CPPFLAGS and LDFLAGS before they are modified by FI_CHECK_PREFIX_DIR.
+ # Provider's local macros could use the value if needed.
+ # Also save LIBS, as a matter of principle.
+ rxd_orig_CPPFLAGS=$CPPFLAGS
+ rxd_orig_LDFLAGS=$LDFLAGS
+ rxd_orig_LIBS=$LIBS
+
+ # Check the --enable-<rxd> value
+ rxd_dl=0
+ case $enable_rxd in #(
+ yes|no) :
+ ;; #(
+ dl) :
+ enable_rxd=yes rxd_dl=1 ;; #(
+ auto) :
+ ;; #(
+ *) :
+
+ # Check that the base directory exists
+ if test ! -d "$enable_rxd"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: supplied directory \"$enable_rxd\" does not exist" >&5
+$as_echo "$as_me: WARNING: supplied directory \"$enable_rxd\" does not exist" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/include exists
+ if test -d "$enable_rxd/include"; then :
+ rxd_PREFIX="$enable_rxd"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"include\" subdirectory in supplied \"$enable_rxd\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"include\" subdirectory in supplied \"$enable_rxd\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+ # Check that base/lib or base/lib64 exists
+ if test -d "$enable_rxd/lib"; then :
+ rxd_LIBDIR="$enable_rxd/lib"
+else
+ if test -d "$enable_rxd/lib64"; then :
+ rxd_LIBDIR="$enable_rxd/lib64"
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_rxd\" directory\"" >&5
+$as_echo "$as_me: WARNING: could not find \"lib\" or \"lib64\" subdirectories in supplied \"$enable_rxd\" directory\"" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+
+fi
+
+ enable_rxd=yes
+ ;;
+esac
+
+ # Call the provider's CONFIGURE and CONDITIONALS macros
+
+
+
+
+
+ # Determine if we can support the rxd provider
+ rxd_h_happy=0
+ if test x"$enable_rxd" != x"no"; then :
+ rxd_h_happy=1
+fi
+ if test $rxd_h_happy -eq 1; then :
+ rxd_happy=1
+else
+ rxd_happy=0
+fi
+
+
+
+
+
+
+
+ # See if the provider configured successfully
+ if test $rxd_happy -eq 1; then :
+ PROVIDERS_TO_BUILD="$PROVIDERS_TO_BUILD rxd"
+ PROVIDERS_COUNT=$((PROVIDERS_COUNT+1))
+
+ FI_PC_CFLAGS="$FI_PC_CFLAGS $rxd_CFLAGS $rxd_CPPFLAGS"
+ FI_PC_PRIVATE_LIBS="$FI_PC_PRIVATE_LIBS $rxd_LDFLAGS $rxd_LIBS"
+
+ if test $rxd_dl -eq 1; then :
+ PROVIDERS_DL="prov/rxd/librxd.la $PROVIDERS_DL"
+ if test x"$enable_static" = x"yes" &&
+ test x"$enable_shared" = x"no"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: rxd provider was selected to be built as DL" >&5
+$as_echo "$as_me: WARNING: rxd provider was selected to be built as DL" >&2;}
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: but libfabric is being built as static-only" >&5
+$as_echo "$as_me: WARNING: but libfabric is being built as static-only" >&2;}
+ as_fn_error $? "This is an impossible situation. Cannot continue." "$LINENO" 5
+fi
+ { $as_echo "$as_me:${as_lineno-$LINENO}: rxd provider: build as plugin" >&5
+$as_echo "$as_me: rxd provider: build as plugin" >&6;}
+
+else
+ PROVIDERS_STATIC="prov/rxd/librxd.la $PROVIDERS_STATIC"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: rxd provider: include in libfabric" >&5
+$as_echo "$as_me: rxd provider: include in libfabric" >&6;}
+fi
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: rxd provider: disabled" >&5
+$as_echo "$as_me: rxd provider: disabled" >&6;}
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_RXD $rxd_happy
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_RXD_DL $rxd_dl
+_ACEOF
+
+
+ # Set AM conditionals for HAVE_<provider> and HAVE_<provider>_DL
+ # as well as AC defines
+ if test $rxd_happy -eq 1; then
+ HAVE_RXD_TRUE=
+ HAVE_RXD_FALSE='#'
+else
+ HAVE_RXD_TRUE='#'
+ HAVE_RXD_FALSE=
+fi
+
+ if test $rxd_dl -eq 1; then
+ HAVE_RXD_DL_TRUE=
+ HAVE_RXD_DL_FALSE='#'
+else
+ HAVE_RXD_DL_TRUE='#'
+ HAVE_RXD_DL_FALSE=
+fi
+
+
+ # If this provider was specifically requested but we can't
+ # build it, error.
+ if test "$enable_rxd $rxd_happy" = "yes 0"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: rxd provider was requested, but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: rxd provider was requested, but cannot be compiled" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+
+fi
+ # If this provider was requested for direct build, ensure that
+ # provider's fi_direct.h exists in tree. Error otherwise.
+ if test x"$enable_direct" = x"rxd"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $srcdir/prov/rxd/include/rdma/fi_direct.h" >&5
+$as_echo_n "checking for $srcdir/prov/rxd/include/rdma/fi_direct.h... " >&6; }
+ if test -f "$srcdir/prov/rxd/include/rdma/fi_direct.h"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ as_fn_error $? "rxd provider was requested as direct, but is missing fi_direct.h" "$LINENO" 5
+
+fi
+fi
+
+ # Restore CPPFLAGS/LDFLAGS/LIBS
+ CPPFLAGS=$rxd_orig_CPPFLAGS
+ unset rxd_orig_CPPFLAGS
+ LDFLAGS=$rxd_orig_LDFLAGS
+ unset rxd_orig_LDFLAGS
+ LIBS=$rxd_orig_LIBS
+ unset rxd_orig_LIBS
+
+
+
+
+
+
+
+ FI_PC_PRIVATE_LIBS="$FI_PC_PRIVATE_LIBS $LIBS"
+
+ seen=""
+ for arg in $FI_PC_PRIVATE_LIBS
+ do
+ found=no
+ for s in $seen
+ do
+ if test "$s" = "$arg"; then :
+ found=yes
+fi
+ done
+ if test "$found" = "no"; then :
+ seen="$seen $arg"
+fi
+ done
+ FI_PC_PRIVATE_LIBS=$seen
+
+ if test "x$enable_shared" != xyes; then :
+ FI_PC_PUBLIC_LIBS="$FI_PC_PUBLIC_LIBS $FI_PC_PRIVATE_LIBS"
+fi
+
+
+
+
+ ac_config_files="$ac_config_files libfabric.pc"
+
+
+
+# If the user requested to build in direct mode, but
+# we have more than one provider, error.
+PROVIDER_DIRECT=no
+FI_DIRECT_PROVIDER_API_10=/dev/null
+if test x"$enable_direct" != x"no"; then :
+ if test "$PROVIDERS_COUNT" -gt 1; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Only one provider can be chosen when using --enable-direct" >&5
+$as_echo "$as_me: WARNING: Only one provider can be chosen when using --enable-direct" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+fi
+
+ PROVIDER_DIRECT=$enable_direct
+
+cat >>confdefs.h <<_ACEOF
+#define FABRIC_DIRECT_ENABLED 1
+_ACEOF
+
+ FI_DIRECT_PROVIDER_API_10="$srcdir/prov/$enable_direct/provider_FABRIC_1.0.map"
+ if test ! -r "$FI_DIRECT_PROVIDER_API_10"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --enable-direct=$enable_direct specified, but $FI_DIRECT_PROVIDER_API_10 does not exist" >&5
+$as_echo "$as_me: WARNING: --enable-direct=$enable_direct specified, but $FI_DIRECT_PROVIDER_API_10 does not exist" >&2;}
+ as_fn_error $? "Cannot continue" "$LINENO" 5
+fi
+fi
+
+
+
+ if test x"$enable_direct" != x"no"; then
+ HAVE_DIRECT_TRUE=
+ HAVE_DIRECT_FALSE='#'
+else
+ HAVE_DIRECT_TRUE='#'
+ HAVE_DIRECT_FALSE=
+fi
+
+
+ac_config_files="$ac_config_files Makefile libfabric.spec libfabric.map"
+
+cat >confcache <<\_ACEOF
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs, see configure's option --config-cache.
+# It is not useful on other systems. If it contains results you don't
+# want to keep, you may remove or edit it.
+#
+# config.status only pays attention to the cache file if you give it
+# the --recheck option to rerun configure.
+#
+# `ac_cv_env_foo' variables (set or unset) will be overridden when
+# loading this file, other *unset* `ac_cv_foo' will be assigned the
+# following values.
+
+_ACEOF
+
+# The following way of writing the cache mishandles newlines in values,
+# but we know of no workaround that is simple, portable, and efficient.
+# So, we kill variables containing newlines.
+# Ultrix sh set writes to stderr and can't be redirected directly,
+# and sets the high bit in the cache file unless we assign to the vars.
+(
+ for ac_var in `(set) 2>&1 | sed -n 's/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'`; do
+ eval ac_val=\$$ac_var
+ case $ac_val in #(
+ *${as_nl}*)
+ case $ac_var in #(
+ *_cv_*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5
+$as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;;
+ esac
+ case $ac_var in #(
+ _ | IFS | as_nl) ;; #(
+ BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #(
+ *) { eval $ac_var=; unset $ac_var;} ;;
+ esac ;;
+ esac
+ done
+
+ (set) 2>&1 |
+ case $as_nl`(ac_space=' '; set) 2>&1` in #(
+ *${as_nl}ac_space=\ *)
+ # `set' does not quote correctly, so add quotes: double-quote
+ # substitution turns \\\\ into \\, and sed turns \\ into \.
+ sed -n \
+ "s/'/'\\\\''/g;
+ s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p"
+ ;; #(
+ *)
+ # `set' quotes correctly as required by POSIX, so do not add quotes.
+ sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p"
+ ;;
+ esac |
+ sort
+) |
+ sed '
+ /^ac_cv_env_/b end
+ t clear
+ :clear
+ s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/
+ t end
+ s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/
+ :end' >>confcache
+if diff "$cache_file" confcache >/dev/null 2>&1; then :; else
+ if test -w "$cache_file"; then
+ test "x$cache_file" != "x/dev/null" &&
+ { $as_echo "$as_me:${as_lineno-$LINENO}: updating cache $cache_file" >&5
+$as_echo "$as_me: updating cache $cache_file" >&6;}
+ cat confcache >$cache_file
+ else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: not updating unwritable cache $cache_file" >&5
+$as_echo "$as_me: not updating unwritable cache $cache_file" >&6;}
+ fi
+fi
+rm -f confcache
+
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+# Let make expand exec_prefix.
+test "x$exec_prefix" = xNONE && exec_prefix='${prefix}'
+
+DEFS=-DHAVE_CONFIG_H
+
+ac_libobjs=
+ac_ltlibobjs=
+U=
+for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue
+ # 1. Remove the extension, and $U if already installed.
+ ac_script='s/\$U\././;s/\.o$//;s/\.obj$//'
+ ac_i=`$as_echo "$ac_i" | sed "$ac_script"`
+ # 2. Prepend LIBOBJDIR. When used with automake>=1.10 LIBOBJDIR
+ # will be set to the directory where LIBOBJS objects are built.
+ as_fn_append ac_libobjs " \${LIBOBJDIR}$ac_i\$U.$ac_objext"
+ as_fn_append ac_ltlibobjs " \${LIBOBJDIR}$ac_i"'$U.lo'
+done
+LIBOBJS=$ac_libobjs
+
+LTLIBOBJS=$ac_ltlibobjs
+
+
+ if test -n "$EXEEXT"; then
+ am__EXEEXT_TRUE=
+ am__EXEEXT_FALSE='#'
+else
+ am__EXEEXT_TRUE='#'
+ am__EXEEXT_FALSE=
+fi
+
+if test -z "${MACOS_TRUE}" && test -z "${MACOS_FALSE}"; then
+ as_fn_error $? "conditional \"MACOS\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${LINUX_TRUE}" && test -z "${LINUX_FALSE}"; then
+ as_fn_error $? "conditional \"LINUX\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${FREEBSD_TRUE}" && test -z "${FREEBSD_FALSE}"; then
+ as_fn_error $? "conditional \"FREEBSD\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${AMDEP_TRUE}" && test -z "${AMDEP_FALSE}"; then
+ as_fn_error $? "conditional \"AMDEP\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${am__fastdepCC_TRUE}" && test -z "${am__fastdepCC_FALSE}"; then
+ as_fn_error $? "conditional \"am__fastdepCC\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${EMBEDDED_TRUE}" && test -z "${EMBEDDED_FALSE}"; then
+ as_fn_error $? "conditional \"EMBEDDED\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_LD_VERSION_SCRIPT_TRUE}" && test -z "${HAVE_LD_VERSION_SCRIPT_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_LD_VERSION_SCRIPT\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_PSM_TRUE}" && test -z "${HAVE_PSM_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_PSM\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_PSM_DL_TRUE}" && test -z "${HAVE_PSM_DL_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_PSM_DL\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_PSM2_TRUE}" && test -z "${HAVE_PSM2_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_PSM2\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_PSM2_DL_TRUE}" && test -z "${HAVE_PSM2_DL_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_PSM2_DL\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_SOCKETS_TRUE}" && test -z "${HAVE_SOCKETS_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_SOCKETS\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_SOCKETS_DL_TRUE}" && test -z "${HAVE_SOCKETS_DL_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_SOCKETS_DL\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_VERBS_TRUE}" && test -z "${HAVE_VERBS_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_VERBS\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_VERBS_DL_TRUE}" && test -z "${HAVE_VERBS_DL_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_VERBS_DL\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_USNIC_TRUE}" && test -z "${HAVE_USNIC_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_USNIC\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_USNIC_DL_TRUE}" && test -z "${HAVE_USNIC_DL_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_USNIC_DL\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_MXM_TRUE}" && test -z "${HAVE_MXM_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_MXM\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_MXM_DL_TRUE}" && test -z "${HAVE_MXM_DL_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_MXM_DL\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_CRITERION_TRUE}" && test -z "${HAVE_CRITERION_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_CRITERION\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_GNI_TRUE}" && test -z "${HAVE_GNI_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_GNI\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_GNI_DL_TRUE}" && test -z "${HAVE_GNI_DL_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_GNI_DL\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_UDP_TRUE}" && test -z "${HAVE_UDP_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_UDP\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_UDP_DL_TRUE}" && test -z "${HAVE_UDP_DL_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_UDP_DL\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_RXM_TRUE}" && test -z "${HAVE_RXM_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_RXM\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_RXM_DL_TRUE}" && test -z "${HAVE_RXM_DL_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_RXM_DL\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_RXD_TRUE}" && test -z "${HAVE_RXD_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_RXD\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_RXD_DL_TRUE}" && test -z "${HAVE_RXD_DL_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_RXD_DL\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${HAVE_DIRECT_TRUE}" && test -z "${HAVE_DIRECT_FALSE}"; then
+ as_fn_error $? "conditional \"HAVE_DIRECT\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+
+: ${CONFIG_STATUS=./config.status}
+ac_write_fail=0
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files $CONFIG_STATUS"
+{ $as_echo "$as_me:${as_lineno-$LINENO}: creating $CONFIG_STATUS" >&5
+$as_echo "$as_me: creating $CONFIG_STATUS" >&6;}
+as_write_fail=0
+cat >$CONFIG_STATUS <<_ASEOF || as_write_fail=1
+#! $SHELL
+# Generated by $as_me.
+# Run this file to recreate the current configuration.
+# Compiler output produced by configure, useful for debugging
+# configure, is in config.log if it exists.
+
+debug=false
+ac_cs_recheck=false
+ac_cs_silent=false
+
+SHELL=\${CONFIG_SHELL-$SHELL}
+export SHELL
+_ASEOF
+cat >>$CONFIG_STATUS <<\_ASEOF || as_write_fail=1
+## -------------------- ##
+## M4sh Initialization. ##
+## -------------------- ##
+
+# Be more Bourne compatible
+DUALCASE=1; export DUALCASE # for MKS sh
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then :
+ emulate sh
+ NULLCMD=:
+ # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which
+ # is contrary to our usage. Disable this feature.
+ alias -g '${1+"$@"}'='"$@"'
+ setopt NO_GLOB_SUBST
+else
+ case `(set -o) 2>/dev/null` in #(
+ *posix*) :
+ set -o posix ;; #(
+ *) :
+ ;;
+esac
+fi
+
+
+as_nl='
+'
+export as_nl
+# Printing a long string crashes Solaris 7 /usr/bin/printf.
+as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'
+as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo
+as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo
+# Prefer a ksh shell builtin over an external printf program on Solaris,
+# but without wasting forks for bash or zsh.
+if test -z "$BASH_VERSION$ZSH_VERSION" \
+ && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then
+ as_echo='print -r --'
+ as_echo_n='print -rn --'
+elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then
+ as_echo='printf %s\n'
+ as_echo_n='printf %s'
+else
+ if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then
+ as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"'
+ as_echo_n='/usr/ucb/echo -n'
+ else
+ as_echo_body='eval expr "X$1" : "X\\(.*\\)"'
+ as_echo_n_body='eval
+ arg=$1;
+ case $arg in #(
+ *"$as_nl"*)
+ expr "X$arg" : "X\\(.*\\)$as_nl";
+ arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;;
+ esac;
+ expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl"
+ '
+ export as_echo_n_body
+ as_echo_n='sh -c $as_echo_n_body as_echo'
+ fi
+ export as_echo_body
+ as_echo='sh -c $as_echo_body as_echo'
+fi
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+ PATH_SEPARATOR=:
+ (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && {
+ (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 ||
+ PATH_SEPARATOR=';'
+ }
+fi
+
+
+# IFS
+# We need space, tab and new line, in precisely that order. Quoting is
+# there to prevent editors from complaining about space-tab.
+# (If _AS_PATH_WALK were called with IFS unset, it would disable word
+# splitting by setting IFS to empty value.)
+IFS=" "" $as_nl"
+
+# Find who we are. Look in the path if we contain no directory separator.
+case $0 in #((
+ *[\\/]* ) as_myself=$0 ;;
+ *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+ done
+IFS=$as_save_IFS
+
+ ;;
+esac
+# We did not find ourselves, most probably we were run as `sh COMMAND'
+# in which case we are not to be found in the path.
+if test "x$as_myself" = x; then
+ as_myself=$0
+fi
+if test ! -f "$as_myself"; then
+ $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2
+ exit 1
+fi
+
+# Unset variables that we do not need and which cause bugs (e.g. in
+# pre-3.0 UWIN ksh). But do not cause bugs in bash 2.01; the "|| exit 1"
+# suppresses any "Segmentation fault" message there. '((' could
+# trigger a bug in pdksh 5.2.14.
+for as_var in BASH_ENV ENV MAIL MAILPATH
+do eval test x\${$as_var+set} = xset \
+ && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || :
+done
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+LC_ALL=C
+export LC_ALL
+LANGUAGE=C
+export LANGUAGE
+
+# CDPATH.
+(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
+
+
+# as_fn_error STATUS ERROR [LINENO LOG_FD]
+# ----------------------------------------
+# Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are
+# provided, also output the error to LOG_FD, referencing LINENO. Then exit the
+# script with STATUS, using 1 if that was 0.
+as_fn_error ()
+{
+ as_status=$1; test $as_status -eq 0 && as_status=1
+ if test "$4"; then
+ as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+ $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4
+ fi
+ $as_echo "$as_me: error: $2" >&2
+ as_fn_exit $as_status
+} # as_fn_error
+
+
+# as_fn_set_status STATUS
+# -----------------------
+# Set $? to STATUS, without forking.
+as_fn_set_status ()
+{
+ return $1
+} # as_fn_set_status
+
+# as_fn_exit STATUS
+# -----------------
+# Exit the shell with STATUS, even in a "trap 0" or "set -e" context.
+as_fn_exit ()
+{
+ set +e
+ as_fn_set_status $1
+ exit $1
+} # as_fn_exit
+
+# as_fn_unset VAR
+# ---------------
+# Portably unset VAR.
+as_fn_unset ()
+{
+ { eval $1=; unset $1;}
+}
+as_unset=as_fn_unset
+# as_fn_append VAR VALUE
+# ----------------------
+# Append the text in VALUE to the end of the definition contained in VAR. Take
+# advantage of any shell optimizations that allow amortized linear growth over
+# repeated appends, instead of the typical quadratic growth present in naive
+# implementations.
+if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null; then :
+ eval 'as_fn_append ()
+ {
+ eval $1+=\$2
+ }'
+else
+ as_fn_append ()
+ {
+ eval $1=\$$1\$2
+ }
+fi # as_fn_append
+
+# as_fn_arith ARG...
+# ------------------
+# Perform arithmetic evaluation on the ARGs, and store the result in the
+# global $as_val. Take advantage of shells that can avoid forks. The arguments
+# must be portable across $(()) and expr.
+if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null; then :
+ eval 'as_fn_arith ()
+ {
+ as_val=$(( $* ))
+ }'
+else
+ as_fn_arith ()
+ {
+ as_val=`expr "$@" || test $? -eq 1`
+ }
+fi # as_fn_arith
+
+
+if expr a : '\(a\)' >/dev/null 2>&1 &&
+ test "X`expr 00001 : '.*\(...\)'`" = X001; then
+ as_expr=expr
+else
+ as_expr=false
+fi
+
+if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then
+ as_basename=basename
+else
+ as_basename=false
+fi
+
+if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then
+ as_dirname=dirname
+else
+ as_dirname=false
+fi
+
+as_me=`$as_basename -- "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+ X"$0" : 'X\(//\)$' \| \
+ X"$0" : 'X\(/\)' \| . 2>/dev/null ||
+$as_echo X/"$0" |
+ sed '/^.*\/\([^/][^/]*\)\/*$/{
+ s//\1/
+ q
+ }
+ /^X\/\(\/\/\)$/{
+ s//\1/
+ q
+ }
+ /^X\/\(\/\).*/{
+ s//\1/
+ q
+ }
+ s/.*/./; q'`
+
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+ECHO_C= ECHO_N= ECHO_T=
+case `echo -n x` in #(((((
+-n*)
+ case `echo 'xy\c'` in
+ *c*) ECHO_T=' ';; # ECHO_T is single tab character.
+ xy) ECHO_C='\c';;
+ *) echo `echo ksh88 bug on AIX 6.1` > /dev/null
+ ECHO_T=' ';;
+ esac;;
+*)
+ ECHO_N='-n';;
+esac
+
+rm -f conf$$ conf$$.exe conf$$.file
+if test -d conf$$.dir; then
+ rm -f conf$$.dir/conf$$.file
+else
+ rm -f conf$$.dir
+ mkdir conf$$.dir 2>/dev/null
+fi
+if (echo >conf$$.file) 2>/dev/null; then
+ if ln -s conf$$.file conf$$ 2>/dev/null; then
+ as_ln_s='ln -s'
+ # ... but there are two gotchas:
+ # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail.
+ # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable.
+ # In both cases, we have to default to `cp -p'.
+ ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe ||
+ as_ln_s='cp -p'
+ elif ln conf$$.file conf$$ 2>/dev/null; then
+ as_ln_s=ln
+ else
+ as_ln_s='cp -p'
+ fi
+else
+ as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file
+rmdir conf$$.dir 2>/dev/null
+
+
+# as_fn_mkdir_p
+# -------------
+# Create "$as_dir" as a directory, including parents if necessary.
+as_fn_mkdir_p ()
+{
+
+ case $as_dir in #(
+ -*) as_dir=./$as_dir;;
+ esac
+ test -d "$as_dir" || eval $as_mkdir_p || {
+ as_dirs=
+ while :; do
+ case $as_dir in #(
+ *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'(
+ *) as_qdir=$as_dir;;
+ esac
+ as_dirs="'$as_qdir' $as_dirs"
+ as_dir=`$as_dirname -- "$as_dir" ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+ X"$as_dir" : 'X\(//\)[^/]' \| \
+ X"$as_dir" : 'X\(//\)$' \| \
+ X"$as_dir" : 'X\(/\)' \| . 2>/dev/null ||
+$as_echo X"$as_dir" |
+ sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)[^/].*/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\).*/{
+ s//\1/
+ q
+ }
+ s/.*/./; q'`
+ test -d "$as_dir" && break
+ done
+ test -z "$as_dirs" || eval "mkdir $as_dirs"
+ } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir"
+
+
+} # as_fn_mkdir_p
+if mkdir -p . 2>/dev/null; then
+ as_mkdir_p='mkdir -p "$as_dir"'
+else
+ test -d ./-p && rmdir ./-p
+ as_mkdir_p=false
+fi
+
+if test -x / >/dev/null 2>&1; then
+ as_test_x='test -x'
+else
+ if ls -dL / >/dev/null 2>&1; then
+ as_ls_L_option=L
+ else
+ as_ls_L_option=
+ fi
+ as_test_x='
+ eval sh -c '\''
+ if test -d "$1"; then
+ test -d "$1/.";
+ else
+ case $1 in #(
+ -*)set "./$1";;
+ esac;
+ case `ls -ld'$as_ls_L_option' "$1" 2>/dev/null` in #((
+ ???[sx]*):;;*)false;;esac;fi
+ '\'' sh
+ '
+fi
+as_executable_p=$as_test_x
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+exec 6>&1
+## ----------------------------------- ##
+## Main body of $CONFIG_STATUS script. ##
+## ----------------------------------- ##
+_ASEOF
+test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
+
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+# Save the log message, to keep $0 and so on meaningful, and to
+# report actual input values of CONFIG_FILES etc. instead of their
+# values after options handling.
+ac_log="
+This file was extended by libfabric $as_me 1.4.0rc2, which was
+generated by GNU Autoconf 2.67. Invocation command line was
+
+ CONFIG_FILES = $CONFIG_FILES
+ CONFIG_HEADERS = $CONFIG_HEADERS
+ CONFIG_LINKS = $CONFIG_LINKS
+ CONFIG_COMMANDS = $CONFIG_COMMANDS
+ $ $0 $@
+
+on `(hostname || uname -n) 2>/dev/null | sed 1q`
+"
+
+_ACEOF
+
+case $ac_config_files in *"
+"*) set x $ac_config_files; shift; ac_config_files=$*;;
+esac
+
+case $ac_config_headers in *"
+"*) set x $ac_config_headers; shift; ac_config_headers=$*;;
+esac
+
+
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+# Files that config.status was made for.
+config_files="$ac_config_files"
+config_headers="$ac_config_headers"
+config_commands="$ac_config_commands"
+
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+ac_cs_usage="\
+\`$as_me' instantiates files and other configuration actions
+from templates according to the current configuration. Unless the files
+and actions are specified as TAGs, all are instantiated by default.
+
+Usage: $0 [OPTION]... [TAG]...
+
+ -h, --help print this help, then exit
+ -V, --version print version number and configuration settings, then exit
+ --config print configuration, then exit
+ -q, --quiet, --silent
+ do not print progress messages
+ -d, --debug don't remove temporary files
+ --recheck update $as_me by reconfiguring in the same conditions
+ --file=FILE[:TEMPLATE]
+ instantiate the configuration file FILE
+ --header=FILE[:TEMPLATE]
+ instantiate the configuration header FILE
+
+Configuration files:
+$config_files
+
+Configuration headers:
+$config_headers
+
+Configuration commands:
+$config_commands
+
+Report bugs to <ofiwg at lists.openfabrics.org>."
+
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
+ac_cs_version="\\
+libfabric config.status 1.4.0rc2
+configured by $0, generated by GNU Autoconf 2.67,
+ with options \\"\$ac_cs_config\\"
+
+Copyright (C) 2010 Free Software Foundation, Inc.
+This config.status script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it."
+
+ac_pwd='$ac_pwd'
+srcdir='$srcdir'
+INSTALL='$INSTALL'
+MKDIR_P='$MKDIR_P'
+AWK='$AWK'
+test -n "\$AWK" || AWK=awk
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+# The default lists apply if the user does not specify any file.
+ac_need_defaults=:
+while test $# != 0
+do
+ case $1 in
+ --*=?*)
+ ac_option=`expr "X$1" : 'X\([^=]*\)='`
+ ac_optarg=`expr "X$1" : 'X[^=]*=\(.*\)'`
+ ac_shift=:
+ ;;
+ --*=)
+ ac_option=`expr "X$1" : 'X\([^=]*\)='`
+ ac_optarg=
+ ac_shift=:
+ ;;
+ *)
+ ac_option=$1
+ ac_optarg=$2
+ ac_shift=shift
+ ;;
+ esac
+
+ case $ac_option in
+ # Handling of the options.
+ -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r)
+ ac_cs_recheck=: ;;
+ --version | --versio | --versi | --vers | --ver | --ve | --v | -V )
+ $as_echo "$ac_cs_version"; exit ;;
+ --config | --confi | --conf | --con | --co | --c )
+ $as_echo "$ac_cs_config"; exit ;;
+ --debug | --debu | --deb | --de | --d | -d )
+ debug=: ;;
+ --file | --fil | --fi | --f )
+ $ac_shift
+ case $ac_optarg in
+ *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;;
+ '') as_fn_error $? "missing file argument" ;;
+ esac
+ as_fn_append CONFIG_FILES " '$ac_optarg'"
+ ac_need_defaults=false;;
+ --header | --heade | --head | --hea )
+ $ac_shift
+ case $ac_optarg in
+ *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;;
+ esac
+ as_fn_append CONFIG_HEADERS " '$ac_optarg'"
+ ac_need_defaults=false;;
+ --he | --h)
+ # Conflict between --help and --header
+ as_fn_error $? "ambiguous option: \`$1'
+Try \`$0 --help' for more information.";;
+ --help | --hel | -h )
+ $as_echo "$ac_cs_usage"; exit ;;
+ -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+ | -silent | --silent | --silen | --sile | --sil | --si | --s)
+ ac_cs_silent=: ;;
+
+ # This is an error.
+ -*) as_fn_error $? "unrecognized option: \`$1'
+Try \`$0 --help' for more information." ;;
+
+ *) as_fn_append ac_config_targets " $1"
+ ac_need_defaults=false ;;
+
+ esac
+ shift
+done
+
+ac_configure_extra_args=
+
+if $ac_cs_silent; then
+ exec 6>/dev/null
+ ac_configure_extra_args="$ac_configure_extra_args --silent"
+fi
+
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+if \$ac_cs_recheck; then
+ set X '$SHELL' '$0' $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion
+ shift
+ \$as_echo "running CONFIG_SHELL=$SHELL \$*" >&6
+ CONFIG_SHELL='$SHELL'
+ export CONFIG_SHELL
+ exec "\$@"
+fi
+
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+exec 5>>config.log
+{
+ echo
+ sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX
+## Running $as_me. ##
+_ASBOX
+ $as_echo "$ac_log"
+} >&5
+
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+#
+# INIT-COMMANDS
+#
+AMDEP_TRUE="$AMDEP_TRUE" ac_aux_dir="$ac_aux_dir"
+
+
+# The HP-UX ksh and POSIX shell print the target directory to stdout
+# if CDPATH is set.
+(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
+
+sed_quote_subst='$sed_quote_subst'
+double_quote_subst='$double_quote_subst'
+delay_variable_subst='$delay_variable_subst'
+macro_version='`$ECHO "X$macro_version" | $Xsed -e "$delay_single_quote_subst"`'
+macro_revision='`$ECHO "X$macro_revision" | $Xsed -e "$delay_single_quote_subst"`'
+enable_shared='`$ECHO "X$enable_shared" | $Xsed -e "$delay_single_quote_subst"`'
+enable_static='`$ECHO "X$enable_static" | $Xsed -e "$delay_single_quote_subst"`'
+pic_mode='`$ECHO "X$pic_mode" | $Xsed -e "$delay_single_quote_subst"`'
+enable_fast_install='`$ECHO "X$enable_fast_install" | $Xsed -e "$delay_single_quote_subst"`'
+host_alias='`$ECHO "X$host_alias" | $Xsed -e "$delay_single_quote_subst"`'
+host='`$ECHO "X$host" | $Xsed -e "$delay_single_quote_subst"`'
+host_os='`$ECHO "X$host_os" | $Xsed -e "$delay_single_quote_subst"`'
+build_alias='`$ECHO "X$build_alias" | $Xsed -e "$delay_single_quote_subst"`'
+build='`$ECHO "X$build" | $Xsed -e "$delay_single_quote_subst"`'
+build_os='`$ECHO "X$build_os" | $Xsed -e "$delay_single_quote_subst"`'
+SED='`$ECHO "X$SED" | $Xsed -e "$delay_single_quote_subst"`'
+Xsed='`$ECHO "X$Xsed" | $Xsed -e "$delay_single_quote_subst"`'
+GREP='`$ECHO "X$GREP" | $Xsed -e "$delay_single_quote_subst"`'
+EGREP='`$ECHO "X$EGREP" | $Xsed -e "$delay_single_quote_subst"`'
+FGREP='`$ECHO "X$FGREP" | $Xsed -e "$delay_single_quote_subst"`'
+LD='`$ECHO "X$LD" | $Xsed -e "$delay_single_quote_subst"`'
+NM='`$ECHO "X$NM" | $Xsed -e "$delay_single_quote_subst"`'
+LN_S='`$ECHO "X$LN_S" | $Xsed -e "$delay_single_quote_subst"`'
+max_cmd_len='`$ECHO "X$max_cmd_len" | $Xsed -e "$delay_single_quote_subst"`'
+ac_objext='`$ECHO "X$ac_objext" | $Xsed -e "$delay_single_quote_subst"`'
+exeext='`$ECHO "X$exeext" | $Xsed -e "$delay_single_quote_subst"`'
+lt_unset='`$ECHO "X$lt_unset" | $Xsed -e "$delay_single_quote_subst"`'
+lt_SP2NL='`$ECHO "X$lt_SP2NL" | $Xsed -e "$delay_single_quote_subst"`'
+lt_NL2SP='`$ECHO "X$lt_NL2SP" | $Xsed -e "$delay_single_quote_subst"`'
+reload_flag='`$ECHO "X$reload_flag" | $Xsed -e "$delay_single_quote_subst"`'
+reload_cmds='`$ECHO "X$reload_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+OBJDUMP='`$ECHO "X$OBJDUMP" | $Xsed -e "$delay_single_quote_subst"`'
+deplibs_check_method='`$ECHO "X$deplibs_check_method" | $Xsed -e "$delay_single_quote_subst"`'
+file_magic_cmd='`$ECHO "X$file_magic_cmd" | $Xsed -e "$delay_single_quote_subst"`'
+AR='`$ECHO "X$AR" | $Xsed -e "$delay_single_quote_subst"`'
+AR_FLAGS='`$ECHO "X$AR_FLAGS" | $Xsed -e "$delay_single_quote_subst"`'
+STRIP='`$ECHO "X$STRIP" | $Xsed -e "$delay_single_quote_subst"`'
+RANLIB='`$ECHO "X$RANLIB" | $Xsed -e "$delay_single_quote_subst"`'
+old_postinstall_cmds='`$ECHO "X$old_postinstall_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+old_postuninstall_cmds='`$ECHO "X$old_postuninstall_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+old_archive_cmds='`$ECHO "X$old_archive_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+CC='`$ECHO "X$CC" | $Xsed -e "$delay_single_quote_subst"`'
+CFLAGS='`$ECHO "X$CFLAGS" | $Xsed -e "$delay_single_quote_subst"`'
+compiler='`$ECHO "X$compiler" | $Xsed -e "$delay_single_quote_subst"`'
+GCC='`$ECHO "X$GCC" | $Xsed -e "$delay_single_quote_subst"`'
+lt_cv_sys_global_symbol_pipe='`$ECHO "X$lt_cv_sys_global_symbol_pipe" | $Xsed -e "$delay_single_quote_subst"`'
+lt_cv_sys_global_symbol_to_cdecl='`$ECHO "X$lt_cv_sys_global_symbol_to_cdecl" | $Xsed -e "$delay_single_quote_subst"`'
+lt_cv_sys_global_symbol_to_c_name_address='`$ECHO "X$lt_cv_sys_global_symbol_to_c_name_address" | $Xsed -e "$delay_single_quote_subst"`'
+lt_cv_sys_global_symbol_to_c_name_address_lib_prefix='`$ECHO "X$lt_cv_sys_global_symbol_to_c_name_address_lib_prefix" | $Xsed -e "$delay_single_quote_subst"`'
+objdir='`$ECHO "X$objdir" | $Xsed -e "$delay_single_quote_subst"`'
+SHELL='`$ECHO "X$SHELL" | $Xsed -e "$delay_single_quote_subst"`'
+ECHO='`$ECHO "X$ECHO" | $Xsed -e "$delay_single_quote_subst"`'
+MAGIC_CMD='`$ECHO "X$MAGIC_CMD" | $Xsed -e "$delay_single_quote_subst"`'
+lt_prog_compiler_no_builtin_flag='`$ECHO "X$lt_prog_compiler_no_builtin_flag" | $Xsed -e "$delay_single_quote_subst"`'
+lt_prog_compiler_wl='`$ECHO "X$lt_prog_compiler_wl" | $Xsed -e "$delay_single_quote_subst"`'
+lt_prog_compiler_pic='`$ECHO "X$lt_prog_compiler_pic" | $Xsed -e "$delay_single_quote_subst"`'
+lt_prog_compiler_static='`$ECHO "X$lt_prog_compiler_static" | $Xsed -e "$delay_single_quote_subst"`'
+lt_cv_prog_compiler_c_o='`$ECHO "X$lt_cv_prog_compiler_c_o" | $Xsed -e "$delay_single_quote_subst"`'
+need_locks='`$ECHO "X$need_locks" | $Xsed -e "$delay_single_quote_subst"`'
+DSYMUTIL='`$ECHO "X$DSYMUTIL" | $Xsed -e "$delay_single_quote_subst"`'
+NMEDIT='`$ECHO "X$NMEDIT" | $Xsed -e "$delay_single_quote_subst"`'
+LIPO='`$ECHO "X$LIPO" | $Xsed -e "$delay_single_quote_subst"`'
+OTOOL='`$ECHO "X$OTOOL" | $Xsed -e "$delay_single_quote_subst"`'
+OTOOL64='`$ECHO "X$OTOOL64" | $Xsed -e "$delay_single_quote_subst"`'
+libext='`$ECHO "X$libext" | $Xsed -e "$delay_single_quote_subst"`'
+shrext_cmds='`$ECHO "X$shrext_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+extract_expsyms_cmds='`$ECHO "X$extract_expsyms_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+archive_cmds_need_lc='`$ECHO "X$archive_cmds_need_lc" | $Xsed -e "$delay_single_quote_subst"`'
+enable_shared_with_static_runtimes='`$ECHO "X$enable_shared_with_static_runtimes" | $Xsed -e "$delay_single_quote_subst"`'
+export_dynamic_flag_spec='`$ECHO "X$export_dynamic_flag_spec" | $Xsed -e "$delay_single_quote_subst"`'
+whole_archive_flag_spec='`$ECHO "X$whole_archive_flag_spec" | $Xsed -e "$delay_single_quote_subst"`'
+compiler_needs_object='`$ECHO "X$compiler_needs_object" | $Xsed -e "$delay_single_quote_subst"`'
+old_archive_from_new_cmds='`$ECHO "X$old_archive_from_new_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+old_archive_from_expsyms_cmds='`$ECHO "X$old_archive_from_expsyms_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+archive_cmds='`$ECHO "X$archive_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+archive_expsym_cmds='`$ECHO "X$archive_expsym_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+module_cmds='`$ECHO "X$module_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+module_expsym_cmds='`$ECHO "X$module_expsym_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+with_gnu_ld='`$ECHO "X$with_gnu_ld" | $Xsed -e "$delay_single_quote_subst"`'
+allow_undefined_flag='`$ECHO "X$allow_undefined_flag" | $Xsed -e "$delay_single_quote_subst"`'
+no_undefined_flag='`$ECHO "X$no_undefined_flag" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_libdir_flag_spec='`$ECHO "X$hardcode_libdir_flag_spec" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_libdir_flag_spec_ld='`$ECHO "X$hardcode_libdir_flag_spec_ld" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_libdir_separator='`$ECHO "X$hardcode_libdir_separator" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_direct='`$ECHO "X$hardcode_direct" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_direct_absolute='`$ECHO "X$hardcode_direct_absolute" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_minus_L='`$ECHO "X$hardcode_minus_L" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_shlibpath_var='`$ECHO "X$hardcode_shlibpath_var" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_automatic='`$ECHO "X$hardcode_automatic" | $Xsed -e "$delay_single_quote_subst"`'
+inherit_rpath='`$ECHO "X$inherit_rpath" | $Xsed -e "$delay_single_quote_subst"`'
+link_all_deplibs='`$ECHO "X$link_all_deplibs" | $Xsed -e "$delay_single_quote_subst"`'
+fix_srcfile_path='`$ECHO "X$fix_srcfile_path" | $Xsed -e "$delay_single_quote_subst"`'
+always_export_symbols='`$ECHO "X$always_export_symbols" | $Xsed -e "$delay_single_quote_subst"`'
+export_symbols_cmds='`$ECHO "X$export_symbols_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+exclude_expsyms='`$ECHO "X$exclude_expsyms" | $Xsed -e "$delay_single_quote_subst"`'
+include_expsyms='`$ECHO "X$include_expsyms" | $Xsed -e "$delay_single_quote_subst"`'
+prelink_cmds='`$ECHO "X$prelink_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+file_list_spec='`$ECHO "X$file_list_spec" | $Xsed -e "$delay_single_quote_subst"`'
+variables_saved_for_relink='`$ECHO "X$variables_saved_for_relink" | $Xsed -e "$delay_single_quote_subst"`'
+need_lib_prefix='`$ECHO "X$need_lib_prefix" | $Xsed -e "$delay_single_quote_subst"`'
+need_version='`$ECHO "X$need_version" | $Xsed -e "$delay_single_quote_subst"`'
+version_type='`$ECHO "X$version_type" | $Xsed -e "$delay_single_quote_subst"`'
+runpath_var='`$ECHO "X$runpath_var" | $Xsed -e "$delay_single_quote_subst"`'
+shlibpath_var='`$ECHO "X$shlibpath_var" | $Xsed -e "$delay_single_quote_subst"`'
+shlibpath_overrides_runpath='`$ECHO "X$shlibpath_overrides_runpath" | $Xsed -e "$delay_single_quote_subst"`'
+libname_spec='`$ECHO "X$libname_spec" | $Xsed -e "$delay_single_quote_subst"`'
+library_names_spec='`$ECHO "X$library_names_spec" | $Xsed -e "$delay_single_quote_subst"`'
+soname_spec='`$ECHO "X$soname_spec" | $Xsed -e "$delay_single_quote_subst"`'
+postinstall_cmds='`$ECHO "X$postinstall_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+postuninstall_cmds='`$ECHO "X$postuninstall_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+finish_cmds='`$ECHO "X$finish_cmds" | $Xsed -e "$delay_single_quote_subst"`'
+finish_eval='`$ECHO "X$finish_eval" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_into_libs='`$ECHO "X$hardcode_into_libs" | $Xsed -e "$delay_single_quote_subst"`'
+sys_lib_search_path_spec='`$ECHO "X$sys_lib_search_path_spec" | $Xsed -e "$delay_single_quote_subst"`'
+sys_lib_dlsearch_path_spec='`$ECHO "X$sys_lib_dlsearch_path_spec" | $Xsed -e "$delay_single_quote_subst"`'
+hardcode_action='`$ECHO "X$hardcode_action" | $Xsed -e "$delay_single_quote_subst"`'
+enable_dlopen='`$ECHO "X$enable_dlopen" | $Xsed -e "$delay_single_quote_subst"`'
+enable_dlopen_self='`$ECHO "X$enable_dlopen_self" | $Xsed -e "$delay_single_quote_subst"`'
+enable_dlopen_self_static='`$ECHO "X$enable_dlopen_self_static" | $Xsed -e "$delay_single_quote_subst"`'
+old_striplib='`$ECHO "X$old_striplib" | $Xsed -e "$delay_single_quote_subst"`'
+striplib='`$ECHO "X$striplib" | $Xsed -e "$delay_single_quote_subst"`'
+
+LTCC='$LTCC'
+LTCFLAGS='$LTCFLAGS'
+compiler='$compiler_DEFAULT'
+
+# Quote evaled strings.
+for var in SED \
+GREP \
+EGREP \
+FGREP \
+LD \
+NM \
+LN_S \
+lt_SP2NL \
+lt_NL2SP \
+reload_flag \
+OBJDUMP \
+deplibs_check_method \
+file_magic_cmd \
+AR \
+AR_FLAGS \
+STRIP \
+RANLIB \
+CC \
+CFLAGS \
+compiler \
+lt_cv_sys_global_symbol_pipe \
+lt_cv_sys_global_symbol_to_cdecl \
+lt_cv_sys_global_symbol_to_c_name_address \
+lt_cv_sys_global_symbol_to_c_name_address_lib_prefix \
+SHELL \
+ECHO \
+lt_prog_compiler_no_builtin_flag \
+lt_prog_compiler_wl \
+lt_prog_compiler_pic \
+lt_prog_compiler_static \
+lt_cv_prog_compiler_c_o \
+need_locks \
+DSYMUTIL \
+NMEDIT \
+LIPO \
+OTOOL \
+OTOOL64 \
+shrext_cmds \
+export_dynamic_flag_spec \
+whole_archive_flag_spec \
+compiler_needs_object \
+with_gnu_ld \
+allow_undefined_flag \
+no_undefined_flag \
+hardcode_libdir_flag_spec \
+hardcode_libdir_flag_spec_ld \
+hardcode_libdir_separator \
+fix_srcfile_path \
+exclude_expsyms \
+include_expsyms \
+file_list_spec \
+variables_saved_for_relink \
+libname_spec \
+library_names_spec \
+soname_spec \
+finish_eval \
+old_striplib \
+striplib; do
+ case \`eval \\\\\$ECHO "X\\\\\$\$var"\` in
+ *[\\\\\\\`\\"\\\$]*)
+ eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"X\\\$\$var\\" | \\\$Xsed -e \\"\\\$sed_quote_subst\\"\\\`\\\\\\""
+ ;;
+ *)
+ eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\""
+ ;;
+ esac
+done
+
+# Double-quote double-evaled strings.
+for var in reload_cmds \
+old_postinstall_cmds \
+old_postuninstall_cmds \
+old_archive_cmds \
+extract_expsyms_cmds \
+old_archive_from_new_cmds \
+old_archive_from_expsyms_cmds \
+archive_cmds \
+archive_expsym_cmds \
+module_cmds \
+module_expsym_cmds \
+export_symbols_cmds \
+prelink_cmds \
+postinstall_cmds \
+postuninstall_cmds \
+finish_cmds \
+sys_lib_search_path_spec \
+sys_lib_dlsearch_path_spec; do
+ case \`eval \\\\\$ECHO "X\\\\\$\$var"\` in
+ *[\\\\\\\`\\"\\\$]*)
+ eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"X\\\$\$var\\" | \\\$Xsed -e \\"\\\$double_quote_subst\\" -e \\"\\\$sed_quote_subst\\" -e \\"\\\$delay_variable_subst\\"\\\`\\\\\\""
+ ;;
+ *)
+ eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\""
+ ;;
+ esac
+done
+
+# Fix-up fallback echo if it was mangled by the above quoting rules.
+case \$lt_ECHO in
+*'\\\$0 --fallback-echo"') lt_ECHO=\`\$ECHO "X\$lt_ECHO" | \$Xsed -e 's/\\\\\\\\\\\\\\\$0 --fallback-echo"\$/\$0 --fallback-echo"/'\`
+ ;;
+esac
+
+ac_aux_dir='$ac_aux_dir'
+xsi_shell='$xsi_shell'
+lt_shell_append='$lt_shell_append'
+
+# See if we are running on zsh, and set the options which allow our
+# commands through without removal of \ escapes INIT.
+if test -n "\${ZSH_VERSION+set}" ; then
+ setopt NO_GLOB_SUBST
+fi
+
+
+ PACKAGE='$PACKAGE'
+ VERSION='$VERSION'
+ TIMESTAMP='$TIMESTAMP'
+ RM='$RM'
+ ofile='$ofile'
+
+ac_aux_dir='$ac_aux_dir'
+
+
+
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+
+# Handling of arguments.
+for ac_config_target in $ac_config_targets
+do
+ case $ac_config_target in
+ "config.h") CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;;
+ "depfiles") CONFIG_COMMANDS="$CONFIG_COMMANDS depfiles" ;;
+ "libtool") CONFIG_COMMANDS="$CONFIG_COMMANDS libtool" ;;
+ "libfabric.pc") CONFIG_FILES="$CONFIG_FILES libfabric.pc" ;;
+ "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;;
+ "libfabric.spec") CONFIG_FILES="$CONFIG_FILES libfabric.spec" ;;
+ "libfabric.map") CONFIG_FILES="$CONFIG_FILES libfabric.map" ;;
+
+ *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;;
+ esac
+done
+
+
+# If the user did not use the arguments to specify the items to instantiate,
+# then the envvar interface is used. Set only those that are not.
+# We use the long form for the default assignment because of an extremely
+# bizarre bug on SunOS 4.1.3.
+if $ac_need_defaults; then
+ test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files
+ test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers
+ test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands
+fi
+
+# Have a temporary directory for convenience. Make it in the build tree
+# simply because there is no reason against having it here, and in addition,
+# creating and moving files from /tmp can sometimes cause problems.
+# Hook for its removal unless debugging.
+# Note that there is a small window in which the directory will not be cleaned:
+# after its creation but before its name has been assigned to `$tmp'.
+$debug ||
+{
+ tmp=
+ trap 'exit_status=$?
+ { test -z "$tmp" || test ! -d "$tmp" || rm -fr "$tmp"; } && exit $exit_status
+' 0
+ trap 'as_fn_exit 1' 1 2 13 15
+}
+# Create a (secure) tmp directory for tmp files.
+
+{
+ tmp=`(umask 077 && mktemp -d "./confXXXXXX") 2>/dev/null` &&
+ test -n "$tmp" && test -d "$tmp"
+} ||
+{
+ tmp=./conf$$-$RANDOM
+ (umask 077 && mkdir "$tmp")
+} || as_fn_error $? "cannot create a temporary directory in ." "$LINENO" 5
+
+# Set up the scripts for CONFIG_FILES section.
+# No need to generate them if there are no CONFIG_FILES.
+# This happens for instance with `./config.status config.h'.
+if test -n "$CONFIG_FILES"; then
+
+if $AWK 'BEGIN { getline <"/dev/null" }' </dev/null 2>/dev/null; then
+ ac_cs_awk_getline=:
+ ac_cs_awk_pipe_init=
+ ac_cs_awk_read_file='
+ while ((getline aline < (F[key])) > 0)
+ print(aline)
+ close(F[key])'
+ ac_cs_awk_pipe_fini=
+else
+ ac_cs_awk_getline=false
+ ac_cs_awk_pipe_init="print \"cat <<'|#_!!_#|' &&\""
+ ac_cs_awk_read_file='
+ print "|#_!!_#|"
+ print "cat " F[key] " &&"
+ '$ac_cs_awk_pipe_init
+ # The final `:' finishes the AND list.
+ ac_cs_awk_pipe_fini='END { print "|#_!!_#|"; print ":" }'
+fi
+ac_cr=`echo X | tr X '\015'`
+# On cygwin, bash can eat \r inside `` if the user requested igncr.
+# But we know of no other shell where ac_cr would be empty at this
+# point, so we can use a bashism as a fallback.
+if test "x$ac_cr" = x; then
+ eval ac_cr=\$\'\\r\'
+fi
+ac_cs_awk_cr=`$AWK 'BEGIN { print "a\rb" }' </dev/null 2>/dev/null`
+if test "$ac_cs_awk_cr" = "a${ac_cr}b"; then
+ ac_cs_awk_cr='\\r'
+else
+ ac_cs_awk_cr=$ac_cr
+fi
+
+echo 'BEGIN {' >"$tmp/subs1.awk" &&
+_ACEOF
+
+# Create commands to substitute file output variables.
+{
+ echo "cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1" &&
+ echo 'cat >>"\$tmp/subs1.awk" <<\\_ACAWK &&' &&
+ echo "$ac_subst_files" | sed 's/.*/F["&"]="$&"/' &&
+ echo "_ACAWK" &&
+ echo "_ACEOF"
+} >conf$$files.sh &&
+. ./conf$$files.sh ||
+ as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5
+rm -f conf$$files.sh
+
+{
+ echo "cat >conf$$subs.awk <<_ACEOF" &&
+ echo "$ac_subst_vars" | sed 's/.*/&!$&$ac_delim/' &&
+ echo "_ACEOF"
+} >conf$$subs.sh ||
+ as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5
+ac_delim_num=`echo "$ac_subst_vars" | grep -c '^'`
+ac_delim='%!_!# '
+for ac_last_try in false false false false false :; do
+ . ./conf$$subs.sh ||
+ as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5
+
+ ac_delim_n=`sed -n "s/.*$ac_delim\$/X/p" conf$$subs.awk | grep -c X`
+ if test $ac_delim_n = $ac_delim_num; then
+ break
+ elif $ac_last_try; then
+ as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5
+ else
+ ac_delim="$ac_delim!$ac_delim _$ac_delim!! "
+ fi
+done
+rm -f conf$$subs.sh
+
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+cat >>"\$tmp/subs1.awk" <<\\_ACAWK &&
+_ACEOF
+sed -n '
+h
+s/^/S["/; s/!.*/"]=/
+p
+g
+s/^[^!]*!//
+:repl
+t repl
+s/'"$ac_delim"'$//
+t delim
+:nl
+h
+s/\(.\{148\}\)..*/\1/
+t more1
+s/["\\]/\\&/g; s/^/"/; s/$/\\n"\\/
+p
+n
+b repl
+:more1
+s/["\\]/\\&/g; s/^/"/; s/$/"\\/
+p
+g
+s/.\{148\}//
+t nl
+:delim
+h
+s/\(.\{148\}\)..*/\1/
+t more2
+s/["\\]/\\&/g; s/^/"/; s/$/"/
+p
+b
+:more2
+s/["\\]/\\&/g; s/^/"/; s/$/"\\/
+p
+g
+s/.\{148\}//
+t delim
+' <conf$$subs.awk | sed '
+/^[^""]/{
+ N
+ s/\n//
+}
+' >>$CONFIG_STATUS || ac_write_fail=1
+rm -f conf$$subs.awk
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+_ACAWK
+cat >>"\$tmp/subs1.awk" <<_ACAWK &&
+ for (key in S) S_is_set[key] = 1
+ FS = ""
+ \$ac_cs_awk_pipe_init
+}
+{
+ line = $ 0
+ nfields = split(line, field, "@")
+ substed = 0
+ len = length(field[1])
+ for (i = 2; i < nfields; i++) {
+ key = field[i]
+ keylen = length(key)
+ if (S_is_set[key]) {
+ value = S[key]
+ line = substr(line, 1, len) "" value "" substr(line, len + keylen + 3)
+ len += length(value) + length(field[++i])
+ substed = 1
+ } else
+ len += 1 + keylen
+ }
+ if (nfields == 3 && !substed) {
+ key = field[2]
+ if (F[key] != "" && line ~ /^[ ]*@.*@[ ]*$/) {
+ \$ac_cs_awk_read_file
+ next
+ }
+ }
+ print line
+}
+\$ac_cs_awk_pipe_fini
+_ACAWK
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+if sed "s/$ac_cr//" < /dev/null > /dev/null 2>&1; then
+ sed "s/$ac_cr\$//; s/$ac_cr/$ac_cs_awk_cr/g"
+else
+ cat
+fi < "$tmp/subs1.awk" > "$tmp/subs.awk" \
+ || as_fn_error $? "could not setup config files machinery" "$LINENO" 5
+_ACEOF
+
+# VPATH may cause trouble with some makes, so we remove sole $(srcdir),
+# ${srcdir} and @srcdir@ entries from VPATH if srcdir is ".", strip leading and
+# trailing colons and then remove the whole line if VPATH becomes empty
+# (actually we leave an empty line to preserve line numbers).
+if test "x$srcdir" = x.; then
+ ac_vpsub='/^[ ]*VPATH[ ]*=[ ]*/{
+h
+s///
+s/^/:/
+s/[ ]*$/:/
+s/:\$(srcdir):/:/g
+s/:\${srcdir}:/:/g
+s/:@srcdir@:/:/g
+s/^:*//
+s/:*$//
+x
+s/\(=[ ]*\).*/\1/
+G
+s/\n//
+s/^[^=]*=[ ]*$//
+}'
+fi
+
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+fi # test -n "$CONFIG_FILES"
+
+# Set up the scripts for CONFIG_HEADERS section.
+# No need to generate them if there are no CONFIG_HEADERS.
+# This happens for instance with `./config.status Makefile'.
+if test -n "$CONFIG_HEADERS"; then
+cat >"$tmp/defines.awk" <<\_ACAWK ||
+BEGIN {
+_ACEOF
+
+# Transform confdefs.h into an awk script `defines.awk', embedded as
+# here-document in config.status, that substitutes the proper values into
+# config.h.in to produce config.h.
+
+# Create a delimiter string that does not exist in confdefs.h, to ease
+# handling of long lines.
+ac_delim='%!_!# '
+for ac_last_try in false false :; do
+ ac_t=`sed -n "/$ac_delim/p" confdefs.h`
+ if test -z "$ac_t"; then
+ break
+ elif $ac_last_try; then
+ as_fn_error $? "could not make $CONFIG_HEADERS" "$LINENO" 5
+ else
+ ac_delim="$ac_delim!$ac_delim _$ac_delim!! "
+ fi
+done
+
+# For the awk script, D is an array of macro values keyed by name,
+# likewise P contains macro parameters if any. Preserve backslash
+# newline sequences.
+
+ac_word_re=[_$as_cr_Letters][_$as_cr_alnum]*
+sed -n '
+s/.\{148\}/&'"$ac_delim"'/g
+t rset
+:rset
+s/^[ ]*#[ ]*define[ ][ ]*/ /
+t def
+d
+:def
+s/\\$//
+t bsnl
+s/["\\]/\\&/g
+s/^ \('"$ac_word_re"'\)\(([^()]*)\)[ ]*\(.*\)/P["\1"]="\2"\
+D["\1"]=" \3"/p
+s/^ \('"$ac_word_re"'\)[ ]*\(.*\)/D["\1"]=" \2"/p
+d
+:bsnl
+s/["\\]/\\&/g
+s/^ \('"$ac_word_re"'\)\(([^()]*)\)[ ]*\(.*\)/P["\1"]="\2"\
+D["\1"]=" \3\\\\\\n"\\/p
+t cont
+s/^ \('"$ac_word_re"'\)[ ]*\(.*\)/D["\1"]=" \2\\\\\\n"\\/p
+t cont
+d
+:cont
+n
+s/.\{148\}/&'"$ac_delim"'/g
+t clear
+:clear
+s/\\$//
+t bsnlc
+s/["\\]/\\&/g; s/^/"/; s/$/"/p
+d
+:bsnlc
+s/["\\]/\\&/g; s/^/"/; s/$/\\\\\\n"\\/p
+b cont
+' <confdefs.h | sed '
+s/'"$ac_delim"'/"\\\
+"/g' >>$CONFIG_STATUS || ac_write_fail=1
+
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+ for (key in D) D_is_set[key] = 1
+ FS = ""
+}
+/^[\t ]*#[\t ]*(define|undef)[\t ]+$ac_word_re([\t (]|\$)/ {
+ line = \$ 0
+ split(line, arg, " ")
+ if (arg[1] == "#") {
+ defundef = arg[2]
+ mac1 = arg[3]
+ } else {
+ defundef = substr(arg[1], 2)
+ mac1 = arg[2]
+ }
+ split(mac1, mac2, "(") #)
+ macro = mac2[1]
+ prefix = substr(line, 1, index(line, defundef) - 1)
+ if (D_is_set[macro]) {
+ # Preserve the white space surrounding the "#".
+ print prefix "define", macro P[macro] D[macro]
+ next
+ } else {
+ # Replace #undef with comments. This is necessary, for example,
+ # in the case of _POSIX_SOURCE, which is predefined and required
+ # on some systems where configure will not decide to define it.
+ if (defundef == "undef") {
+ print "/*", prefix defundef, macro, "*/"
+ next
+ }
+ }
+}
+{ print }
+_ACAWK
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+ as_fn_error $? "could not setup config headers machinery" "$LINENO" 5
+fi # test -n "$CONFIG_HEADERS"
+
+
+eval set X " :F $CONFIG_FILES :H $CONFIG_HEADERS :C $CONFIG_COMMANDS"
+shift
+for ac_tag
+do
+ case $ac_tag in
+ :[FHLC]) ac_mode=$ac_tag; continue;;
+ esac
+ case $ac_mode$ac_tag in
+ :[FHL]*:*);;
+ :L* | :C*:*) as_fn_error $? "invalid tag \`$ac_tag'" "$LINENO" 5;;
+ :[FH]-) ac_tag=-:-;;
+ :[FH]*) ac_tag=$ac_tag:$ac_tag.in;;
+ esac
+ ac_save_IFS=$IFS
+ IFS=:
+ set x $ac_tag
+ IFS=$ac_save_IFS
+ shift
+ ac_file=$1
+ shift
+
+ case $ac_mode in
+ :L) ac_source=$1;;
+ :[FH])
+ ac_file_inputs=
+ for ac_f
+ do
+ case $ac_f in
+ -) ac_f="$tmp/stdin";;
+ *) # Look for the file first in the build tree, then in the source tree
+ # (if the path is not absolute). The absolute path cannot be DOS-style,
+ # because $ac_f cannot contain `:'.
+ test -f "$ac_f" ||
+ case $ac_f in
+ [\\/$]*) false;;
+ *) test -f "$srcdir/$ac_f" && ac_f="$srcdir/$ac_f";;
+ esac ||
+ as_fn_error 1 "cannot find input file: \`$ac_f'" "$LINENO" 5;;
+ esac
+ case $ac_f in *\'*) ac_f=`$as_echo "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac
+ as_fn_append ac_file_inputs " '$ac_f'"
+ done
+
+ # Let's still pretend it is `configure' which instantiates (i.e., don't
+ # use $as_me), people would be surprised to read:
+ # /* config.h. Generated by config.status. */
+ configure_input='Generated from '`
+ $as_echo "$*" | sed 's|^[^:]*/||;s|:[^:]*/|, |g'
+ `' by configure.'
+ if test x"$ac_file" != x-; then
+ configure_input="$ac_file. $configure_input"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: creating $ac_file" >&5
+$as_echo "$as_me: creating $ac_file" >&6;}
+ fi
+ # Neutralize special characters interpreted by sed in replacement strings.
+ case $configure_input in #(
+ *\&* | *\|* | *\\* )
+ ac_sed_conf_input=`$as_echo "$configure_input" |
+ sed 's/[\\\\&|]/\\\\&/g'`;; #(
+ *) ac_sed_conf_input=$configure_input;;
+ esac
+
+ case $ac_tag in
+ *:-:* | *:-) cat >"$tmp/stdin" \
+ || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;;
+ esac
+ ;;
+ esac
+
+ ac_dir=`$as_dirname -- "$ac_file" ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+ X"$ac_file" : 'X\(//\)[^/]' \| \
+ X"$ac_file" : 'X\(//\)$' \| \
+ X"$ac_file" : 'X\(/\)' \| . 2>/dev/null ||
+$as_echo X"$ac_file" |
+ sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)[^/].*/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\).*/{
+ s//\1/
+ q
+ }
+ s/.*/./; q'`
+ as_dir="$ac_dir"; as_fn_mkdir_p
+ ac_builddir=.
+
+case "$ac_dir" in
+.) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;;
+*)
+ ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'`
+ # A ".." for each directory in $ac_dir_suffix.
+ ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'`
+ case $ac_top_builddir_sub in
+ "") ac_top_builddir_sub=. ac_top_build_prefix= ;;
+ *) ac_top_build_prefix=$ac_top_builddir_sub/ ;;
+ esac ;;
+esac
+ac_abs_top_builddir=$ac_pwd
+ac_abs_builddir=$ac_pwd$ac_dir_suffix
+# for backward compatibility:
+ac_top_builddir=$ac_top_build_prefix
+
+case $srcdir in
+ .) # We are building in place.
+ ac_srcdir=.
+ ac_top_srcdir=$ac_top_builddir_sub
+ ac_abs_top_srcdir=$ac_pwd ;;
+ [\\/]* | ?:[\\/]* ) # Absolute name.
+ ac_srcdir=$srcdir$ac_dir_suffix;
+ ac_top_srcdir=$srcdir
+ ac_abs_top_srcdir=$srcdir ;;
+ *) # Relative name.
+ ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix
+ ac_top_srcdir=$ac_top_build_prefix$srcdir
+ ac_abs_top_srcdir=$ac_pwd/$srcdir ;;
+esac
+ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix
+
+
+ case $ac_mode in
+ :F)
+ #
+ # CONFIG_FILE
+ #
+
+ case $INSTALL in
+ [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;;
+ *) ac_INSTALL=$ac_top_build_prefix$INSTALL ;;
+ esac
+ ac_MKDIR_P=$MKDIR_P
+ case $MKDIR_P in
+ [\\/$]* | ?:[\\/]* ) ;;
+ */*) ac_MKDIR_P=$ac_top_build_prefix$MKDIR_P ;;
+ esac
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+# If the template does not know about datarootdir, expand it.
+# FIXME: This hack should be removed a few years after 2.60.
+ac_datarootdir_hack=; ac_datarootdir_seen=
+ac_sed_dataroot='
+/datarootdir/ {
+ p
+ q
+}
+/@datadir@/p
+/@docdir@/p
+/@infodir@/p
+/@localedir@/p
+/@mandir@/p'
+case `eval "sed -n \"\$ac_sed_dataroot\" $ac_file_inputs"` in
+*datarootdir*) ac_datarootdir_seen=yes;;
+*@datadir@*|*@docdir@*|*@infodir@*|*@localedir@*|*@mandir@*)
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&5
+$as_echo "$as_me: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&2;}
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+ ac_datarootdir_hack='
+ s&@datadir@&$datadir&g
+ s&@docdir@&$docdir&g
+ s&@infodir@&$infodir&g
+ s&@localedir@&$localedir&g
+ s&@mandir@&$mandir&g
+ s&\\\${datarootdir}&$datarootdir&g' ;;
+esac
+_ACEOF
+
+# Neutralize VPATH when `$srcdir' = `.'.
+# Shell code in configure.ac might set extrasub.
+# FIXME: do we really want to maintain this feature?
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+ac_sed_extra="$ac_vpsub
+$extrasub
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+:t
+/@[a-zA-Z_][a-zA-Z_0-9]*@/!b
+s|@configure_input@|$ac_sed_conf_input|;t t
+s&@top_builddir@&$ac_top_builddir_sub&;t t
+s&@top_build_prefix@&$ac_top_build_prefix&;t t
+s&@srcdir@&$ac_srcdir&;t t
+s&@abs_srcdir@&$ac_abs_srcdir&;t t
+s&@top_srcdir@&$ac_top_srcdir&;t t
+s&@abs_top_srcdir@&$ac_abs_top_srcdir&;t t
+s&@builddir@&$ac_builddir&;t t
+s&@abs_builddir@&$ac_abs_builddir&;t t
+s&@abs_top_builddir@&$ac_abs_top_builddir&;t t
+s&@INSTALL@&$ac_INSTALL&;t t
+s&@MKDIR_P@&$ac_MKDIR_P&;t t
+$ac_datarootdir_hack
+"
+eval sed \"\$ac_sed_extra\" "$ac_file_inputs" |
+if $ac_cs_awk_getline; then
+ $AWK -f "$tmp/subs.awk"
+else
+ $AWK -f "$tmp/subs.awk" | $SHELL
+fi >$tmp/out \
+ || as_fn_error $? "could not create $ac_file" "$LINENO" 5
+
+test -z "$ac_datarootdir_hack$ac_datarootdir_seen" &&
+ { ac_out=`sed -n '/\${datarootdir}/p' "$tmp/out"`; test -n "$ac_out"; } &&
+ { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' "$tmp/out"`; test -z "$ac_out"; } &&
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable \`datarootdir'
+which seems to be undefined. Please make sure it is defined" >&5
+$as_echo "$as_me: WARNING: $ac_file contains a reference to the variable \`datarootdir'
+which seems to be undefined. Please make sure it is defined" >&2;}
+
+ rm -f "$tmp/stdin"
+ case $ac_file in
+ -) cat "$tmp/out" && rm -f "$tmp/out";;
+ *) rm -f "$ac_file" && mv "$tmp/out" "$ac_file";;
+ esac \
+ || as_fn_error $? "could not create $ac_file" "$LINENO" 5
+ ;;
+ :H)
+ #
+ # CONFIG_HEADER
+ #
+ if test x"$ac_file" != x-; then
+ {
+ $as_echo "/* $configure_input */" \
+ && eval '$AWK -f "$tmp/defines.awk"' "$ac_file_inputs"
+ } >"$tmp/config.h" \
+ || as_fn_error $? "could not create $ac_file" "$LINENO" 5
+ if diff "$ac_file" "$tmp/config.h" >/dev/null 2>&1; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: $ac_file is unchanged" >&5
+$as_echo "$as_me: $ac_file is unchanged" >&6;}
+ else
+ rm -f "$ac_file"
+ mv "$tmp/config.h" "$ac_file" \
+ || as_fn_error $? "could not create $ac_file" "$LINENO" 5
+ fi
+ else
+ $as_echo "/* $configure_input */" \
+ && eval '$AWK -f "$tmp/defines.awk"' "$ac_file_inputs" \
+ || as_fn_error $? "could not create -" "$LINENO" 5
+ fi
+# Compute "$ac_file"'s index in $config_headers.
+_am_arg="$ac_file"
+_am_stamp_count=1
+for _am_header in $config_headers :; do
+ case $_am_header in
+ $_am_arg | $_am_arg:* )
+ break ;;
+ * )
+ _am_stamp_count=`expr $_am_stamp_count + 1` ;;
+ esac
+done
+echo "timestamp for $_am_arg" >`$as_dirname -- "$_am_arg" ||
+$as_expr X"$_am_arg" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+ X"$_am_arg" : 'X\(//\)[^/]' \| \
+ X"$_am_arg" : 'X\(//\)$' \| \
+ X"$_am_arg" : 'X\(/\)' \| . 2>/dev/null ||
+$as_echo X"$_am_arg" |
+ sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)[^/].*/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\).*/{
+ s//\1/
+ q
+ }
+ s/.*/./; q'`/stamp-h$_am_stamp_count
+ ;;
+
+ :C) { $as_echo "$as_me:${as_lineno-$LINENO}: executing $ac_file commands" >&5
+$as_echo "$as_me: executing $ac_file commands" >&6;}
+ ;;
+ esac
+
+
+ case $ac_file$ac_mode in
+ "depfiles":C) test x"$AMDEP_TRUE" != x"" || {
+ # Autoconf 2.62 quotes --file arguments for eval, but not when files
+ # are listed without --file. Let's play safe and only enable the eval
+ # if we detect the quoting.
+ case $CONFIG_FILES in
+ *\'*) eval set x "$CONFIG_FILES" ;;
+ *) set x $CONFIG_FILES ;;
+ esac
+ shift
+ for mf
+ do
+ # Strip MF so we end up with the name of the file.
+ mf=`echo "$mf" | sed -e 's/:.*$//'`
+ # Check whether this is an Automake generated Makefile or not.
+ # We used to match only the files named `Makefile.in', but
+ # some people rename them; so instead we look at the file content.
+ # Grep'ing the first line is not enough: some people post-process
+ # each Makefile.in and add a new line on top of each file to say so.
+ # Grep'ing the whole file is not good either: AIX grep has a line
+ # limit of 2048, but all sed's we know have understand at least 4000.
+ if sed -n 's,^#.*generated by automake.*,X,p' "$mf" | grep X >/dev/null 2>&1; then
+ dirpart=`$as_dirname -- "$mf" ||
+$as_expr X"$mf" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+ X"$mf" : 'X\(//\)[^/]' \| \
+ X"$mf" : 'X\(//\)$' \| \
+ X"$mf" : 'X\(/\)' \| . 2>/dev/null ||
+$as_echo X"$mf" |
+ sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)[^/].*/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\).*/{
+ s//\1/
+ q
+ }
+ s/.*/./; q'`
+ else
+ continue
+ fi
+ # Extract the definition of DEPDIR, am__include, and am__quote
+ # from the Makefile without running `make'.
+ DEPDIR=`sed -n 's/^DEPDIR = //p' < "$mf"`
+ test -z "$DEPDIR" && continue
+ am__include=`sed -n 's/^am__include = //p' < "$mf"`
+ test -z "am__include" && continue
+ am__quote=`sed -n 's/^am__quote = //p' < "$mf"`
+ # When using ansi2knr, U may be empty or an underscore; expand it
+ U=`sed -n 's/^U = //p' < "$mf"`
+ # Find all dependency output files, they are included files with
+ # $(DEPDIR) in their names. We invoke sed twice because it is the
+ # simplest approach to changing $(DEPDIR) to its actual value in the
+ # expansion.
+ for file in `sed -n "
+ s/^$am__include $am__quote\(.*(DEPDIR).*\)$am__quote"'$/\1/p' <"$mf" | \
+ sed -e 's/\$(DEPDIR)/'"$DEPDIR"'/g' -e 's/\$U/'"$U"'/g'`; do
+ # Make sure the directory exists.
+ test -f "$dirpart/$file" && continue
+ fdir=`$as_dirname -- "$file" ||
+$as_expr X"$file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+ X"$file" : 'X\(//\)[^/]' \| \
+ X"$file" : 'X\(//\)$' \| \
+ X"$file" : 'X\(/\)' \| . 2>/dev/null ||
+$as_echo X"$file" |
+ sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)[^/].*/{
+ s//\1/
+ q
+ }
+ /^X\(\/\/\)$/{
+ s//\1/
+ q
+ }
+ /^X\(\/\).*/{
+ s//\1/
+ q
+ }
+ s/.*/./; q'`
+ as_dir=$dirpart/$fdir; as_fn_mkdir_p
+ # echo "creating $dirpart/$file"
+ echo '# dummy' > "$dirpart/$file"
+ done
+ done
+}
+ ;;
+ "libtool":C)
+
+ # See if we are running on zsh, and set the options which allow our
+ # commands through without removal of \ escapes.
+ if test -n "${ZSH_VERSION+set}" ; then
+ setopt NO_GLOB_SUBST
+ fi
+
+ cfgfile="${ofile}T"
+ trap "$RM \"$cfgfile\"; exit 1" 1 2 15
+ $RM "$cfgfile"
+
+ cat <<_LT_EOF >> "$cfgfile"
+#! $SHELL
+
+# `$ECHO "$ofile" | sed 's%^.*/%%'` - Provide generalized library-building support services.
+# Generated automatically by $as_me ($PACKAGE$TIMESTAMP) $VERSION
+# Libtool was configured on host `(hostname || uname -n) 2>/dev/null | sed 1q`:
+# NOTE: Changes made to this file will be lost: look at ltmain.sh.
+#
+# Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, 2005,
+# 2006, 2007, 2008 Free Software Foundation, Inc.
+# Written by Gordon Matzigkeit, 1996
+#
+# This file is part of GNU Libtool.
+#
+# GNU Libtool is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of
+# the License, or (at your option) any later version.
+#
+# As a special exception to the GNU General Public License,
+# if you distribute this file as part of a program or library that
+# is built using GNU Libtool, you may include this file under the
+# same distribution terms that you use for the rest of that program.
+#
+# GNU Libtool is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Libtool; see the file COPYING. If not, a copy
+# can be downloaded from http://www.gnu.org/licenses/gpl.html, or
+# obtained by writing to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+
+# The names of the tagged configurations supported by this script.
+available_tags=""
+
+# ### BEGIN LIBTOOL CONFIG
+
+# Which release of libtool.m4 was used?
+macro_version=$macro_version
+macro_revision=$macro_revision
+
+# Whether or not to build shared libraries.
+build_libtool_libs=$enable_shared
+
+# Whether or not to build static libraries.
+build_old_libs=$enable_static
+
+# What type of objects to build.
+pic_mode=$pic_mode
+
+# Whether or not to optimize for fast installation.
+fast_install=$enable_fast_install
+
+# The host system.
+host_alias=$host_alias
+host=$host
+host_os=$host_os
+
+# The build system.
+build_alias=$build_alias
+build=$build
+build_os=$build_os
+
+# A sed program that does not truncate output.
+SED=$lt_SED
+
+# Sed that helps us avoid accidentally triggering echo(1) options like -n.
+Xsed="\$SED -e 1s/^X//"
+
+# A grep program that handles long lines.
+GREP=$lt_GREP
+
+# An ERE matcher.
+EGREP=$lt_EGREP
+
+# A literal string matcher.
+FGREP=$lt_FGREP
+
+# A BSD- or MS-compatible name lister.
+NM=$lt_NM
+
+# Whether we need soft or hard links.
+LN_S=$lt_LN_S
+
+# What is the maximum length of a command?
+max_cmd_len=$max_cmd_len
+
+# Object file suffix (normally "o").
+objext=$ac_objext
+
+# Executable file suffix (normally "").
+exeext=$exeext
+
+# whether the shell understands "unset".
+lt_unset=$lt_unset
+
+# turn spaces into newlines.
+SP2NL=$lt_lt_SP2NL
+
+# turn newlines into spaces.
+NL2SP=$lt_lt_NL2SP
+
+# How to create reloadable object files.
+reload_flag=$lt_reload_flag
+reload_cmds=$lt_reload_cmds
+
+# An object symbol dumper.
+OBJDUMP=$lt_OBJDUMP
+
+# Method to check whether dependent libraries are shared objects.
+deplibs_check_method=$lt_deplibs_check_method
+
+# Command to use when deplibs_check_method == "file_magic".
+file_magic_cmd=$lt_file_magic_cmd
+
+# The archiver.
+AR=$lt_AR
+AR_FLAGS=$lt_AR_FLAGS
+
+# A symbol stripping program.
+STRIP=$lt_STRIP
+
+# Commands used to install an old-style archive.
+RANLIB=$lt_RANLIB
+old_postinstall_cmds=$lt_old_postinstall_cmds
+old_postuninstall_cmds=$lt_old_postuninstall_cmds
+
+# A C compiler.
+LTCC=$lt_CC
+
+# LTCC compiler flags.
+LTCFLAGS=$lt_CFLAGS
+
+# Take the output of nm and produce a listing of raw symbols and C names.
+global_symbol_pipe=$lt_lt_cv_sys_global_symbol_pipe
+
+# Transform the output of nm in a proper C declaration.
+global_symbol_to_cdecl=$lt_lt_cv_sys_global_symbol_to_cdecl
+
+# Transform the output of nm in a C name address pair.
+global_symbol_to_c_name_address=$lt_lt_cv_sys_global_symbol_to_c_name_address
+
+# Transform the output of nm in a C name address pair when lib prefix is needed.
+global_symbol_to_c_name_address_lib_prefix=$lt_lt_cv_sys_global_symbol_to_c_name_address_lib_prefix
+
+# The name of the directory that contains temporary libtool files.
+objdir=$objdir
+
+# Shell to use when invoking shell scripts.
+SHELL=$lt_SHELL
+
+# An echo program that does not interpret backslashes.
+ECHO=$lt_ECHO
+
+# Used to examine libraries when file_magic_cmd begins with "file".
+MAGIC_CMD=$MAGIC_CMD
+
+# Must we lock files when doing compilation?
+need_locks=$lt_need_locks
+
+# Tool to manipulate archived DWARF debug symbol files on Mac OS X.
+DSYMUTIL=$lt_DSYMUTIL
+
+# Tool to change global to local symbols on Mac OS X.
+NMEDIT=$lt_NMEDIT
+
+# Tool to manipulate fat objects and archives on Mac OS X.
+LIPO=$lt_LIPO
+
+# ldd/readelf like tool for Mach-O binaries on Mac OS X.
+OTOOL=$lt_OTOOL
+
+# ldd/readelf like tool for 64 bit Mach-O binaries on Mac OS X 10.4.
+OTOOL64=$lt_OTOOL64
+
+# Old archive suffix (normally "a").
+libext=$libext
+
+# Shared library suffix (normally ".so").
+shrext_cmds=$lt_shrext_cmds
+
+# The commands to extract the exported symbol list from a shared archive.
+extract_expsyms_cmds=$lt_extract_expsyms_cmds
+
+# Variables whose values should be saved in libtool wrapper scripts and
+# restored at link time.
+variables_saved_for_relink=$lt_variables_saved_for_relink
+
+# Do we need the "lib" prefix for modules?
+need_lib_prefix=$need_lib_prefix
+
+# Do we need a version for libraries?
+need_version=$need_version
+
+# Library versioning type.
+version_type=$version_type
+
+# Shared library runtime path variable.
+runpath_var=$runpath_var
+
+# Shared library path variable.
+shlibpath_var=$shlibpath_var
+
+# Is shlibpath searched before the hard-coded library search path?
+shlibpath_overrides_runpath=$shlibpath_overrides_runpath
+
+# Format of library name prefix.
+libname_spec=$lt_libname_spec
+
+# List of archive names. First name is the real one, the rest are links.
+# The last name is the one that the linker finds with -lNAME
+library_names_spec=$lt_library_names_spec
+
+# The coded name of the library, if different from the real name.
+soname_spec=$lt_soname_spec
+
+# Command to use after installation of a shared archive.
+postinstall_cmds=$lt_postinstall_cmds
+
+# Command to use after uninstallation of a shared archive.
+postuninstall_cmds=$lt_postuninstall_cmds
+
+# Commands used to finish a libtool library installation in a directory.
+finish_cmds=$lt_finish_cmds
+
+# As "finish_cmds", except a single script fragment to be evaled but
+# not shown.
+finish_eval=$lt_finish_eval
+
+# Whether we should hardcode library paths into libraries.
+hardcode_into_libs=$hardcode_into_libs
+
+# Compile-time system search path for libraries.
+sys_lib_search_path_spec=$lt_sys_lib_search_path_spec
+
+# Run-time system search path for libraries.
+sys_lib_dlsearch_path_spec=$lt_sys_lib_dlsearch_path_spec
+
+# Whether dlopen is supported.
+dlopen_support=$enable_dlopen
+
+# Whether dlopen of programs is supported.
+dlopen_self=$enable_dlopen_self
+
+# Whether dlopen of statically linked programs is supported.
+dlopen_self_static=$enable_dlopen_self_static
+
+# Commands to strip libraries.
+old_striplib=$lt_old_striplib
+striplib=$lt_striplib
+
+
+# The linker used to build libraries.
+LD=$lt_LD
+
+# Commands used to build an old-style archive.
+old_archive_cmds=$lt_old_archive_cmds
+
+# A language specific compiler.
+CC=$lt_compiler
+
+# Is the compiler the GNU compiler?
+with_gcc=$GCC
+
+# Compiler flag to turn off builtin functions.
+no_builtin_flag=$lt_lt_prog_compiler_no_builtin_flag
+
+# How to pass a linker flag through the compiler.
+wl=$lt_lt_prog_compiler_wl
+
+# Additional compiler flags for building library objects.
+pic_flag=$lt_lt_prog_compiler_pic
+
+# Compiler flag to prevent dynamic linking.
+link_static_flag=$lt_lt_prog_compiler_static
+
+# Does compiler simultaneously support -c and -o options?
+compiler_c_o=$lt_lt_cv_prog_compiler_c_o
+
+# Whether or not to add -lc for building shared libraries.
+build_libtool_need_lc=$archive_cmds_need_lc
+
+# Whether or not to disallow shared libs when runtime libs are static.
+allow_libtool_libs_with_static_runtimes=$enable_shared_with_static_runtimes
+
+# Compiler flag to allow reflexive dlopens.
+export_dynamic_flag_spec=$lt_export_dynamic_flag_spec
+
+# Compiler flag to generate shared objects directly from archives.
+whole_archive_flag_spec=$lt_whole_archive_flag_spec
+
+# Whether the compiler copes with passing no objects directly.
+compiler_needs_object=$lt_compiler_needs_object
+
+# Create an old-style archive from a shared archive.
+old_archive_from_new_cmds=$lt_old_archive_from_new_cmds
+
+# Create a temporary old-style archive to link instead of a shared archive.
+old_archive_from_expsyms_cmds=$lt_old_archive_from_expsyms_cmds
+
+# Commands used to build a shared archive.
+archive_cmds=$lt_archive_cmds
+archive_expsym_cmds=$lt_archive_expsym_cmds
+
+# Commands used to build a loadable module if different from building
+# a shared archive.
+module_cmds=$lt_module_cmds
+module_expsym_cmds=$lt_module_expsym_cmds
+
+# Whether we are building with GNU ld or not.
+with_gnu_ld=$lt_with_gnu_ld
+
+# Flag that allows shared libraries with undefined symbols to be built.
+allow_undefined_flag=$lt_allow_undefined_flag
+
+# Flag that enforces no undefined symbols.
+no_undefined_flag=$lt_no_undefined_flag
+
+# Flag to hardcode \$libdir into a binary during linking.
+# This must work even if \$libdir does not exist
+hardcode_libdir_flag_spec=$lt_hardcode_libdir_flag_spec
+
+# If ld is used when linking, flag to hardcode \$libdir into a binary
+# during linking. This must work even if \$libdir does not exist.
+hardcode_libdir_flag_spec_ld=$lt_hardcode_libdir_flag_spec_ld
+
+# Whether we need a single "-rpath" flag with a separated argument.
+hardcode_libdir_separator=$lt_hardcode_libdir_separator
+
+# Set to "yes" if using DIR/libNAME\${shared_ext} during linking hardcodes
+# DIR into the resulting binary.
+hardcode_direct=$hardcode_direct
+
+# Set to "yes" if using DIR/libNAME\${shared_ext} during linking hardcodes
+# DIR into the resulting binary and the resulting library dependency is
+# "absolute",i.e impossible to change by setting \${shlibpath_var} if the
+# library is relocated.
+hardcode_direct_absolute=$hardcode_direct_absolute
+
+# Set to "yes" if using the -LDIR flag during linking hardcodes DIR
+# into the resulting binary.
+hardcode_minus_L=$hardcode_minus_L
+
+# Set to "yes" if using SHLIBPATH_VAR=DIR during linking hardcodes DIR
+# into the resulting binary.
+hardcode_shlibpath_var=$hardcode_shlibpath_var
+
+# Set to "yes" if building a shared library automatically hardcodes DIR
+# into the library and all subsequent libraries and executables linked
+# against it.
+hardcode_automatic=$hardcode_automatic
+
+# Set to yes if linker adds runtime paths of dependent libraries
+# to runtime path list.
+inherit_rpath=$inherit_rpath
+
+# Whether libtool must link a program against all its dependency libraries.
+link_all_deplibs=$link_all_deplibs
+
+# Fix the shell variable \$srcfile for the compiler.
+fix_srcfile_path=$lt_fix_srcfile_path
+
+# Set to "yes" if exported symbols are required.
+always_export_symbols=$always_export_symbols
+
+# The commands to list exported symbols.
+export_symbols_cmds=$lt_export_symbols_cmds
+
+# Symbols that should not be listed in the preloaded symbols.
+exclude_expsyms=$lt_exclude_expsyms
+
+# Symbols that must always be exported.
+include_expsyms=$lt_include_expsyms
+
+# Commands necessary for linking programs (against libraries) with templates.
+prelink_cmds=$lt_prelink_cmds
+
+# Specify filename containing input files.
+file_list_spec=$lt_file_list_spec
+
+# How to hardcode a shared library path into an executable.
+hardcode_action=$hardcode_action
+
+# ### END LIBTOOL CONFIG
+
+_LT_EOF
+
+ case $host_os in
+ aix3*)
+ cat <<\_LT_EOF >> "$cfgfile"
+# AIX sometimes has problems with the GCC collect2 program. For some
+# reason, if we set the COLLECT_NAMES environment variable, the problems
+# vanish in a puff of smoke.
+if test "X${COLLECT_NAMES+set}" != Xset; then
+ COLLECT_NAMES=
+ export COLLECT_NAMES
+fi
+_LT_EOF
+ ;;
+ esac
+
+
+ltmain="$ac_aux_dir/ltmain.sh"
+
+
+ # We use sed instead of cat because bash on DJGPP gets confused if
+ # if finds mixed CR/LF and LF-only lines. Since sed operates in
+ # text mode, it properly converts lines to CR/LF. This bash problem
+ # is reportedly fixed, but why not run on old versions too?
+ sed '/^# Generated shell functions inserted here/q' "$ltmain" >> "$cfgfile" \
+ || (rm -f "$cfgfile"; exit 1)
+
+ case $xsi_shell in
+ yes)
+ cat << \_LT_EOF >> "$cfgfile"
+
+# func_dirname file append nondir_replacement
+# Compute the dirname of FILE. If nonempty, add APPEND to the result,
+# otherwise set result to NONDIR_REPLACEMENT.
+func_dirname ()
+{
+ case ${1} in
+ */*) func_dirname_result="${1%/*}${2}" ;;
+ * ) func_dirname_result="${3}" ;;
+ esac
+}
+
+# func_basename file
+func_basename ()
+{
+ func_basename_result="${1##*/}"
+}
+
+# func_dirname_and_basename file append nondir_replacement
+# perform func_basename and func_dirname in a single function
+# call:
+# dirname: Compute the dirname of FILE. If nonempty,
+# add APPEND to the result, otherwise set result
+# to NONDIR_REPLACEMENT.
+# value returned in "$func_dirname_result"
+# basename: Compute filename of FILE.
+# value retuned in "$func_basename_result"
+# Implementation must be kept synchronized with func_dirname
+# and func_basename. For efficiency, we do not delegate to
+# those functions but instead duplicate the functionality here.
+func_dirname_and_basename ()
+{
+ case ${1} in
+ */*) func_dirname_result="${1%/*}${2}" ;;
+ * ) func_dirname_result="${3}" ;;
+ esac
+ func_basename_result="${1##*/}"
+}
+
+# func_stripname prefix suffix name
+# strip PREFIX and SUFFIX off of NAME.
+# PREFIX and SUFFIX must not contain globbing or regex special
+# characters, hashes, percent signs, but SUFFIX may contain a leading
+# dot (in which case that matches only a dot).
+func_stripname ()
+{
+ # pdksh 5.2.14 does not do ${X%$Y} correctly if both X and Y are
+ # positional parameters, so assign one to ordinary parameter first.
+ func_stripname_result=${3}
+ func_stripname_result=${func_stripname_result#"${1}"}
+ func_stripname_result=${func_stripname_result%"${2}"}
+}
+
+# func_opt_split
+func_opt_split ()
+{
+ func_opt_split_opt=${1%%=*}
+ func_opt_split_arg=${1#*=}
+}
+
+# func_lo2o object
+func_lo2o ()
+{
+ case ${1} in
+ *.lo) func_lo2o_result=${1%.lo}.${objext} ;;
+ *) func_lo2o_result=${1} ;;
+ esac
+}
+
+# func_xform libobj-or-source
+func_xform ()
+{
+ func_xform_result=${1%.*}.lo
+}
+
+# func_arith arithmetic-term...
+func_arith ()
+{
+ func_arith_result=$(( $* ))
+}
+
+# func_len string
+# STRING may not start with a hyphen.
+func_len ()
+{
+ func_len_result=${#1}
+}
+
+_LT_EOF
+ ;;
+ *) # Bourne compatible functions.
+ cat << \_LT_EOF >> "$cfgfile"
+
+# func_dirname file append nondir_replacement
+# Compute the dirname of FILE. If nonempty, add APPEND to the result,
+# otherwise set result to NONDIR_REPLACEMENT.
+func_dirname ()
+{
+ # Extract subdirectory from the argument.
+ func_dirname_result=`$ECHO "X${1}" | $Xsed -e "$dirname"`
+ if test "X$func_dirname_result" = "X${1}"; then
+ func_dirname_result="${3}"
+ else
+ func_dirname_result="$func_dirname_result${2}"
+ fi
+}
+
+# func_basename file
+func_basename ()
+{
+ func_basename_result=`$ECHO "X${1}" | $Xsed -e "$basename"`
+}
+
+
+# func_stripname prefix suffix name
+# strip PREFIX and SUFFIX off of NAME.
+# PREFIX and SUFFIX must not contain globbing or regex special
+# characters, hashes, percent signs, but SUFFIX may contain a leading
+# dot (in which case that matches only a dot).
+# func_strip_suffix prefix name
+func_stripname ()
+{
+ case ${2} in
+ .*) func_stripname_result=`$ECHO "X${3}" \
+ | $Xsed -e "s%^${1}%%" -e "s%\\\\${2}\$%%"`;;
+ *) func_stripname_result=`$ECHO "X${3}" \
+ | $Xsed -e "s%^${1}%%" -e "s%${2}\$%%"`;;
+ esac
+}
+
+# sed scripts:
+my_sed_long_opt='1s/^\(-[^=]*\)=.*/\1/;q'
+my_sed_long_arg='1s/^-[^=]*=//'
+
+# func_opt_split
+func_opt_split ()
+{
+ func_opt_split_opt=`$ECHO "X${1}" | $Xsed -e "$my_sed_long_opt"`
+ func_opt_split_arg=`$ECHO "X${1}" | $Xsed -e "$my_sed_long_arg"`
+}
+
+# func_lo2o object
+func_lo2o ()
+{
+ func_lo2o_result=`$ECHO "X${1}" | $Xsed -e "$lo2o"`
+}
+
+# func_xform libobj-or-source
+func_xform ()
+{
+ func_xform_result=`$ECHO "X${1}" | $Xsed -e 's/\.[^.]*$/.lo/'`
+}
+
+# func_arith arithmetic-term...
+func_arith ()
+{
+ func_arith_result=`expr "$@"`
+}
+
+# func_len string
+# STRING may not start with a hyphen.
+func_len ()
+{
+ func_len_result=`expr "$1" : ".*" 2>/dev/null || echo $max_cmd_len`
+}
+
+_LT_EOF
+esac
+
+case $lt_shell_append in
+ yes)
+ cat << \_LT_EOF >> "$cfgfile"
+
+# func_append var value
+# Append VALUE to the end of shell variable VAR.
+func_append ()
+{
+ eval "$1+=\$2"
+}
+_LT_EOF
+ ;;
+ *)
+ cat << \_LT_EOF >> "$cfgfile"
+
+# func_append var value
+# Append VALUE to the end of shell variable VAR.
+func_append ()
+{
+ eval "$1=\$$1\$2"
+}
+
+_LT_EOF
+ ;;
+ esac
+
+
+ sed -n '/^# Generated shell functions inserted here/,$p' "$ltmain" >> "$cfgfile" \
+ || (rm -f "$cfgfile"; exit 1)
+
+ mv -f "$cfgfile" "$ofile" ||
+ (rm -f "$ofile" && cp "$cfgfile" "$ofile" && rm -f "$cfgfile")
+ chmod +x "$ofile"
+
+ ;;
+
+ esac
+done # for ac_tag
+
+
+as_fn_exit 0
+_ACEOF
+ac_clean_files=$ac_clean_files_save
+
+test $ac_write_fail = 0 ||
+ as_fn_error $? "write failure creating $CONFIG_STATUS" "$LINENO" 5
+
+
+# configure is writing to config.log, and then calls config.status.
+# config.status does its own redirection, appending to config.log.
+# Unfortunately, on DOS this fails, as config.log is still kept open
+# by configure, so config.status won't be able to write to it; its
+# output is simply discarded. So we exec the FD to /dev/null,
+# effectively closing config.log, so it can be properly (re)opened and
+# appended to by config.status. When coming back to configure, we
+# need to make the FD available again.
+if test "$no_create" != yes; then
+ ac_cs_success=:
+ ac_config_status_args=
+ test "$silent" = yes &&
+ ac_config_status_args="$ac_config_status_args --quiet"
+ exec 5>/dev/null
+ $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false
+ exec 5>>config.log
+ # Use ||, not &&, to avoid exiting from the if with $? = 1, which
+ # would make configure fail if this is the last instruction.
+ $ac_cs_success || as_fn_exit 1
+fi
+if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unrecognized options: $ac_unrecognized_opts" >&5
+$as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;}
+fi
+
+
+if test "$PROVIDERS_TO_BUILD" = ""; then
+ echo "***"
+ echo "*** No providers were configured. This may not be what you wanted."
+ echo "***"
+ exit 1
+fi
+
+for i in $PROVIDERS_TO_BUILD; do
+ v=${i}_dl
+ if test `eval echo \\$${v}` == "1"; then
+ dso="$i ${dso}"
+ else
+ builtin="$i ${builtin}"
+ fi
+done
+cat <<EOF
+***
+*** Built-in providers: ${builtin}
+*** DSO providers: ${dso}
+***
+EOF
diff --git a/configure.ac b/configure.ac
index e17d27f..28ca767 100644
--- a/configure.ac
+++ b/configure.ac
@@ -4,7 +4,7 @@ dnl
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ([2.60])
-AC_INIT([libfabric], [1.3.0], [ofiwg at lists.openfabrics.org])
+AC_INIT([libfabric], [1.4.0rc2], [ofiwg at lists.openfabrics.org])
AC_CONFIG_SRCDIR([src/fabric.c])
AC_CONFIG_AUX_DIR(config)
AC_CONFIG_MACRO_DIR(config)
@@ -37,11 +37,13 @@ AM_CONDITIONAL([MACOS], [test "x$macos" = "x1"])
AM_CONDITIONAL([LINUX], [test "x$linux" = "x1"])
AM_CONDITIONAL([FREEBSD], [test "x$freebsd" = "x1"])
+base_c_warn_flags="-Wall -Wundef -Wpointer-arith"
+
AC_ARG_ENABLE([debug],
[AS_HELP_STRING([--enable-debug],
[Enable debugging @<:@default=no@:>@])
],
- [CFLAGS="$CFLAGS -g -O0 -Wall -Wextra -Wundef -Wno-unused-parameter -Wno-sign-compare -Wno-missing-field-initializers"
+ [CFLAGS="-g -O0 ${base_c_warn_flags} -Wextra -Wno-unused-parameter -Wno-sign-compare -Wno-missing-field-initializers $CFLAGS"
dbg=1],
[enable_debug=no
dbg=0])
@@ -49,9 +51,9 @@ AC_ARG_ENABLE([debug],
AC_DEFINE_UNQUOTED([ENABLE_DEBUG],[$dbg],
[defined to 1 if libfabric was configured with --enable-debug, 0 otherwise])
-dnl Fix autoconf's habit of adding -g -O2 by default
-AS_IF([test -z "$CFLAGS"],
- [CFLAGS='-fvisibility=hidden -O2 -DNDEBUG -Wall -Wundef'])
+# Fix autoconf's habit of setting CFLAGS="-g -O2" by default while still
+# allowing the user to explicitly set CFLAGS=""
+: ${CFLAGS="-fvisibility=hidden -O2 -DNDEBUG ${base_c_warn_flags}"}
# AM PROG_AR did not exist pre AM 1.11.x (where x is somewhere >0 and
# <3), but it is necessary in AM 1.12.x.
@@ -65,7 +67,7 @@ if test "$with_valgrind" != "" && test "$with_valgrind" != "no"; then
AC_DEFINE([INCLUDE_VALGRIND], 1,
[Define to 1 to enable valgrind annotations])
if test -d $with_valgrind; then
- CPPFLAGS="$CPPLFAGS -I$with_valgrind/include"
+ CPPFLAGS="$CPPFLAGS -I$with_valgrind/include"
fi
fi
@@ -88,6 +90,7 @@ dnl Check for compiler features
AC_C_TYPEOF
LT_INIT
+LT_OUTPUT
dnl dlopen support is optional
AC_ARG_WITH([dlopen],
@@ -110,6 +113,15 @@ AC_CHECK_FUNC([pthread_spin_init],
[have_spinlock=1],
[have_spinlock=0])
+dnl shm_open not used in the common code on os-x
+
+if test "$macos" -eq 0; then
+AC_CHECK_FUNC([shm_open],
+ [],
+ [AC_SEARCH_LIBS([shm_open],[rt],[],
+ [AC_MSG_ERROR([shm_open() not found. libfabric requires shm_open.])])])
+fi
+
AC_DEFINE_UNQUOTED([PT_LOCK_SPIN], [$have_spinlock],
[Define to 1 if pthread_spin_init is available.])
@@ -162,6 +174,15 @@ AC_CACHE_CHECK(whether ld accepts --version-script, ac_cv_version_script,
ac_cv_version_script=no
fi])
+AC_ARG_ENABLE([embedded],
+ [AS_HELP_STRING([--enable-embedded],
+ [Enable embedded support (turns off symbol versioning) @<:@default=no@:>@])
+ ],
+ [ac_asm_symver_support=0
+ icc_symver_hack=1],
+ [enable_embedded=no])
+AM_CONDITIONAL([EMBEDDED], [test x"$enable_embedded" = x"yes"])
+
AM_CONDITIONAL(HAVE_LD_VERSION_SCRIPT, test "$ac_cv_version_script" = "yes")
dnl Disable symbol versioning when -ipo is in CFLAGS or ipo is disabled by icc.
@@ -217,6 +238,7 @@ AC_TRY_LINK(
AC_DEFINE_UNQUOTED([HAVE_ALIAS_ATTRIBUTE], [$ac_prog_cc_alias_symbols],
[Define to 1 if the linker supports alias attribute.])
+AC_CHECK_FUNCS([getifaddrs])
dnl Provider-specific checks
FI_PROVIDER_INIT
@@ -230,6 +252,8 @@ FI_PROVIDER_SETUP([usnic])
FI_PROVIDER_SETUP([mxm])
FI_PROVIDER_SETUP([gni])
FI_PROVIDER_SETUP([udp])
+FI_PROVIDER_SETUP([rxm])
+FI_PROVIDER_SETUP([rxd])
FI_PROVIDER_FINI
dnl Configure the .pc file
FI_PROVIDER_SETUP_PC
diff --git a/docs/providers b/docs/providers
deleted file mode 100644
index e964d2e..0000000
--- a/docs/providers
+++ /dev/null
@@ -1,111 +0,0 @@
-Provider Helper Routines
-========================
-In tree providers are strongly encouraged to take advantage of available
-common functionality supplied by the framework. This includes extending
-functionality where needed. Common functionality is defined by header
-files under the include directory, with source code available under src.
-
-
-// include/fi_enosys.h
-// src/enosys.c
-
-Providers that do not implement a specific interface should set their
-function pointers to one of the functions defined by these files. For
-example:
-
-static struct fi_ops prov_obj_ops = {
- .size = sizeof(struct fi_ops),
- .close = prov_obj_close, /* defined by provider */
- .bind = fi_no_bind,
- .control = fi_no_control,
- .ops_open = fi_no_ops_open,
-};
-
-Providers may find it helpful to copy the structures from fi_enosys.h into
-their source files, then modify them to reference their own implementations.
-If an 'enosys' version of a function (e.g. fi_no_xxx) is not defined in
-fi_enosys.h, then that function must be defined by the provider.
-
-
-// include/fi_indexer.h
-// src/indexer.c
-
-The indexer files provider routines for quickly converting between a pointer
-and an integer index.
-
-struct indexer:
-This converts a pointer into a integer, with integer values starting at 0.
-The returned value is selected by the indexer from a free list of available
-values. The indexer storage grows dynamically. It is useful for providers
-that must return an integer value to the application, but need to retrieve
-the pointer for use. This allows the provider to validate a value given
-by an application before the pointer is dereferenced.
-
-struct index_map:
-The index map is similar to the indexer, except that the caller selects the
-index location which to store the original pointer. An example use case is
-to associate a pointer with a file descriptor.
-
-
-// include/fi_list.h
-
-This provides inline implementations of single and double linked lists.
-
-dlist:
-This defines a double linked list.
-
-slist:
-This defines a single linked list. It may be used to implement variable
-sized software queues.
-
-dlistfd:
-This is a double linked list that is associated with a socketpair. It
-may be used when a thread should wait until an item is inserted onto the
-list. An example use of a dlistfd object is to communicate with a thread
-that handles progress or timeout/retries.
-
-
-// include/fi_rbuf.h
-
-This provides inline implemenations of ring buffers.
-
-ringbuf:
-This defines a fixed-sized ring buffer that supports variable-sized
-reads and writes. It may be used to implement a pre-allocated software
-queue.
-
-ringbuffd:
-This is a ring buffer that is associated with a socketpair. It may be
-used when a thread should wait until data has been inserted into the ring
-buffer. An example use is to store requests into a pre-allocated software
-queue, followed by a notification to a waiting thread that work is pending.
-
-
-// include/fi_signal.h
-
-This provides an inline implementation of a thread signaling object based
-on a socketpair. It only writes to the socketpair if a signal is not
-already pending.
-
-
-// include/fi.h
-// src/common.c
-
-This header file provides several helper functions and abstractions. The
-following are of note:
-
-mem_dup() - memdup equivalent
-htonll() / ntohll()
-sizeof_field()
-MIN / MAX - simple (i.e. potentially unsafe) min/max macros
-roundup_power_of_two()
-fastlock_t - lock object (spinlock or mutex based on availability)
-atomic operations - implementation based on availability
-fi_read_file() - read a value from a file
-fi_poll_fd() - call poll() on an fd
-fi_wait_cond() - wait on a mutex
-fi_datatype_size() - return size of an atomic datatype
-fi_[capability]_allowed() - routines to check caps bits
-fi_gettime_ms() - return current time in milliseconds
-fi_fd_nonblock() - set fd to nonblocking
-
diff --git a/include/fi.h b/include/fi.h
index 48c09c1..7357506 100644
--- a/include/fi.h
+++ b/include/fi.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013-2014 Intel Corporation. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -40,17 +41,18 @@
#include <stdlib.h>
#include <string.h>
#include <sys/param.h>
+#include <netinet/in.h>
#include <fi_abi.h>
#include <fi_file.h>
#include <fi_lock.h>
#include <fi_atom.h>
#include <fi_mem.h>
+#include <rdma/providers/fi_prov.h>
+#include <rdma/providers/fi_log.h>
#include <rdma/fabric.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_atomic.h>
-#include <rdma/fi_log.h>
#include <fi_osd.h>
@@ -118,6 +120,7 @@ struct fi_filter {
};
extern struct fi_filter prov_log_filter;
+extern struct fi_provider core_prov;
void fi_create_filter(struct fi_filter *filter, const char *env_name);
void fi_free_filter(struct fi_filter *filter);
@@ -131,7 +134,6 @@ void fi_param_init(void);
void fi_param_fini(void);
void fi_param_undefine(const struct fi_provider *provider);
-
static inline uint64_t roundup_power_of_two(uint64_t n)
{
if (!n || !(n & (n - 1)))
@@ -160,13 +162,27 @@ size_t fi_datatype_size(enum fi_datatype datatype);
uint64_t fi_tag_bits(uint64_t mem_tag_format);
uint64_t fi_tag_format(uint64_t tag_bits);
-int fi_send_allowed(uint64_t caps);
-int fi_recv_allowed(uint64_t caps);
-int fi_rma_initiate_allowed(uint64_t caps);
-int fi_rma_target_allowed(uint64_t caps);
+int ofi_send_allowed(uint64_t caps);
+int ofi_recv_allowed(uint64_t caps);
+int ofi_rma_initiate_allowed(uint64_t caps);
+int ofi_rma_target_allowed(uint64_t caps);
+int ofi_ep_bind_valid(struct fi_provider *prov, struct fid *bfid, uint64_t flags);
uint64_t fi_gettime_ms(void);
+uint64_t fi_gettime_us(void);
+
+static inline int ofi_equals_ipaddr(struct sockaddr_in *addr1,
+ struct sockaddr_in *addr2)
+{
+ return (addr1->sin_addr.s_addr == addr2->sin_addr.s_addr);
+}
+static inline int ofi_equals_sockaddr(struct sockaddr_in *addr1,
+ struct sockaddr_in *addr2)
+{
+ return (ofi_equals_ipaddr(addr1, addr2) &&
+ (addr1->sin_port == addr2->sin_port));
+}
#ifdef __cplusplus
}
diff --git a/include/fi_enosys.h b/include/fi_enosys.h
index 6cc2129..e8dc327 100644
--- a/include/fi_enosys.h
+++ b/include/fi_enosys.h
@@ -33,6 +33,8 @@
#ifndef _FI_ENOSYS_H_
#define _FI_ENOSYS_H_
+#include <fi_osd.h>
+
#include <rdma/fabric.h>
#include <rdma/fi_atomic.h>
#include <rdma/fi_cm.h>
@@ -70,7 +72,7 @@ static struct fi_ops_fabric X = {
.trywait = fi_no_trywait,
};
*/
-int fi_no_domain(struct fid_fabric *fabric, struct fi_domain_attr *attr,
+int fi_no_domain(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **dom, void *context);
int fi_no_passive_ep(struct fid_fabric *fabric, struct fi_info *info,
struct fid_pep **pep, void *context);
diff --git a/include/fi_list.h b/include/fi_list.h
index dbe7e90..08be1e8 100644
--- a/include/fi_list.h
+++ b/include/fi_list.h
@@ -187,6 +187,19 @@ static inline struct slist_entry *slist_remove_head(struct slist *list)
typedef int slist_func_t(struct slist_entry *item, const void *arg);
static inline struct slist_entry *
+slist_find_first_match(const struct slist *list, slist_func_t *match,
+ const void *arg)
+{
+ struct slist_entry *item;
+ for (item = list->head; item; item = item->next) {
+ if (match(item, arg))
+ return item;
+ }
+
+ return NULL;
+}
+
+static inline struct slist_entry *
slist_remove_first_match(struct slist *list, slist_func_t *match, const void *arg)
{
struct slist_entry *item, *prev;
@@ -270,7 +283,7 @@ static inline int dlistfd_wait_avail(struct dlistfd_head *head, int timeout)
if (!dlistfd_empty(head))
return 1;
-
+
ret = fd_signal_poll(&head->signal, timeout);
return ret ? ret : !dlistfd_empty(head);
}
diff --git a/include/fi_mem.h b/include/fi_mem.h
index 7ce310d..67930cc 100644
--- a/include/fi_mem.h
+++ b/include/fi_mem.h
@@ -39,6 +39,7 @@
#include <stdlib.h>
#include <string.h>
#include <fi_list.h>
+#include <fi_osd.h>
#ifdef INCLUDE_VALGRIND
@@ -66,26 +67,33 @@ static inline void *mem_dup(const void *src, size_t size)
/*
* Buffer pool (free stack) template
*/
-#define FREESTACK_EMPTY -1
+#define FREESTACK_EMPTY NULL
+
+#define FREESTACK_HEADER \
+ size_t size; \
+ void *next; \
#define freestack_isempty(fs) ((fs)->next == FREESTACK_EMPTY)
-#define freestack_push(fs, x) \
-{ \
- *(int *) &(fs)->buf[x] = (fs)->next; \
- (fs)->next = x; \
+#define freestack_push(fs, p) \
+do { \
+ *(void **) p = (fs)->next; \
+ (fs)->next = p; \
+} while (0)
+#define freestack_pop(fs) freestack_pop_impl(fs, (fs)->next)
+
+static inline void* freestack_pop_impl(void *fs, void *fs_next)
+{
+ struct {
+ FREESTACK_HEADER
+ } *freestack = fs;
+ assert(!freestack_isempty(freestack));
+ freestack->next = *((void **)fs_next);
+ return fs_next;
}
-#define freestack_pop(fs) \
- ({ \
- int i = (fs)->next; \
- assert(!freestack_isempty(fs)); \
- (fs)->next = *((int*) &(fs)->buf[i]); \
- i; \
- })
#define DECLARE_FREESTACK(entrytype, name) \
struct name { \
- size_t size; \
- size_t next; \
+ FREESTACK_HEADER \
entrytype buf[]; \
}; \
\
@@ -93,23 +101,29 @@ static inline void name ## _init(struct name *fs, size_t size) \
{ \
int i; \
assert(size == roundup_power_of_two(size)); \
- assert(sizeof(fs->buf[0]) >= sizeof(int)); \
+ assert(sizeof(fs->buf[0]) >= sizeof(void *)); \
fs->size = size; \
fs->next = FREESTACK_EMPTY; \
for (i = size - 1; i >= 0; i--) \
- freestack_push(fs, i); \
+ freestack_push(fs, &fs->buf[i]); \
} \
\
static inline struct name * name ## _create(size_t size) \
{ \
struct name *fs; \
fs = calloc(1, sizeof(*fs) + sizeof(entrytype) * \
- (roundup_power_of_two(size) - 1)); \
+ (roundup_power_of_two(size))); \
if (fs) \
name ##_init(fs, roundup_power_of_two(size)); \
return fs; \
} \
\
+static inline int name ## _index(struct name *fs, \
+ entrytype *entry) \
+{ \
+ return entry - fs->buf; \
+} \
+ \
static inline void name ## _free(struct name *fs) \
{ \
free(fs); \
diff --git a/include/fi_osd.h b/include/fi_osd.h
index 7488d7e..cc2db36 100644
--- a/include/fi_osd.h
+++ b/include/fi_osd.h
@@ -35,10 +35,15 @@
#ifdef __APPLE__
#include <osx/osd.h>
+#include <unix/osd.h>
#elif defined __FreeBSD__
#include <freebsd/osd.h>
+#include <unix/osd.h>
+#elif defined _WIN32
+#include <windows/osd.h>
#else
#include <linux/osd.h>
+#include <unix/osd.h>
#endif
#endif /* _FI_OSD_H_ */
diff --git a/include/fi_proto.h b/include/fi_proto.h
index 751c9ac..f569468 100644
--- a/include/fi_proto.h
+++ b/include/fi_proto.h
@@ -50,11 +50,14 @@ extern "C" {
/* ofi_ctrl_hdr::type */
enum {
- ofi_ctrl_connect,
+ ofi_ctrl_connreq,
+ ofi_ctrl_connresp,
ofi_ctrl_start_data,
ofi_ctrl_data,
ofi_ctrl_large_data,
- ofi_ctrl_ack
+ ofi_ctrl_ack,
+ ofi_ctrl_nack,
+ ofi_ctrl_discard,
};
/*
@@ -90,10 +93,8 @@ struct ofi_ctrl_hdr {
uint8_t type;
uint16_t seg_size;
uint32_t seg_no;
- union {
- uint64_t conn_id;
- uint64_t msg_id;
- };
+ uint64_t conn_id;
+ uint64_t msg_id;
union {
uint64_t conn_data;
uint64_t rx_key;
@@ -111,7 +112,8 @@ struct ofi_ctrl_hdr {
enum {
ofi_op_msg,
ofi_op_tagged,
- ofi_op_read,
+ ofi_op_read_req,
+ ofi_op_read_rsp,
ofi_op_write,
ofi_op_atomic,
};
@@ -132,8 +134,9 @@ enum {
* size: Size of data transfer
* data: Remote CQ data, if available
* tag: Message tag, used for tagged operations only
- * iov_len: Length of destination iov, used for RMA operations
+ * iov_count: Count of destination iov, used for RMA operations
* atomic: Control fields for atomic operations
+ * remote_idx: Tx request identifier of remote side
* resv: Reserved, used for msg operations
*/
struct ofi_op_hdr {
@@ -147,12 +150,13 @@ struct ofi_op_hdr {
uint64_t data;
union {
uint64_t tag;
- uint8_t iov_len;
+ uint8_t iov_count;
struct {
uint8_t datatype;
uint8_t op;
- uint8_t ioc_len;
+ uint8_t ioc_count;
} atomic;
+ uint64_t remote_idx;
uint64_t resv;
};
};
diff --git a/include/fi_rbuf.h b/include/fi_rbuf.h
index 9b206f8..a4b8868 100644
--- a/include/fi_rbuf.h
+++ b/include/fi_rbuf.h
@@ -43,6 +43,7 @@
#include <unistd.h>
#include <fcntl.h>
#include <fi.h>
+#include <fi_file.h>
#include <stdlib.h>
@@ -214,7 +215,7 @@ struct ringbuffd {
static inline int rbfdinit(struct ringbuffd *rbfd, size_t size)
{
- int ret, flags;
+ int ret;
rbfd->fdrcnt = 0;
rbfd->fdwcnt = 0;
@@ -226,16 +227,15 @@ static inline int rbfdinit(struct ringbuffd *rbfd, size_t size)
if (ret < 0)
goto err1;
- flags = fcntl(rbfd->fd[RB_READ_FD], F_GETFL, 0);
- ret = fcntl(rbfd->fd[RB_READ_FD], F_SETFL, flags | O_NONBLOCK);
- if (ret < 0)
+ ret = fi_fd_nonblock(rbfd->fd[RB_READ_FD]);
+ if (ret)
goto err2;
return 0;
err2:
- close(rbfd->fd[0]);
- close(rbfd->fd[1]);
+ ofi_close_socket(rbfd->fd[0]);
+ ofi_close_socket(rbfd->fd[1]);
err1:
rbfree(&rbfd->rb);
return -errno;
@@ -244,8 +244,8 @@ err1:
static inline void rbfdfree(struct ringbuffd *rbfd)
{
rbfree(&rbfd->rb);
- close(rbfd->fd[0]);
- close(rbfd->fd[1]);
+ ofi_close_socket(rbfd->fd[0]);
+ ofi_close_socket(rbfd->fd[1]);
}
static inline int rbfdfull(struct ringbuffd *rbfd)
@@ -272,7 +272,7 @@ static inline void rbfdsignal(struct ringbuffd *rbfd)
{
char c = 0;
if (rbfd->fdwcnt == rbfd->fdrcnt) {
- if (write(rbfd->fd[RB_WRITE_FD], &c, sizeof c) == sizeof c)
+ if (ofi_write_socket(rbfd->fd[RB_WRITE_FD], &c, sizeof c) == sizeof c)
rbfd->fdwcnt++;
}
}
@@ -282,7 +282,7 @@ static inline void rbfdreset(struct ringbuffd *rbfd)
char c;
if (rbfdempty(rbfd) && (rbfd->fdrcnt != rbfd->fdwcnt)) {
- if (read(rbfd->fd[RB_READ_FD], &c, sizeof c) == sizeof c)
+ if (ofi_read_socket(rbfd->fd[RB_READ_FD], &c, sizeof c) == sizeof c)
rbfd->fdrcnt++;
}
}
diff --git a/include/fi_signal.h b/include/fi_signal.h
index ec8cec6..af66677 100644
--- a/include/fi_signal.h
+++ b/include/fi_signal.h
@@ -43,6 +43,7 @@
#include <sys/socket.h>
#include <fi_file.h>
+#include <fi_osd.h>
#include <rdma/fi_errno.h>
@@ -65,29 +66,29 @@ static inline int fd_signal_init(struct fd_signal *signal)
if (ret < 0)
return -errno;
- ret = fcntl(signal->fd[FI_READ_FD], F_SETFL, O_NONBLOCK);
- if (ret < 0)
+ ret = fi_fd_nonblock(signal->fd[FI_READ_FD]);
+ if (ret)
goto err;
return 0;
err:
- close(signal->fd[0]);
- close(signal->fd[1]);
+ ofi_close_socket(signal->fd[0]);
+ ofi_close_socket(signal->fd[1]);
return -errno;
}
static inline void fd_signal_free(struct fd_signal *signal)
{
- close(signal->fd[0]);
- close(signal->fd[1]);
+ ofi_close_socket(signal->fd[0]);
+ ofi_close_socket(signal->fd[1]);
}
static inline void fd_signal_set(struct fd_signal *signal)
{
char c = 0;
if (signal->wcnt == signal->rcnt) {
- if (write(signal->fd[FI_WRITE_FD], &c, sizeof c) == sizeof c)
+ if (ofi_write_socket(signal->fd[FI_WRITE_FD], &c, sizeof c) == sizeof c)
signal->wcnt++;
}
}
@@ -96,7 +97,7 @@ static inline void fd_signal_reset(struct fd_signal *signal)
{
char c;
if (signal->rcnt != signal->wcnt) {
- if (read(signal->fd[FI_READ_FD], &c, sizeof c) == sizeof c)
+ if (ofi_read_socket(signal->fd[FI_READ_FD], &c, sizeof c) == sizeof c)
signal->rcnt++;
}
}
diff --git a/include/fi_util.h b/include/fi_util.h
index 274ddfc..c8f2e92 100644
--- a/include/fi_util.h
+++ b/include/fi_util.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -35,6 +36,7 @@
#endif /* HAVE_CONFIG_H */
#include <pthread.h>
+#include <stdio.h>
#include <rdma/fabric.h>
#include <rdma/fi_atomic.h>
@@ -43,7 +45,6 @@
#include <rdma/fi_endpoint.h>
#include <rdma/fi_eq.h>
#include <rdma/fi_errno.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_tagged.h>
#include <rdma/fi_trigger.h>
@@ -54,6 +55,7 @@
#include <fi_rbuf.h>
#include <fi_signal.h>
#include <fi_enosys.h>
+#include <fi_osd.h>
#ifndef _FI_UTIL_H_
#define _FI_UTIL_H_
@@ -61,18 +63,53 @@
#define UTIL_FLAG_ERROR (1ULL << 60)
+#define OFI_Q_STRERROR(prov, log, q, q_str, entry, strerror) \
+ FI_WARN(prov, log, "fi_" q_str "_readerr: err: %d, prov_err: %s (%d)\n",\
+ entry.err, \
+ strerror(q, entry.prov_errno, entry.err_data, NULL, 0), \
+ entry.prov_errno)
+
+#define OFI_Q_READERR(prov, log, q, q_str, readerr, strerror, ret, err_entry) \
+ do { \
+ ret = readerr(q, &err_entry, 0); \
+ if (ret != sizeof(err_entry)) { \
+ FI_WARN(prov, log, \
+ "Unable to fi_" q_str "_readerr\n"); \
+ } else { \
+ OFI_Q_STRERROR(prov, log, q, q_str, \
+ err_entry, strerror); \
+ } \
+ } while (0)
+
+#define OFI_CQ_READERR(prov, log, cq, ret, err_entry) \
+ OFI_Q_READERR(prov, log, cq, "cq", fi_cq_readerr, \
+ fi_cq_strerror, ret, err_entry)
+
+#define OFI_EQ_READERR(prov, log, eq, ret, err_entry) \
+ OFI_Q_READERR(prov, log, eq, "eq", fi_eq_readerr, \
+ fi_eq_strerror, ret, err_entry)
+
+#define ofi_sin_addr(addr) (((struct sockaddr_in *)(addr))->sin_addr)
+#define ofi_sin6_addr(addr) (((struct sockaddr_in6 *)(addr))->sin6_addr)
+
+enum fi_match_type {
+ FI_MATCH_EXACT,
+ FI_MATCH_PREFIX,
+};
+
+enum {
+ UTIL_TX_SHARED_CTX = 1 << 0,
+ UTIL_RX_SHARED_CTX = 1 << 1,
+};
/*
* Provider details
- * TODO: Determine if having this structure (with expanded fields)
- * would help support fi_getinfo. If so, fill out and update
- * implementation
+ */
struct util_prov {
- const struct fi_provider *prov;
- const struct fi_info *info; -- list of provider info's
- provider specific handlers, e.g. resolve addressing
+ const struct fi_provider *prov;
+ const struct fi_info *info;
+ const int flags;
};
- */
/*
@@ -89,12 +126,13 @@ struct util_fabric {
struct dlist_entry domain_list;
};
-int fi_fabric_init(const struct fi_provider *prov,
+int ofi_fabric_init(const struct fi_provider *prov,
struct fi_fabric_attr *prov_attr,
struct fi_fabric_attr *user_attr,
- struct util_fabric *fabric, void *context);
-int util_fabric_close(struct util_fabric *fabric);
-int util_trywait(struct fid_fabric *fabric, struct fid **fids, int count);
+ struct util_fabric *fabric, void *context,
+ enum fi_match_type type);
+int ofi_fabric_close(struct util_fabric *fabric);
+int ofi_trywait(struct fid_fabric *fabric, struct fid **fids, int count);
/*
* Domain
@@ -107,17 +145,21 @@ struct util_domain {
atomic_t ref;
const struct fi_provider *prov;
- const char *name;
+ char *name;
uint64_t caps;
uint64_t mode;
uint32_t addr_format;
enum fi_av_type av_type;
};
-int fi_domain_init(struct fid_fabric *fabric_fid, const struct fi_info *info,
+int ofi_domain_init(struct fid_fabric *fabric_fid, const struct fi_info *info,
struct util_domain *domain, void *context);
-int util_domain_close(struct util_domain *domain);
+int ofi_domain_close(struct util_domain *domain);
+
+/*
+ * Endpoint
+ */
struct util_ep;
typedef void (*fi_ep_progress_func)(struct util_ep *util_ep);
@@ -133,6 +175,11 @@ struct util_ep {
fi_ep_progress_func progress;
};
+int ofi_endpoint_init(struct fid_domain *domain, const struct util_prov *util_prov,
+ struct fi_info *info, struct util_ep *ep, void *context,
+ enum fi_match_type type);
+
+int ofi_endpoint_close(struct util_ep *util_ep);
/*
* Completion queue
@@ -152,8 +199,9 @@ struct util_cq_err_entry {
struct slist_entry list_entry;
};
-DECLARE_CIRQUE(struct fi_cq_data_entry, util_comp_cirq);
+DECLARE_CIRQUE(struct fi_cq_tagged_entry, util_comp_cirq);
+typedef void (*fi_cq_progress_func)(struct util_cq *cq);
struct util_cq {
struct fid_cq cq_fid;
struct util_domain *domain;
@@ -169,12 +217,14 @@ struct util_cq {
struct slist err_list;
fi_cq_read_func read_entry;
int internal_wait;
+ fi_cq_progress_func progress;
};
-int util_cq_open(const struct fi_provider *prov,
- struct fid_domain *domain, struct fi_cq_attr *attr,
- struct fid_cq **cq_fid, void *context);
-
+int ofi_cq_init(const struct fi_provider *prov, struct fid_domain *domain,
+ struct fi_cq_attr *attr, struct util_cq *cq,
+ fi_cq_progress_func progress, void *context);
+void ofi_cq_progress(struct util_cq *cq);
+int ofi_cq_cleanup(struct util_cq *cq);
/*
* Counter
@@ -183,6 +233,8 @@ struct util_cntr {
struct fid_cntr cntr_fid;
struct util_domain *domain;
atomic_t ref;
+ uint64_t checkpoint_cnt;
+ uint64_t checkpoint_err;
};
@@ -224,23 +276,90 @@ struct util_av_attr {
uint64_t flags;
};
-int fi_av_create(struct util_domain *domain,
- const struct fi_av_attr *attr, const struct util_av_attr *util_attr,
- struct fid_av **av, void *context);
+int ofi_av_init(struct util_domain *domain,
+ const struct fi_av_attr *attr, const struct util_av_attr *util_attr,
+ struct util_av *av, void *context);
+int ofi_av_close(struct util_av *av);
+
+int ofi_av_insert_addr(struct util_av *av, const void *addr, int slot, int *index);
+int ofi_av_lookup_index(struct util_av *av, const void *addr, int slot);
+int ofi_av_bind(struct fid *av_fid, struct fid *eq_fid, uint64_t flags);
+void ofi_av_write_event(struct util_av *av, uint64_t data,
+ int err, void *context);
+
int ip_av_create(struct fid_domain *domain_fid, struct fi_av_attr *attr,
struct fid_av **av, void *context);
-void *fi_av_get_addr(struct util_av *av, int index);
-#define ip_av_get_addr fi_av_get_addr
+void *ofi_av_get_addr(struct util_av *av, int index);
+#define ip_av_get_addr ofi_av_get_addr
int ip_av_get_index(struct util_av *av, const void *addr);
-int fi_get_addr(uint32_t addr_format, uint64_t flags,
- const char *node, const char *service,
- void **addr, size_t *addrlen);
-int fi_get_src_addr(uint32_t addr_format,
- const void *dest_addr, size_t dest_addrlen,
- void **src_addr, size_t *src_addrlen);
+int ofi_get_addr(uint32_t addr_format, uint64_t flags,
+ const char *node, const char *service,
+ void **addr, size_t *addrlen);
+int ofi_get_src_addr(uint32_t addr_format,
+ const void *dest_addr, size_t dest_addrlen,
+ void **src_addr, size_t *src_addrlen);
+void ofi_getnodename(char *buf, int buflen);
+int ofi_av_get_index(struct util_av *av, const void *addr);
+
+/*
+ * Connection Map
+ */
+enum util_cmap_state {
+ CMAP_UNSPEC,
+ CMAP_CONNECTING,
+ CMAP_CONNECTED
+};
+
+struct util_cmap_handle {
+ struct util_cmap *cmap;
+ enum util_cmap_state state;
+ struct util_cmap_key *key;
+ size_t key_index;
+ fi_addr_t fi_addr;
+ struct util_cmap_peer *peer;
+};
+
+struct util_cmap_key {
+ struct util_cmap_handle *handle;
+};
+DECLARE_FREESTACK(struct util_cmap_key, util_cmap_keypool);
+
+struct util_cmap_peer {
+ struct util_cmap_handle *handle;
+ struct dlist_entry entry;
+ size_t addrlen;
+ uint8_t addr[];
+};
+
+typedef void (*ofi_cmap_free_handle_func)(void *arg);
+
+struct util_cmap {
+ struct util_av *av;
+ struct util_cmap_handle **handles;
+ struct util_cmap_keypool *keypool;
+ struct dlist_entry peer_list;
+ ofi_cmap_free_handle_func free_handle;
+ fastlock_t lock;
+};
+
+void ofi_cmap_update_state(struct util_cmap_handle *handle,
+ enum util_cmap_state state);
+/*
+ * Caller must hold cmap->lock. Either fi_addr or
+ * addr and addrlen args should be present.
+ */
+int ofi_cmap_add_handle(struct util_cmap *cmap, struct util_cmap_handle *handle,
+ enum util_cmap_state state, fi_addr_t fi_addr, void *addr,
+ size_t addrlen);
+/* Caller must hold cmap->lock */
+struct util_cmap_handle *ofi_cmap_get_handle(struct util_cmap *cmap, fi_addr_t fi_addr);
+void ofi_cmap_del_handle(struct util_cmap_handle *handle);
+void ofi_cmap_free(struct util_cmap *cmap);
+struct util_cmap *ofi_cmap_alloc(struct util_av *av,
+ ofi_cmap_free_handle_func free_handle);
/*
* Poll set
@@ -289,7 +408,7 @@ struct util_wait_fd {
fi_epoll_t epoll_fd;
};
-int fi_wait_fd_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
+int ofi_wait_fd_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
struct fid_wait **waitset);
@@ -316,9 +435,42 @@ struct util_event {
uint8_t data[0];
};
-int fi_eq_create(struct fid_fabric *fabric, struct fi_eq_attr *attr,
+int ofi_eq_create(struct fid_fabric *fabric, struct fi_eq_attr *attr,
struct fid_eq **eq_fid, void *context);
+/*
+ * MR
+ */
+
+
+/*hide addr related info & store prov_mr ptr */
+struct ofi_util_mr {
+ void *map_handle;
+ uint64_t b_key; /* track available key (BASIC usage) */
+ enum fi_mr_mode mr_type;
+ const struct fi_provider *prov;
+};
+
+/*create instance of data structure and return handle to user */
+int ofi_mr_init(const struct fi_provider *in_prov, enum fi_mr_mode mode,
+ struct ofi_util_mr ** out_new_mr);
+/*insert user mr struct in data structure*/
+int ofi_mr_insert(struct ofi_util_mr * in_mr_h,
+ const struct fi_mr_attr *in_attr,
+ uint64_t * out_key, void * in_prov_mr);
+/*return on user mr struct */
+void * ofi_mr_retrieve(struct ofi_util_mr * in_mr_h, uint64_t in_key);
+/*need address offsetted, verified, and user mr struct returned*/
+/* io_addr is address of buff (&buf) */
+int ofi_mr_retrieve_and_verify(struct ofi_util_mr * in_mr_h, ssize_t in_len,
+ uintptr_t *io_addr, uint64_t in_key,
+ uint64_t in_access, void **out_prov_mr);
+/*erase a specific item in the data structure */
+int ofi_mr_erase(struct ofi_util_mr * in_mr_h, uint64_t in_key);
+/*close data structure instance */
+void ofi_mr_close(struct ofi_util_mr *in_mr_h);
+
+
/*
* Attributes and capabilities
@@ -333,14 +485,15 @@ int fi_eq_create(struct fid_fabric *fabric, struct fi_eq_attr *attr,
int fi_check_fabric_attr(const struct fi_provider *prov,
const struct fi_fabric_attr *prov_attr,
- const struct fi_fabric_attr *user_attr);
+ const struct fi_fabric_attr *user_attr,
+ enum fi_match_type type);
int fi_check_wait_attr(const struct fi_provider *prov,
const struct fi_wait_attr *attr);
int fi_check_domain_attr(const struct fi_provider *prov,
const struct fi_domain_attr *prov_attr,
- const struct fi_domain_attr *user_attr);
-int fi_check_ep_attr(const struct fi_provider *prov,
- const struct fi_ep_attr *prov_attr,
+ const struct fi_domain_attr *user_attr,
+ enum fi_match_type type);
+int fi_check_ep_attr(const struct util_prov *util_prov,
const struct fi_ep_attr *user_attr);
int fi_check_cq_attr(const struct fi_provider *prov,
const struct fi_cq_attr *attr);
@@ -350,23 +503,20 @@ int fi_check_rx_attr(const struct fi_provider *prov,
int fi_check_tx_attr(const struct fi_provider *prov,
const struct fi_tx_attr *prov_attr,
const struct fi_tx_attr *user_attr);
-int fi_check_info(const struct fi_provider *prov,
- const struct fi_info *prov_info,
- const struct fi_info *user_info);
-void fi_alter_info(struct fi_info *info,
+int fi_check_info(const struct util_prov *util_prov,
+ const struct fi_info *user_info,
+ enum fi_match_type type);
+void ofi_alter_info(struct fi_info *info,
const struct fi_info *hints);
-int util_getinfo(const struct fi_provider *prov, uint32_t version,
+int util_getinfo(const struct util_prov *util_prov, uint32_t version,
const char *node, const char *service, uint64_t flags,
- const struct fi_info *prov_info, struct fi_info *hints,
- struct fi_info **info);
+ struct fi_info *hints, struct fi_info **info);
struct fid_list_entry {
struct dlist_entry entry;
struct fid *fid;
-
- uint64_t last_cntr_val;
};
int fid_list_insert(struct dlist_entry *fid_list, fastlock_t *lock,
@@ -378,5 +528,23 @@ void fi_fabric_insert(struct util_fabric *fabric);
struct util_fabric *fi_fabric_find(const char *name);
void fi_fabric_remove(struct util_fabric *fabric);
+/*
+ * Layered Providers
+ */
+
+typedef int (*ofi_alter_info_t)(struct fi_info *src_info, struct fi_info *dest_info);
+
+int ofix_getinfo(uint32_t version, const char *node, const char *service,
+ uint64_t flags, const struct util_prov *util_prov,
+ struct fi_info *hints,
+ ofi_alter_info_t alter_layer_info,
+ ofi_alter_info_t alter_base_info,
+ int get_base_info, struct fi_info **info);
+char *ofi_strdup_less_prefix(char *name, char *prefix);
+char *ofi_strdup_add_prefix(char *name, char *prefix);
+
+int ofi_shm_map(struct util_shm *shm, const char *name, size_t size,
+ int readonly, void **mapped);
+int ofi_shm_unmap(struct util_shm *shm);
#endif
diff --git a/include/freebsd/osd.h b/include/freebsd/osd.h
index ff11729..b096e6e 100644
--- a/include/freebsd/osd.h
+++ b/include/freebsd/osd.h
@@ -30,9 +30,13 @@
* SOFTWARE.
*/
+#ifndef _FREEBSD_OSD_H_
+#define _FREEBSD_OSD_H_
+
#include <sys/endian.h>
#include <pthread_np.h>
+#include "unix/osd.h"
#define bswap_64 bswap64
@@ -40,3 +44,12 @@
#define HOST_NAME_MAX 128
typedef cpuset_t cpu_set_t;
+
+static inline int ofi_shm_remap(struct util_shm *shm, size_t newsize, void **mapped)
+{
+ return -1;
+}
+
+#endif /* _FREEBSD_OSD_H_ */
+
+
diff --git a/include/linux/osd.h b/include/linux/osd.h
index c3b3e60..517eba2 100644
--- a/include/linux/osd.h
+++ b/include/linux/osd.h
@@ -30,5 +30,26 @@
* SOFTWARE.
*/
+#ifndef _LINUX_OSD_H_
+#define _LINUX_OSD_H_
+
+/*#define _GNU_SOURCE*/
+
#include <byteswap.h>
#include <endian.h>
+#include <sys/mman.h>
+
+#include "unix/osd.h"
+#include "rdma/fi_errno.h"
+
+static inline int ofi_shm_remap(struct util_shm *shm,
+ size_t newsize, void **mapped)
+{
+ shm->ptr = mremap(shm->ptr, shm->size, newsize, 0);
+ shm->size = newsize;
+ *mapped = shm->ptr;
+ return shm->ptr == MAP_FAILED ? -FI_EINVAL : FI_SUCCESS;
+}
+
+#endif /* _LINUX_OSD_H_ */
+
diff --git a/include/osx/osd.h b/include/osx/osd.h
index ed784bc..3e5b913 100644
--- a/include/osx/osd.h
+++ b/include/osx/osd.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2015 Los Alamos Nat. Security, LLC. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -43,6 +44,8 @@
#include <pthread.h>
+#include "unix/osd.h"
+
#define CLOCK_REALTIME CALENDAR_CLOCK
#define CLOCK_MONOTONIC SYSTEM_CLOCK
@@ -64,6 +67,11 @@ extern "C" {
int clock_gettime(clockid_t clk_id, struct timespec *tp);
+static inline int ofi_shm_remap(struct util_shm *shm, size_t newsize, void **mapped)
+{
+ return -1;
+}
+
#ifdef __cplusplus
}
#endif
diff --git a/include/prov.h b/include/prov.h
index 3c5e322..399e600 100644
--- a/include/prov.h
+++ b/include/prov.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013-2014 Intel Corporation. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -35,7 +36,7 @@
#include "config.h"
-#include <rdma/fi_prov.h>
+#include <rdma/providers/fi_prov.h>
/* Provider initialization function signature that built-in providers
* must specify. */
@@ -135,4 +136,26 @@ UDP_INI ;
# define UDP_INIT NULL
#endif
+#if (HAVE_RXM) && (HAVE_RXM_DL)
+# define RXM_INI FI_EXT_INI
+# define RXM_INIT NULL
+#elif (HAVE_RXM)
+# define RXM_INI INI_SIG(fi_rxm_ini)
+# define RXM_INIT fi_rxm_ini()
+RXM_INI ;
+#else
+# define RXM_INIT NULL
+#endif
+
+#if (HAVE_RXD) && (HAVE_RXD_DL)
+# define RXD_INI FI_EXT_INI
+# define RXD_INIT NULL
+#elif (HAVE_RXD)
+# define RXD_INI INI_SIG(fi_rxd_ini)
+# define RXD_INIT fi_rxd_ini()
+RXD_INI ;
+#else
+# define RXD_INIT NULL
+#endif
+
#endif /* _PROV_H_ */
diff --git a/include/rdma/fabric.h b/include/rdma/fabric.h
index 8487ebb..533234f 100644
--- a/include/rdma/fabric.h
+++ b/include/rdma/fabric.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013-2016 Intel Corporation. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -30,25 +31,33 @@
* SOFTWARE.
*/
-#ifndef _FABRIC_H_
-#define _FABRIC_H_
+#ifndef FABRIC_H
+#define FABRIC_H
#include <stdint.h>
#include <stddef.h>
-#include <sys/socket.h>
+#include <sys/types.h>
+
+#if defined(_WIN32)
+#include <BaseTsd.h>
+#include <windows.h>
+typedef SSIZE_T ssize_t;
+#endif
#ifdef __cplusplus
extern "C" {
#endif
-
#ifndef container_of
#define container_of(ptr, type, field) \
((type *) ((char *)ptr - offsetof(type, field)))
#endif
+/* API version (which is not necessarily the same as the
+ * tarball/libfabric package version number).
+ */
#define FI_MAJOR_VERSION 1
-#define FI_MINOR_VERSION 3
+#define FI_MINOR_VERSION 4
enum {
FI_PATH_MAX = 256,
@@ -123,6 +132,7 @@ typedef struct fid *fid_t;
#define FI_INJECT_COMPLETE (1ULL << 26)
#define FI_TRANSMIT_COMPLETE (1ULL << 27)
#define FI_DELIVERY_COMPLETE (1ULL << 28)
+#define FI_AFFINITY (1ULL << 29)
/* fi_getinfo()-specific flags/caps */
#define FI_PROV_ATTR_ONLY (1ULL << 54)
@@ -151,8 +161,8 @@ enum {
FI_ADDR_GNI
};
-#define FI_ADDR_UNSPEC UINT64_MAX
-#define FI_ADDR_NOTAVAIL UINT64_MAX
+#define FI_ADDR_UNSPEC ((uint64_t) -1)
+#define FI_ADDR_NOTAVAIL ((uint64_t) -1)
#define FI_SHARED_CONTEXT (-(size_t)1)
typedef uint64_t fi_addr_t;
@@ -226,7 +236,9 @@ enum {
FI_PROTO_MXM,
FI_PROTO_IWARP_RDM,
FI_PROTO_IB_RDM,
- FI_PROTO_GNI
+ FI_PROTO_GNI,
+ FI_PROTO_RXM,
+ FI_PROTO_RXD
};
/* Mode bits */
@@ -495,4 +507,4 @@ struct fi_context {
}
#endif
-#endif /* _FABRIC_H_ */
+#endif /* FABRIC_H */
diff --git a/include/rdma/fi_atomic.h b/include/rdma/fi_atomic.h
index 6f07975..5929c91 100644
--- a/include/rdma/fi_atomic.h
+++ b/include/rdma/fi_atomic.h
@@ -30,8 +30,8 @@
* SOFTWARE.
*/
-#ifndef _FI_ATOMIC_H_
-#define _FI_ATOMIC_H_
+#ifndef FI_ATOMIC_H
+#define FI_ATOMIC_H
#include <rdma/fabric.h>
#include <rdma/fi_endpoint.h>
@@ -314,4 +314,4 @@ fi_compare_atomicvalid(struct fid_ep *ep,
}
#endif
-#endif /* _FI_ATOMIC_H_ */
+#endif /* FI_ATOMIC_H */
diff --git a/include/rdma/fi_cm.h b/include/rdma/fi_cm.h
index 539d615..d3839c9 100644
--- a/include/rdma/fi_cm.h
+++ b/include/rdma/fi_cm.h
@@ -30,8 +30,8 @@
* SOFTWARE.
*/
-#ifndef _FI_CM_H_
-#define _FI_CM_H_
+#ifndef FI_CM_H
+#define FI_CM_H
#include <rdma/fi_endpoint.h>
@@ -115,4 +115,4 @@ static inline int fi_shutdown(struct fid_ep *ep, uint64_t flags)
}
#endif
-#endif /* _FI_CM_H_ */
+#endif /* FI_CM_H */
diff --git a/include/rdma/fi_domain.h b/include/rdma/fi_domain.h
index bb3b961..fd288d7 100644
--- a/include/rdma/fi_domain.h
+++ b/include/rdma/fi_domain.h
@@ -30,8 +30,8 @@
* SOFTWARE.
*/
-#ifndef _FI_DOMAIN_H_
-#define _FI_DOMAIN_H_
+#ifndef FI_DOMAIN_H
+#define FI_DOMAIN_H
#include <rdma/fabric.h>
#include <rdma/fi_eq.h>
@@ -210,6 +210,23 @@ fi_mr_reg(struct fid_domain *domain, const void *buf, size_t len,
requested_key, flags, mr, context);
}
+static inline int
+fi_mr_regv(struct fid_domain *domain, const struct iovec *iov,
+ size_t count, uint64_t access,
+ uint64_t offset, uint64_t requested_key,
+ uint64_t flags, struct fid_mr **mr, void *context)
+{
+ return domain->mr->regv(&domain->fid, iov, count, access,
+ offset, requested_key, flags, mr, context);
+}
+
+static inline int
+fi_mr_regattr(struct fid_domain *domain, const struct fi_mr_attr *attr,
+ uint64_t flags, struct fid_mr **mr)
+{
+ return domain->mr->regattr(&domain->fid, attr, flags, mr);
+}
+
static inline void *fi_mr_desc(struct fid_mr *mr)
{
return mr->mem_desc;
@@ -291,4 +308,4 @@ fi_rx_addr(fi_addr_t fi_addr, int rx_index, int rx_ctx_bits)
}
#endif
-#endif /* _FI_DOMAIN_H_ */
+#endif /* FI_DOMAIN_H */
diff --git a/include/rdma/fi_endpoint.h b/include/rdma/fi_endpoint.h
index fcaa11e..0c6790e 100644
--- a/include/rdma/fi_endpoint.h
+++ b/include/rdma/fi_endpoint.h
@@ -30,13 +30,11 @@
* SOFTWARE.
*/
-#ifndef _FI_ENDPOINT_H_
-#define _FI_ENDPOINT_H_
+#ifndef FI_ENDPOINT_H
+#define FI_ENDPOINT_H
-#include <sys/socket.h>
#include <rdma/fabric.h>
#include <rdma/fi_domain.h>
-#include <stddef.h>
#ifdef __cplusplus
@@ -208,6 +206,17 @@ fi_getopt(fid_t fid, int level, int optname,
return ep->ops->getopt(fid, level, optname, optval, optlen);
}
+static inline int fi_ep_alias(struct fid_ep *ep, struct fid_ep **alias_ep,
+ uint64_t flags)
+{
+ int ret;
+ struct fid *fid;
+ ret = fi_alias(&ep->fid, &fid, flags);
+ if (!ret)
+ *alias_ep = container_of(fid, struct fid_ep, fid);
+ return ret;
+}
+
static inline int
fi_tx_context(struct fid_ep *ep, int index, struct fi_tx_attr *attr,
struct fid_ep **tx_ep, void *context)
@@ -314,4 +323,4 @@ fi_injectdata(struct fid_ep *ep, const void *buf, size_t len,
}
#endif
-#endif /* _FI_ENDPOINT_H_ */
+#endif /* FI_ENDPOINT_H */
diff --git a/include/rdma/fi_eq.h b/include/rdma/fi_eq.h
index a87ba8a..2cb7d53 100644
--- a/include/rdma/fi_eq.h
+++ b/include/rdma/fi_eq.h
@@ -30,10 +30,12 @@
* SOFTWARE.
*/
-#ifndef _FI_EQ_H_
-#define _FI_EQ_H_
+#ifndef FI_EQ_H
+#define FI_EQ_H
+#ifndef _WIN32
#include <pthread.h>
+#endif /* _WIN32 */
#include <rdma/fabric.h>
@@ -55,7 +57,7 @@ enum fi_wait_obj {
FI_WAIT_UNSPEC,
FI_WAIT_SET,
FI_WAIT_FD,
- FI_WAIT_MUTEX_COND, /* pthread mutex & cond */
+ FI_WAIT_MUTEX_COND /* pthread mutex & cond */
};
struct fi_wait_attr {
@@ -73,10 +75,13 @@ struct fid_wait {
struct fi_ops_wait *ops;
};
+#ifndef _WIN32
struct fi_mutex_cond {
pthread_mutex_t *mutex;
pthread_cond_t *cond;
};
+#endif /* _WIN32 */
+
/*
* Poll Set
@@ -445,4 +450,4 @@ fi_cntr_wait(struct fid_cntr *cntr, uint64_t threshold, int timeout)
}
#endif
-#endif /* _FI_EQ_H_ */
+#endif /* FI_EQ_H */
diff --git a/include/rdma/fi_errno.h b/include/rdma/fi_errno.h
index db78e72..8324671 100644
--- a/include/rdma/fi_errno.h
+++ b/include/rdma/fi_errno.h
@@ -31,8 +31,8 @@
* SOFTWARE.
*/
-#ifndef _FI_ERRNO_H_
-#define _FI_ERRNO_H_
+#ifndef FI_ERRNO_H
+#define FI_ERRNO_H
#include <errno.h>
@@ -191,6 +191,7 @@ enum {
FI_ECRC = 264, /* CRC error */
FI_ETRUNC = 265, /* Truncation error */
FI_ENOKEY = 266, /* Required key not available */
+ FI_ENOAV = 267, /* Missing or unavailable address vector */
FI_ERRNO_MAX
};
@@ -200,4 +201,4 @@ const char *fi_strerror(int errnum);
}
#endif
-#endif /* _FI_ERRNO_H_ */
+#endif /* FI_ERRNO_H */
diff --git a/include/rdma/fi_rma.h b/include/rdma/fi_rma.h
index c59d04a..003c10f 100644
--- a/include/rdma/fi_rma.h
+++ b/include/rdma/fi_rma.h
@@ -30,8 +30,8 @@
* SOFTWARE.
*/
-#ifndef _FI_RMA_H_
-#define _FI_RMA_H_
+#ifndef FI_RMA_H
+#define FI_RMA_H
#include <rdma/fabric.h>
#include <rdma/fi_endpoint.h>
@@ -165,4 +165,4 @@ fi_inject_writedata(struct fid_ep *ep, const void *buf, size_t len,
}
#endif
-#endif /* _FI_RMA_H_ */
+#endif /* FI_RMA_H */
diff --git a/include/rdma/fi_tagged.h b/include/rdma/fi_tagged.h
index a67f519..75e9785 100644
--- a/include/rdma/fi_tagged.h
+++ b/include/rdma/fi_tagged.h
@@ -30,8 +30,8 @@
* SOFTWARE.
*/
-#ifndef _FI_TAGGED_H_
-#define _FI_TAGGED_H_
+#ifndef FI_TAGGED_H
+#define FI_TAGGED_H
#include <rdma/fabric.h>
#include <rdma/fi_endpoint.h>
@@ -157,4 +157,4 @@ fi_tinjectdata(struct fid_ep *ep, const void *buf, size_t len,
}
#endif
-#endif /* _FI_TAGGED_H_ */
+#endif /* FI_TAGGED_H */
diff --git a/include/rdma/fi_trigger.h b/include/rdma/fi_trigger.h
index 9b3e5db..df170ec 100644
--- a/include/rdma/fi_trigger.h
+++ b/include/rdma/fi_trigger.h
@@ -30,11 +30,9 @@
* SOFTWARE.
*/
-#ifndef _FI_TRIGGER_H_
-#define _FI_TRIGGER_H_
+#ifndef FI_TRIGGER_H
+#define FI_TRIGGER_H
-#include <stdint.h>
-#include <stddef.h>
#include <rdma/fabric.h>
#ifdef __cplusplus
@@ -72,4 +70,4 @@ struct fi_triggered_context {
}
#endif
-#endif /* _FI_TRIGGER_H_ */
+#endif /* FI_TRIGGER_H */
diff --git a/include/rdma/fi_log.h b/include/rdma/providers/fi_log.h
similarity index 89%
rename from include/rdma/fi_log.h
rename to include/rdma/providers/fi_log.h
index 90497cf..ed21009 100644
--- a/include/rdma/fi_log.h
+++ b/include/rdma/providers/fi_log.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2015-2016, Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2015, Intel Corp., Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -32,13 +32,13 @@
*
*/
-#ifndef _FI_LOG_H_
-#define _FI_LOG_H_
+#ifndef FI_LOG_H
+#define FI_LOG_H
#include "config.h"
#include <rdma/fabric.h>
-#include <rdma/fi_prov.h>
+#include <rdma/providers/fi_prov.h>
#ifdef __cplusplus
extern "C" {
@@ -90,13 +90,17 @@ void fi_log(const struct fi_provider *prov, enum fi_log_level level,
#if ENABLE_DEBUG
#define FI_DBG(prov, subsystem, ...) \
FI_LOG(prov, FI_LOG_DEBUG, subsystem, __VA_ARGS__)
+#define FI_DBG_TRACE(prov, subsystem, ...) \
+ FI_LOG(prov, FI_LOG_TRACE, subsystem, __VA_ARGS__)
#else
#define FI_DBG(prov_name, subsystem, ...) \
do {} while (0)
+#define FI_DBG_TRACE(prov, subsystem, ...) \
+ do {} while (0)
#endif
#ifdef __cplusplus
}
#endif
-#endif /*_FI_LOG_H_ */
+#endif /* FI_LOG_H */
diff --git a/include/rdma/fi_prov.h b/include/rdma/providers/fi_prov.h
similarity index 97%
rename from include/rdma/fi_prov.h
rename to include/rdma/providers/fi_prov.h
index 62c39b9..0a39ae4 100644
--- a/include/rdma/fi_prov.h
+++ b/include/rdma/providers/fi_prov.h
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2005, 2006 Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2005, 2006, 2016 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2005 PathScale, Inc. All rights reserved.
* Copyright (c) 2013-2014 Intel Corporation. All rights reserved.
*
@@ -33,8 +33,8 @@
* SOFTWARE.
*/
-#ifndef _FI_PROV_H_
-#define _FI_PROV_H_
+#ifndef FI_PROV_H
+#define FI_PROV_H
#include <rdma/fabric.h>
@@ -133,4 +133,4 @@ fi_param_get_bool(struct fi_provider *provider, const char *param_name, int *val
}
#endif
-#endif /* _FI_PROV_H_ */
+#endif /* FI_PROV_H */
diff --git a/prov/gni/test/pmi_utils.c b/include/unix/osd.h
similarity index 58%
rename from prov/gni/test/pmi_utils.c
rename to include/unix/osd.h
index dda646e..c83eefd 100644
--- a/prov/gni/test/pmi_utils.c
+++ b/include/unix/osd.h
@@ -1,6 +1,5 @@
/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2013-2016 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -31,27 +30,60 @@
* SOFTWARE.
*/
+#ifndef _FI_UNIX_OSD_H_
+#define _FI_UNIX_OSD_H_
+
#include <stdlib.h>
-#include <stdio.h>
-#include "pmi.h"
-#include "criterion/criterion.h"
+#include <unistd.h>
+#include <errno.h>
+
+#define FI_DESTRUCTOR(func) static __attribute__((destructor)) void func
+
+struct util_shm
+{
+ int shared_fd;
+ void *ptr;
+ const char *name;
+ size_t size;
+};
+
+static inline int ofi_memalign(void **memptr, size_t alignment, size_t size)
+{
+ return posix_memalign(memptr, alignment, size);
+}
+
+static inline void ofi_freealign(void *memptr)
+{
+ free(memptr);
+}
+
+static inline ssize_t ofi_read_socket(int fd, void *buf, size_t count)
+{
+ return read(fd, buf, count);
+}
-static int pmi_initialized;
+static inline ssize_t ofi_write_socket(int fd, const void *buf, size_t count)
+{
+ return write(fd, buf, count);
+}
-ReportHook(PRE_ALL)(struct criterion_test_set *test)
+static inline int ofi_close_socket(int socket)
{
- int rc, spawned;
+ return close(socket);
+}
- rc = PMI_Init(&spawned);
- if (rc != PMI_SUCCESS) {
- fprintf(stderr, "PMI_Init failed - returned %d.\n", rc);
- fprintf(stderr, "Debugging support may not be available\n");
- } else
- pmi_initialized = 1;
+static inline int ofi_sockerr(void)
+{
+ return errno;
}
-ReportHook(POST_ALL)(struct criterion_global_stats *stats)
+static inline void ofi_osd_init(void)
{
- if (pmi_initialized == 1)
- PMI_Finalize();
}
+
+static inline void ofi_osd_fini(void)
+{
+}
+
+#endif /* _FI_UNIX_OSD_H_ */
+
diff --git a/libfabric.spec.in b/libfabric.spec
similarity index 89%
copy from libfabric.spec.in
copy to libfabric.spec
index b82574e..5caa027 100644
--- a/libfabric.spec.in
+++ b/libfabric.spec
@@ -1,14 +1,14 @@
%{!?configopts: %global configopts %{nil}}
Name: libfabric
-Version: @VERSION@
+Version: 1.4.0rc2
Release: 1%{?dist}
Summary: User-space RDMA Fabric Interfaces
Group: System Environment/Libraries
License: GPLv2 or BSD
Url: http://www.github.com/ofiwg/libfabric
-Source: http://www.openfabrics.org/downloads/ofi/%{name}-%{version}.tar.bz2
+Source: http://www.github.org/ofiwg/%{name}/releases/download/v{%version}/%{name}-%{version}.tar.bz2
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
%description
@@ -47,6 +47,8 @@ rm -rf %{buildroot}
%defattr(-,root,root,-)
%{_libdir}/lib*.so.*
%{_bindir}/fi_info
+%{_bindir}/fi_strerror
+%{_bindir}/fi_pingpong
%dir %{_libdir}/libfabric/
%doc AUTHORS COPYING README
@@ -56,6 +58,7 @@ rm -rf %{buildroot}
%{_libdir}/*.a
%{_libdir}/pkgconfig/libfabric.pc
%{_includedir}/*
+%{_mandir}/man1/*
%{_mandir}/man3/*
%{_mandir}/man7/*
diff --git a/libfabric.spec.in b/libfabric.spec.in
index b82574e..7e546df 100644
--- a/libfabric.spec.in
+++ b/libfabric.spec.in
@@ -8,7 +8,7 @@ Summary: User-space RDMA Fabric Interfaces
Group: System Environment/Libraries
License: GPLv2 or BSD
Url: http://www.github.com/ofiwg/libfabric
-Source: http://www.openfabrics.org/downloads/ofi/%{name}-%{version}.tar.bz2
+Source: http://www.github.org/ofiwg/%{name}/releases/download/v{%version}/%{name}-%{version}.tar.bz2
BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
%description
@@ -47,6 +47,8 @@ rm -rf %{buildroot}
%defattr(-,root,root,-)
%{_libdir}/lib*.so.*
%{_bindir}/fi_info
+%{_bindir}/fi_strerror
+%{_bindir}/fi_pingpong
%dir %{_libdir}/libfabric/
%doc AUTHORS COPYING README
@@ -56,6 +58,7 @@ rm -rf %{buildroot}
%{_libdir}/*.a
%{_libdir}/pkgconfig/libfabric.pc
%{_includedir}/*
+%{_mandir}/man1/*
%{_mandir}/man3/*
%{_mandir}/man7/*
diff --git a/man/README b/man/README
deleted file mode 100644
index 71e52bf..0000000
--- a/man/README
+++ /dev/null
@@ -1,168 +0,0 @@
-This file describes how the developer side of man pages work in
-libfabric.
-
-The Definitive Source Of Truth man pages are the Markdown man pages in
-this directory (i.e., the files ending in .<digit>.md. If you want to
-edit man pages, you need to edit the .<digit>.md pages. Do NOT edit
-the .<digit> nroff man pages directly; these files are automatically
-generated -- you will lose any manual edits the next time those files
-are generated.
-
-The Markdown web pages are rendered in two different ways:
-
-1. Nroff man pages. These man pages are put into the `master` branch
- and later included in libfabric distribution tarballs.
-
-2. HTML. The http://ofiwg.github.io/libfabric/ web site (which is
- served by the Github web servers) automatically renders the content
- of the `gh-pages` branch of the libfabric repo.
-
-Markdown syntax
-===============
-
-The definitive man pages are the Markdown man pages. To edit them,
-you need to understand the syntax used in these files.
-
-The canonical reference for Markdown is here:
-
- http://daringfireball.net/projects/markdown/syntax
-
-Note, however, that the libfabric Markdown man pages are served via
-the Github Pages web servers, which use a system called Jekyll to
-render the Markdown into HTML (https://github.com/jekyll/jekyll).
-As such, there are a few Jekyll annotations in the libfabric Markdown
-pages (so that they can be served up properly from Github's web
-servers).
-
-If you're familiar with Markdown, you should be ok. But there are a
-small number differences and quirks with which you should be familiar:
-
-1. The first few lines of each file are a YAML header and include
- directive for Jekyll. DO NOT REMOVE THIS HEADER (or the file will
- not render to HTML properly when served up from Github's web
- servers). Here's a sample YAML header from fabric.7.md:
-
----
-layout: page
-title: fabric(7)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
- The whole block is needed, and it must be the first input in the
- file.
-
-2. The libfabric man pages are full of 2-level lists of things. E.g.,
- lists of functions, and then in some of the functions, there is a
- sub-list of flags that can be used with that function.
-
- The convention used in the libfabric man pages is to highlight a
- word/phrase representing each list item. Then use a ":" to start
- the next line that describes that item. For example:
-
-*FI_FLOAT_COMPLEX*
-: An ordered pair of single-precision floating point values (IEEE
- 754), with the first value representing the real portion of a
- complex number and the second representing the imaginary portion.
-
- This will make the token "FI_FLOAT_COMPLEX" be highlighted in both
- HTML and nroff output, and then the paragraph that comes after it
- will be properly delimited and indented.
-
- To make a sub-list inside an item, use the same format, but prefix
- the sub-list items with "-", like this:
-
-*flags*
-: Flags that control the configuration of the CQ.
-
-- *FI_WRITE*
-: Indicates that the application requires support for inserting user
- events into the CQ. If this flag is set, then the fi_cq_write and
- fi_cq_writeerr operations must be supported by the provider. If the
- FI_WRITE flag is not set, then the application may not invoke
- fi_cq_write of fi_cq_writeerr.
-
-3. There's a small number of places in the libfabric man pages where
- there are unnumbered lists with deliberate line breaks. For
- example:
-
-fi_atomic / fi_atomicv
-fi_atomicto / fi_atomicmsg
-: Initiates an atomic operation to remote memory
-
- Note the first line is "fi_atomic / fi_atomicv", and then there is
- a deliberate line break, and then the second line is "fi_atomicto /
- fi_atomicmsg".
-
- To effect the deliberate line break, you have to put two blank
- spaces after "fi_atomicv". To show that graphically (showing "_"
- for " "):
-
-fi_atomic / fi_atomicv__
-fi_atomicto / fi_atomicmsg
-: Initiates an atomic operation to remote memory
-
-4. The "SEE ALSO" items at the end of each man page are linked to
- their corresponding man pages. Note that the links are made to
- ".html" files -- *not* ".md" files. If you care, the reason is
- because the Github web servers statically generate .html files from
- the .md files when you git push to the gh-pages branch. Hence, the
- man pages are actually served from static .html files on the Github
- web servers.
-
- Also, since links are meaningless in nroff, they are effectively
- ignored in the resulting nroff output.
-
-Workflow
-========
-
-The workflow is like this:
-
-1. Developer edits .<digit>.md files for new changes.
-
-2. In a perfect world, the developer makes perfect edits and pushes
- the changes up to `master`. An automated cron job will eventually
- notice the new pages, and do two things:
-
- 2a. Copy the modified Markdown pages to the `gh-master` branch (so
- that they go live on the web site).
-
- 2b. Re-generate any relevant nroff man pages in `master`.
-
- The automated cron job actually does exist and does these things,
- but it should only be relied upon once a developer is sure that
- their changes to the Markdown man pages are correct.
-
-3. To check that the changes will render properly, developers should
- do two things:
-
- 3a. Run "make nroff". This will convert all the Markdown man pages
- into nroff man pages (in the man/ directory). Check to ensure
- that your changes look appropriate in the rendered nroff
- output.
-
- *CAUTION* The "pandoc" utility is used to generate the nroff
- files from the Markdown source. Different versions of pandoc
- will generate slightly different nroff output. Meaning: when
- you run "make nroff", you might end up changing every nroff man
- page, simply because your version of pandoc is different than
- the last person who ran it. Please only check in your changes,
- if possible.
-
- 3b. Check out the `gh-pages` branch from libfabric and copy any
- modified Markdown pages into the "master/man" directory (i.e.,
- the directory for man pages from the master development
- branch).
-
- Then run the "jekyll serve" command from the top-level
- directory in `gh-pages`. This runs a local web server on your
- computer and renders the Markdown files into HTML such that you
- can point a browser to http://127.0.0.1:4000 and see the web
- site.
-
- If you make any changes to files in the tree where "jekyll" is
- running, Jekyll will notice the changes and automatically
- re-generate the relevant HTML. Meaning: you can just refresh
- the page from http://127.0.0.1:4000 in your browser and you'll
- see your changes -- there's no need to restart Jekyll to force
- it to notice new changes.
diff --git a/man/fabric.7.md b/man/fabric.7.md
deleted file mode 100644
index d34ad34..0000000
--- a/man/fabric.7.md
+++ /dev/null
@@ -1,251 +0,0 @@
----
-layout: page
-title: fabric(7)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-Fabric Interface Library
-
-# SYNOPSIS
-
-```c
-#include <rdma/fabric.h>
-```
-
-Libfabric is a high-performance fabric software library designed to
-provide low-latency interfaces to fabric hardware.
-
-# OVERVIEW
-
-Libfabric provides 'process direct I/O' to application software communicating
-across fabric software and hardware. Process direct I/O, historically
-referred to as RDMA, allows an application to directly access network
-resources without operating system interventions. Data transfers can
-occur directly to and from application memory.
-
-There are two components to the libfabric software:
-
-*Fabric Providers*
-: Conceptually, a fabric provider may be viewed as a local hardware
- NIC driver, though a provider is not limited by this definition.
- The first component of libfabric is a general purpose framework that
- is capable of handling different types of fabric hardware. All
- fabric hardware devices and their software drivers are required to
- support this framework. Devices and the drivers that plug into the
- libfabric framework are referred to as fabric providers, or simply
- providers. Provider details may be found in
- [`fi_provider`(7)](fi_provider.7.html).
-
-*Fabric Interfaces*
-: The second component is a set of communication operations.
- Libfabric defines several sets of communication functions that
- providers can support. It is not required that providers implement
- all the interfaces that are defined; however, providers clearly
- indicate which interfaces they do support.
-
-# FABRIC INTERFACES
-
-The fabric interfaces are designed such that they are cohesive and not
-simply a union of disjoint interfaces. The interfaces are logically
-divided into two groups: control interfaces and communication
-operations. The control interfaces are a common set of operations that
-provide access to local communication resources, such as address
-vectors and event queues. The communication operations expose
-particular models of communication and fabric functionality, such as
-message queues, remote memory access, and atomic operations.
-Communication operations are associated with fabric endpoints.
-
-Applications will typically use the control interfaces to discover
-local capabilities and allocate necessary resources. They will then
-allocate and configure a communication endpoint to send and receive
-data, or perform other types of data transfers, with remote endpoints.
-
-# CONTROL INTERFACES
-
-The control interfaces APIs provide applications access to network
-resources. This involves listing all the interfaces available,
-obtaining the capabilities of the interfaces and opening a provider.
-
-*fi_getinfo - Fabric Information*
-: The fi_getinfo call is the base call used to discover and request
- fabric services offered by the system. Applications can use this
- call to indicate the type of communication that they desire. The
- results from fi_getinfo, fi_info, are used to reserve and configure
- fabric resources.
-
- fi_getinfo returns a list of fi_info structures. Each structure
- references a single fabric provider, indicating the interfaces that
- the provider supports, along with a named set of resources. A
- fabric provider may include multiple fi_info structures in the
- returned list.
-
-*fi_fabric - Fabric Domain*
-: A fabric domain represents a collection of hardware and software
- resources that access a single physical or virtual network. All
- network ports on a system that can communicate with each other
- through the fabric belong to the same fabric domain. A fabric
- domain shares network addresses and can span multiple providers.
- libfabric supports systems connected to multiple fabrics.
-
-*fi_domain - Access Domains*
-: An access domain represents a single logical connection into a
- fabric. It may map to a single physical or virtual NIC or a port.
- An access domain defines the boundary across which fabric resources
- may be associated. Each access domain belongs to a single fabric
- domain.
-
-*fi_endpoint - Fabric Endpoint*
-: A fabric endpoint is a communication portal. An endpoint may be
- either active or passive. Passive endpoints are used to listen for
- connection requests. Active endpoints can perform data transfers.
- Endpoints are configured with specific communication capabilities
- and data transfer interfaces.
-
-*fi_eq - Event Queue*
-: Event queues, are used to collect and report the completion of
- asynchronous operations and events. Event queues report events
- that are not directly associated with data transfer operations.
-
-*fi_cq - Completion Queue*
-: Completion queues are high-performance event queues used to report
- the completion of data transfer operations.
-
-*fi_cntr - Event Counters*
-: Event counters are used to report the number of completed
- asynchronous operations. Event counters are considered
- light-weight, in that a completion simply increments a counter,
- rather than placing an entry into an event queue.
-
-*fi_mr - Memory Region*
-: Memory regions describe application local memory buffers. In order
- for fabric resources to access application memory, the application
- must first grant permission to the fabric provider by constructing a
- memory region. Memory regions are required for specific types of
- data transfer operations, such as RMA transfers (see below).
-
-*fi_av - Address Vector*
-: Address vectors are used to map higher level addresses, such as IP
- addresses, which may be more natural for an application to use, into
- fabric specific addresses. The use of address vectors allows
- providers to reduce the amount of memory required to maintain large
- address look-up tables, and eliminate expensive address resolution
- and look-up methods during data transfer operations.
-
-# DATA TRANSFER INTERFACES
-
-Fabric endpoints are associated with multiple data transfer
-interfaces. Each interface set is designed to support a specific
-style of communication, with an endpoint allowing the different
-interfaces to be used in conjunction. The following data transfer
-interfaces are defined by libfabric.
-
-*fi_msg - Message Queue*
-: Message queues expose a simple, message-based FIFO queue interface
- to the application. Message data transfers allow applications to
- send and receive data with message boundaries being maintained.
-
-*fi_tagged - Tagged Message Queues*
-: Tagged message lists expose send/receive data transfer operations
- built on the concept of tagged messaging. The tagged message queue
- is conceptually similar to standard message queues, but with the
- addition of 64-bit tags for each message. Sent messages are matched
- with receive buffers that are tagged with a similar value.
-
-*fi_rma - Remote Memory Access*
-: RMA transfers are one-sided operations that read or write data
- directly to a remote memory region. Other than defining the
- appropriate memory region, RMA operations do not require interaction
- at the target side for the data transfer to complete.
-
-*fi_atomic - Atomic*
-: Atomic operations can perform one of several operations on a remote
- memory region. Atomic operations include well-known functionality,
- such as atomic-add and compare-and-swap, plus several other
- pre-defined calls. Unlike other data transfer interfaces, atomic
- operations are aware of the data formatting at the target memory
- region.
-
-# LOGGING INTERFACE
-
-Logging can be controlled using the FI_LOG_LEVEL, FI_LOG_PROV, and
-FI_LOG_SUBSYS environment variables.
-
-*FI_LOG_LEVEL*
-: FI_LOG_LEVEL controls the amount of logging data that is output. The
- following log levels are defined.
-
-- *Warn*
-: Warn is the least verbose setting and is intended for reporting errors
- or warnings.
-
-- *Trace*
-: Trace is more verbose and is meant to include non-detailed output helpful to
- tracing program execution.
-
-- *Info*
-: Info is high traffic and meant for detailed output.
-
-- *Debug*
-: Debug is high traffic and is likely to impact application performance.
- Debug output is only available if the library has been compiled with
- debugging enabled.
-
-*FI_LOG_PROV*
-: The FI_LOG_PROV environment variable enables or disables logging from
- specific providers. Providers can be enabled by listing them in a comma
- separated fashion. If the list begins with the '^' symbol, then the list will
- be negated. By default all providers are enabled.
-
- Example: To enable logging from the psm and sockets provider:
- FI_LOG_PROV="psm,sockets"
-
- Example: To enable logging from providers other than psm:
- FI_LOG_PROV="^psm"
-
-*FI_LOG_SUBSYS*
-: The FI_LOG_SUBSYS environment variable enables or disables logging at the
- subsystem level. The syntax for enabling or disabling subsystems is similar to
- that used for FI_LOG_PROV. The following subsystems are defined.
-
-- *core*
-: Provides output related to the core framework and its management of providers.
-
-- *fabric*
-: Provides output specific to interactions associated with the fabric object.
-
-- *domain*
-: Provides output specific to interactions associated with the domain object.
-
-- *ep_ctrl*
-: Provides output specific to endpoint non-data transfer operations,
- such as CM operations.
-
-- *ep_data*
-: Provides output specific to endpoint data transfer operations.
-
-- *av*
-: Provides output specific to address vector operations.
-
-- *cq*
-: Provides output specific to completion queue operations.
-
-- *eq*
-: Provides output specific to event queue operations.
-
-- *mr*
-: Provides output specific to memory registration.
-
-# SEE ALSO
-
-[`fi_provider`(7)](fi_provider.7.html),
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_av`(3)](fi_av.3.html),
-[`fi_eq`(3)](fi_eq.3.html),
-[`fi_cq`(3)](fi_cq.3.html),
-[`fi_cntr`(3)](fi_cntr.3.html),
-[`fi_mr`(3)](fi_mr.3.html)
diff --git a/man/fi_atomic.3.md b/man/fi_atomic.3.md
deleted file mode 100644
index 1bc343a..0000000
--- a/man/fi_atomic.3.md
+++ /dev/null
@@ -1,539 +0,0 @@
----
-layout: page
-title: fi_atomic(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_atomic - Remote atomic functions
-
-fi_atomic / fi_atomicv / fi_atomicmsg / fi_inject_atomic
-: Initiates an atomic operation to remote memory
-
-fi_fetch_atomic / fi_fetch_atomicv / fi_fetch_atomicmsg
-: Initiates an atomic operation to remote memory, retrieving the initial
- value.
-
-fi_compare_atomic / fi_compare_atomicv / fi_compare_atomicmsg
-: Initiates an atomic compare-operation to remote memory, retrieving
- the initial value.
-
-fi_atomic_valid / fi_fetch_atomic_valid / fi_compare_atomic_valid
-: Indicates if a provider supports a specific atomic operation
-
-# SYNOPSIS
-
-```c
-#include <rdma/fi_atomic.h>
-
-ssize_t fi_atomic(struct fid_ep *ep, const void *buf,
- size_t count, void *desc, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key,
- enum fi_datatype datatype, enum fi_op op, void *context);
-
-ssize_t fi_atomicv(struct fid_ep *ep, const struct fi_ioc *iov,
- void **desc, size_t count, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key,
- enum fi_datatype datatype, enum fi_op op, void *context);
-
-ssize_t fi_atomicmsg(struct fid_ep *ep, const struct fi_msg_atomic *msg,
- uint64_t flags);
-
-ssize_t fi_inject_atomic(struct fid_ep *ep, const void *buf,
- size_t count, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key,
- enum fi_datatype datatype, enum fi_op op);
-
-ssize_t fi_fetch_atomic(struct fid_ep *ep, const void *buf,
- size_t count, void *desc, void *result, void *result_desc,
- fi_addr_t dest_addr, uint64_t addr, uint64_t key,
- enum fi_datatype datatype, enum fi_op op, void *context);
-
-ssize_t fi_fetch_atomicv(struct fid_ep *ep, const struct fi_ioc *iov,
- void **desc, size_t count, struct fi_ioc *resultv,
- void **result_desc, size_t result_count, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key, enum fi_datatype datatype,
- enum fi_op op, void *context);
-
-ssize_t fi_fetch_atomicmsg(struct fid_ep *ep,
- const struct fi_msg_atomic *msg, struct fi_ioc *resultv,
- void **result_desc, size_t result_count, uint64_t flags);
-
-ssize_t fi_compare_atomic(struct fid_ep *ep, const void *buf,
- size_t count, void *desc, const void *compare,
- void *compare_desc, void *result, void *result_desc,
- fi_addr_t dest_addr, uint64_t addr, uint64_t key,
- enum fi_datatype datatype, enum fi_op op, void *context);
-
-size_t fi_compare_atomicv(struct fid_ep *ep, const struct fi_ioc *iov,
- void **desc, size_t count, const struct fi_ioc *comparev,
- void **compare_desc, size_t compare_count, struct fi_ioc *resultv,
- void **result_desc, size_t result_count, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key, enum fi_datatype datatype,
- enum fi_op op, void *context);
-
-ssize_t fi_compare_atomicmsg(struct fid_ep *ep,
- const struct fi_msg_atomic *msg, const struct fi_ioc *comparev,
- void **compare_desc, size_t compare_count,
- struct fi_ioc *resultv, void **result_desc, size_t result_count,
- uint64_t flags);
-
-int fi_atomicvalid(struct fid_ep *ep, enum fi_datatype datatype,
- enum fi_op op, size_t *count);
-
-int fi_fetch_atomicvalid(struct fid_ep *ep, enum fi_datatype datatype,
- enum fi_op op, size_t *count);
-
-int fi_compare_atomicvalid(struct fid_ep *ep, enum fi_datatype datatype,
- enum fi_op op, size_t *count);
-```
-
-# ARGUMENTS
-
-*ep*
-: Fabric endpoint on which to initiate atomic operation.
-
-*buf*
-: Local data buffer that specifies first operand of atomic operation
-
-*iov / comparev / resultv*
-: Vectored data buffer(s).
-
-*count / compare_count / result_count*
-: Count of vectored data entries. The number of elements referenced,
- where each element is the indicated datatype.
-
-*addr*
-: Address of remote memory to access.
-
-*key*
-: Protection key associated with the remote memory.
-
-*datatype*
-: Datatype associated with atomic operands
-
-*op*
-: Atomic operation to perform
-
-*compare*
-: Local compare buffer, containing comparison data.
-
-*result*
-: Local data buffer to store initial value of remote buffer
-
-*desc / compare_desc / result_desc*
-: Data descriptor associated with the local data buffer, local compare
- buffer, and local result buffer, respectively.
-
-*dest_addr*
-: Destination address for connectionless atomic operations. Ignored for
- connected endpoints.
-
-*msg*
-: Message descriptor for atomic operations
-
-*flags*
-: Additional flags to apply for the atomic operation
-
-*context*
-: User specified pointer to associate with the operation.
-
-# DESCRIPTION
-
-Atomic transfers are used to read and update data located in remote
-memory regions in an atomic fashion. Conceptually, they are similar
-to local atomic operations of a similar nature (e.g. atomic increment,
-compare and swap, etc.). Updates to remote data involve one of
-several operations on the data, and act on specific types of data, as
-listed below. As such, atomic transfers have knowledge of the format
-of the data being accessed. A single atomic function may operate
-across an array of data applying an atomic operation to each entry,
-but the atomicity of an operation is limited to a single datatype or
-entry.
-
-## Atomic Data Types
-
-Atomic functions may operate on one of the following identified data
-types. A given atomic function may support any datatype, subject to
-provider implementation constraints.
-
-*FI_INT8*
-: Signed 8-bit integer.
-
-*FI_UINT8*
-: Unsigned 8-bit integer.
-
-*FI_INT16*
-: Signed 16-bit integer.
-
-*FI_UINT16*
-: Unsigned 16-bit integer.
-
-*FI_INT32*
-: Signed 32-bit integer.
-
-*FI_UINT32*
-: Unsigned 32-bit integer.
-
-*FI_INT64*
-: Signed 64-bit integer.
-
-*FI_UINT64*
-: Unsigned 64-bit integer.
-
-*FI_FLOAT*
-: A single-precision floating point value (IEEE 754).
-
-*FI_DOUBLE*
-: A double-precision floating point value (IEEE 754).
-
-*FI_FLOAT_COMPLEX*
-: An ordered pair of single-precision floating point values (IEEE
- 754), with the first value representing the real portion of a
- complex number and the second representing the imaginary portion.
-
-*FI_DOUBLE_COMPLEX*
-: An ordered pair of double-precision floating point values (IEEE
- 754), with the first value representing the real portion of a
- complex number and the second representing the imaginary portion.
-
-*FI_LONG_DOUBLE*
-: A double-extended precision floating point value (IEEE 754).
-
-*FI_LONG_DOUBLE_COMPLEX*
-: An ordered pair of double-extended precision floating point values
- (IEEE 754), with the first value representing the real portion of
- a complex number and the second representing the imaginary
- portion.
-
-## Atomic Operations
-
-The following atomic operations are defined. An atomic operation
-often acts against a target value in the remote memory buffer and
-source value provided with the atomic function. It may also carry
-source data to replace the target value in compare and swap
-operations. A conceptual description of each operation is provided.
-
-*FI_MIN*
-: Minimum
-```c
-if (buf[i] < addr[i])
- addr[i] = buf[i]
-```
-
-*FI_MAX*
-: Maximum
-```c
-if (buf[i] > addr[i])
- addr[i] = buf[i]
-```
-
-*FI_SUM*
-: Sum
-```c
-addr[i] = addr[i] + buf[i]
-```
-
-*FI_PROD*
-: Product
-```c
-addr[i] = addr[i] * buf[i]
-```
-
-*FI_LOR*
-: Logical OR
-```c
-addr[i] = (addr[i] || buf[i])
-```
-
-*FI_LAND*
-: Logical AND
-```c
-addr[i] = (addr[i] && buf[i])
-```
-
-*FI_BOR*
-: Bitwise OR
-```c
-addr[i] = addr[i] | buf[i]
-```
-
-*FI_BAND*
-: Bitwise AND
-```c
-addr[i] = addr[i] & buf[i]
-```
-
-*FI_LXOR*
-: Logical exclusive-OR (XOR)
-```c
-addr[i] = ((addr[i] && !buf[i]) || (!addr[i] && buf[i]))
-```
-
-*FI_BXOR*
-: Bitwise exclusive-OR (XOR)
-```c
-addr[i] = addr[i] ^ buf[i]
-```
-
-*FI_ATOMIC_READ*
-: Read data atomically
-```c
-buf[i] = addr[i]
-```
-
-*FI_ATOMIC_WRITE*
-: Write data atomically
-```c
-addr[i] = buf[i]
-```
-
-*FI_CSWAP*
-: Compare values and if equal swap with data
-```c
-if (compare[i] == addr[i])
- addr[i] = buf[i]
-```
-
-*FI_CSWAP_NE*
-: Compare values and if not equal swap with data
-```c
-if (compare[i] != addr[i])
- addr[i] = buf[i]
-```
-
-*FI_CSWAP_LE*
-: Compare values and if less than or equal swap with data
-```c
-if (compare[i] <= addr[i])
- addr[i] = buf[i]
-```
-
-*FI_CSWAP_LT*
-: Compare values and if less than swap with data
-```c
-if (compare[i] < addr[i])
- addr[i] = buf[i]
-```
-
-*FI_CSWAP_GE*
-: Compare values and if greater than or equal swap with data
-```c
-if (compare[i] >= addr[i])
- addr[i] = buf[i]
-```
-
-*FI_CSWAP_GT*
-: Compare values and if greater than swap with data
-```c
-if (compare[i] > addr[i])
- addr[i] = buf[i]
-```
-
-*FI_MSWAP*
-: Swap masked bits with data
-```c
-addr[i] = (buf[i] & compare[i]) | (addr[i] & ~compare[i])
-```
-
-## Base Atomic Functions
-
-The base atomic functions -- fi_atomic, fi_atomicv,
-fi_atomicmsg -- are used to transmit data to a remote node, where the
-specified atomic operation is performed against the target data. The
-result of a base atomic function is stored at the remote memory
-region. The main difference between atomic functions are the number
-and type of parameters that they accept as input. Otherwise, they
-perform the same general function.
-
-The call fi_atomic transfers the data contained in the user-specified
-data buffer to a remote node. For unconnected endpoints, the destination
-endpoint is specified through the dest_addr parameter. Unless
-the endpoint has been configured differently, the data buffer passed
-into fi_atomic must not be touched by the application until the
-fi_atomic call completes asynchronously. The target buffer of a base
-atomic operation must allow for remote read an/or write access, as
-appropriate.
-
-The fi_atomicv call adds support for a scatter-gather list to
-fi_atomic. The fi_atomicv transfers the set of data buffers
-referenced by the ioc parameter to the remote node for processing.
-
-The fi_inject_atomic call is an optimized version of fi_atomic. The
-fi_inject_atomic function behaves as if the FI_INJECT transfer flag
-were set, and FI_COMPLETION were not. That is, the data buffer is
-available for reuse immediately on returning from from
-fi_inject_atomic, and no completion event will be generated for this
-atomic. The completion event will be suppressed even if the endpoint
-has not been configured with FI_SELECTIVE_COMPLETION. See the flags
-discussion below for more details.
-
-The fi_atomicmsg call supports atomic functions over both connected
-and unconnected endpoints, with the ability to control the atomic
-operation per call through the use of flags. The fi_atomicmsg
-function takes a struct fi_msg_atomic as input.
-
-```c
-struct fi_msg_atomic {
- const struct fi_ioc *msg_iov; /* local scatter-gather array */
- void **desc; /* local access descriptors */
- size_t iov_count;/* # elements in ioc */
- const void *addr; /* optional endpoint address */
- const struct fi_rma_ioc *rma_iov; /* remote SGL */
- size_t rma_iov_count;/* # elements in remote SGL */
- enum fi_datatype datatype; /* operand datatype */
- enum fi_op op; /* atomic operation */
- void *context; /* user-defined context */
- uint64_t data; /* optional data */
-};
-
-struct fi_rma_ioc {
- uint64_t addr; /* target address */
- size_t count; /* # target operands */
- uint64_t key; /* access key */
-};
-```
-
-The following list of atomic operations are usable with base
-atomic operations: FI_MIN, FI_MAX, FI_SUM, FI_PROD,
-FI_LOR, FI_LAND, FI_BOR, FI_BAND, FI_LXOR, FI_BXOR,
-and FI_ATOMIC_WRITE.
-
-## Fetch-Atomic Functions
-
-The fetch atomic functions -- fi_fetch_atomic, fi_fetch_atomicv,
-and fi_fetch atomicmsg -- behave similar to the
-equivalent base atomic function. The difference between the fetch and
-base atomic calls are the fetch atomic routines return the initial
-value that was stored at the target to the user. The initial value is
-read into the user provided result buffer. The target buffer of
-fetch-atomic operations must be enabled for remote read access.
-
-The following list of atomic operations are usable with
-fetch atomic operations: FI_MIN, FI_MAX, FI_SUM, FI_PROD,
-FI_LOR, FI_LAND, FI_BOR, FI_BAND, FI_LXOR, FI_BXOR, FI_ATOMIC_READ,
-and FI_ATOMIC_WRITE.
-
-## Compare-Atomic Functions
-
-The compare atomic functions -- fi_compare_atomic, fi_compare_atomicv,
-and fi_compare atomicmsg -- are used for
-operations that require comparing the target data against a value
-before performing a swap operation. The compare atomic functions
-support: FI_CSWAP, FI_CSWAP_NE, FI_CSWAP_LE, FI_CSWAP_LT, FI_CSWAP_GE,
-FI_CSWAP_GT, and FI_MSWAP.
-
-## Atomic Valid Functions
-
-The atomic valid functions -- fi_atomicvalid, fi_fetch_atomicvalid,
-and fi_compare_atomicvalid --indicate which operations the local
-provider supports. Needed operations not supported by the provider
-must be emulated by the application. Each valid call corresponds to a
-set of atomic functions. fi_atomicvalid checks whether a provider
-supports a specific base atomic operation for a given datatype and
-operation. fi_fetch_atomicvalid indicates if a provider supports a
-specific fetch-atomic operation for a given datatype and operation.
-And fi_compare_atomicvalid checks if a provider supports a specified
-compare-atomic operation for a given datatype and operation.
-
-If an operation is supported, an atomic valid call will return 0,
-along with a count of atomic data units that a single function call
-will operate on.
-
-## Completions
-
-Completed atomic operations are reported to the user through one or
-more event collectors associated with the endpoint. Users provide
-context which are associated with each operation, and is returned to
-the user as part of the event completion. See fi_cq for completion
-event details.
-
-Updates to the target buffer of an atomic operation are visible to
-processes running on the target system either after a completion has
-been generated, or after the completion of an operation initiated
-after the atomic call with a fencing operation occurring in between.
-For example, the target process may be notified by the initiator
-sending a message after the atomic call completes, or sending a fenced
-message immediately after initiating the atomic operation.
-
-# FLAGS
-
-The fi_atomicmsg, fi_fetch_atomicmsg, and fi_compare_atomicmsg calls
-allow the user to specify flags which can change the default data
-transfer operation. Flags specified with atomic message operations
-override most flags previously configured with the endpoint, except
-where noted (see fi_control). The following list of flags are usable
-with atomic message calls.
-
-*FI_COMPLETION*
-: Indicates that a completion entry should be generated for the
- specified operation. The endpoint must be bound to a completion queue
- with FI_SELECTIVE_COMPLETION that corresponds to the specified operation,
- or this flag is ignored.
-
-*FI_MORE*
-: Indicates that the user has additional requests that will
- immediately be posted after the current call returns. Use of this
- flag may improve performance by enabling the provider to optimize
- its access to the fabric hardware.
-
-*FI_INJECT*
-: Indicates that the outbound non-const data buffers (buf and compare
- parameters) should be returned to user immediately after the call
- returns, even if the operation is handled asynchronously. This may
- require that the underlying provider implementation copy the data
- into a local buffer and transfer out of that buffer. The use of
- output result buffers are not affected by this flag.
-
-*FI_FENCE*
-: Indicates that the requested operation, also
- known as the fenced operation, be deferred until all previous operations
- targeting the same target endpoint have completed.
-
-# RETURN VALUE
-
-Returns 0 on success. On error, a negative value corresponding to fabric
-errno is returned. Fabric errno values are defined in
-`rdma/fi_errno.h`.
-
-# ERRORS
-
-*-FI_EOPNOTSUPP*
-: The requested atomic operation is not supported on this endpoint.
-
-*-FI_EMSGSIZE*
-: The number of atomic operations in a single request exceeds that
- supported by the underlying provider.
-
-# NOTES
-
-Atomic operations operate on an array of values of a specific data type.
-Atomicity is only guaranteed for each data type operation, not across
-the entire array. The following pseudo-code demonstrates this operation
-for 64-bit unsigned atomic write. ATOMIC_WRITE_U64 is a platform
-dependent macro that atomically writes 8 bytes to an aligned memory location.
-
-```c
-fi_atomic(ep, buf, count, NULL, dest_addr, addr, key,
- FI_UINT64, FI_ATOMIC_WRITE, context);
-{
- for (i = 1; i < count; i ++)
- ATOMIC_WRITE_U64(((uint64_t *) addr)[i],
- ((uint64_t *) buf)[i]);
-}
-```
-
-The number of array elements to operate on is specified through a count
-parameter. This must be between 1 and the maximum returned through the
-relevant valid operation, inclusive. The requested operation and data
-type must also be valid for the given provider.
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_cq`(3)](fi_cq.3.html),
-[`fi_rma`(3)](fi_rma.3.html)
diff --git a/man/fi_av.3.md b/man/fi_av.3.md
deleted file mode 100644
index 4ad180c..0000000
--- a/man/fi_av.3.md
+++ /dev/null
@@ -1,435 +0,0 @@
----
-layout: page
-title: fi_av(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_av \- Address vector operations
-
-fi_av_open / fi_close
-: Open or close an address vector
-
-fi_av_bind
-: Associate an address vector with an event queue.
-
-fi_av_insert / fi_av_insertsvc / fi_av_remove
-: Insert/remove an address into/from the address vector.
-
-fi_av_lookup
-: Retrieve an address stored in the address vector.
-
-fi_av_straddr
-: Convert an address into a printable string.
-
-# SYNOPSIS
-
-```c
-#include <rdma/fi_domain.h>
-
-int fi_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
- struct fid_av **av, void *context);
-
-int fi_close(struct fid *av);
-
-int fi_av_bind(struct fid_av *av, struct fid *eq, uint64_t flags);
-
-int fi_av_insert(struct fid_av *av, void *addr, size_t count,
- fi_addr_t *fi_addr, uint64_t flags, void *context);
-
-int fi_av_insertsvc(struct fid_av *av, const char *node,
- const char *service, fi_addr_t *fi_addr, uint64_t flags,
- void *context);
-
-int fi_av_insertsym(struct fid_av *av, const char *node,
- size_t nodecnt, const char *service, size_t svccnt,
- fi_addr_t *fi_addr, uint64_t flags, void *context);
-
-int fi_av_remove(struct fid_av *av, fi_addr_t *fi_addr, size_t count,
- uint64_t flags);
-
-int fi_av_lookup(struct fid_av *av, fi_addr_t fi_addr,
- void *addr, size_t *addrlen);
-
-fi_addr_t fi_rx_addr(fi_addr_t fi_addr, int rx_index,
- int rx_ctx_bits);
-
-const char * fi_av_straddr(struct fid_av *av, const void *addr,
- void *buf, size_t *len);
-```
-
-# ARGUMENTS
-
-*domain*
-: Resource domain
-
-*av*
-: Address vector
-
-*eq*
-: Event queue
-
-*attr*
-: Address vector attributes
-
-*context*
-: User specified context associated with the address vector or insert
- operation.
-
-*addr*
-: Buffer containing one or more addresses to insert into address vector.
-
-*addrlen*
-: On input, specifies size of addr buffer. On output, stores number
- of bytes written to addr buffer.
-
-*fi_addr*
-: For insert, a reference to an array where returned fabric addresses
- will be written. For remove, one or more fabric addresses to remove.
-
-*count*
-: Number of addresses to insert/remove from an AV.
-
-*flags*
-: Additional flags to apply to the operation.
-
-# DESCRIPTION
-
-Address vectors are used to map higher level addresses, which may be
-more natural for an application to use, into fabric specific
-addresses. The mapping of addresses is fabric and provider specific,
-but may involve lengthy address resolution and fabric management
-protocols. AV operations are synchronous by default, but may be set
-to operate asynchronously by specifying the FI_EVENT flag to
-`fi_av_open`. When requesting asynchronous operation, the application
-must first bind an event queue to the AV before inserting addresses.
-
-## fi_av_open
-
-fi_av_open allocates or opens an address vector. The properties and
-behavior of the address vector are defined by `struct fi_av_attr`.
-
-```c
-struct fi_av_attr {
- enum fi_av_type type; /* type of AV */
- int rx_ctx_bits; /* address bits to identify rx ctx */
- size_t count; /* # entries for AV */
- size_t ep_per_node; /* # endpoints per fabric address */
- const char *name; /* system name of AV */
- void *map_addr; /* base mmap address */
- uint64_t flags; /* operation flags */
-};
-```
-
-*type*
-: An AV type corresponds to a conceptual implementation of an address
- vector. The type specifies how an application views data stored in
- the AV, including how it may be accessed. Valid values are:
-
-- *FI_AV_MAP*
-: Addresses which are inserted into an AV are mapped to a native
- fabric address for use by the application. The use of FI_AV_MAP
- requires that an application store the returned fi_addr_t value
- that is associated with each inserted address. The advantage of
- using FI_AV_MAP is that the returned fi_addr_t value may contain
- encoded address data, which is immediately available when
- processing data transfer requests. This can eliminate or reduce
- the number of memory lookups needed when initiating a transfer.
- The disadvantage of FI_AV_MAP is the increase in memory usage
- needed to store the returned addresses. Addresses are stored in
- the AV using a provider specific mechanism, including, but not
- limited to a tree, hash table, or maintained on the heap.
-
-- *FI_AV_TABLE*
-: Addresses which are inserted into an AV of type FI_AV_TABLE are
- accessible using a simple index. Conceptually, the AV may be
- treated as an array of addresses, though the provider may implement
- the AV using a variety of mechanisms. When FI_AV_TABLE is used, the
- returned fi_addr_t is an index, with the index for an inserted
- address the same as its insertion order into the table. The index
- of the first address inserted into an FI_AV_TABLE will be 0, and
- successive insertions will be given sequential indices. Sequential
- indices will be assigned across insertion calls on the same AV.
-
-- *FI_AV_UNSPEC*
-: Provider will choose its preferred AV type. The AV type used will
- be returned through the type field in fi_av_attr.
-
-*Receive Context Bits (rx_ctx_bits)*
-: The receive context bits field is only for use with scalable
- endpoints. It indicates the number of bits reserved in a returned
- fi_addr_t, which will be used to identify a specific target receive
- context. See fi_rx_addr() and fi_endpoint(3) for additional details
- on receive contexts. The requested number of bits should be
- selected such that 2 ^ rx_ctx_bits >= rx_ctx_cnt for the endpoint.
-
-*count*
-: Indicates the expected number of addresses that will be inserted
- into the AV. The provider uses this to optimize resource
- allocations.
-
-*ep_per_node*
-: This field indicates the number of endpoints that will be associated
- with a specific fabric, or network, address. If the number of
- endpoints per node is unknown, this value should be set to 0. The
- provider uses this value to optimize resource allocations. For
- example, distributed, parallel applications may set this to the
- number of processes allocated per node, times the number of
- endpoints each process will open.
-
-*name*
-: An optional system name associated with the address vector to create
- or open. Address vectors may be shared across multiple processes
- which access the same named domain on the same node. The name field
- allows the underlying provider to identify a shared AV.
-
- If the name field is non-NULL and the AV is not opened for read-only
- access, a named AV will be created, if it does not already exist.
-
-*map_addr*
-: The map_addr determines the base fi_addr_t address that a provider
- should use when sharing an AV of type FI_AV_MAP between processes.
- Processes that provide the same value for map_addr to a shared AV
- may use the same fi_addr_t values returned from an fi_av_insert call.
-
- The map_addr may be used by the provider to mmap memory allocated
- for a shared AV between processes; however, the provider is not
- required to use the map_addr in this fashion. The only requirement
- is that an fi_addr_t returned as part of an fi_av_insert call on one
- process is usable on another process which opens an AV of the same
- name at the same map_addr value. The relationship between the
- map_addr and any returned fi_addr_t is not defined.
-
- If name is non-NULL and map_addr is 0, then the map_addr used by the
- provider will be returned through the attribute structure. The
- map_addr field is ignored if name is NULL.
-
-*flags*
-: The following flags may be used when opening an AV.
-
-- *FI_EVENT*
-: When the flag FI_EVENT is specified, all insert operations on this
- AV will occur asynchronously. There will be one EQ error entry
- generated for each failed address insertion, followed by one
- non-error event indicating that the insertion operation has
- completed. There will always be one non-error completion event for
- each insert operation, even if all addresses fail. The context
- field in all completions will be the context specified to the insert
- call, and the data field in the final completion entry will report
- the number of addresses successfully inserted.
- If an error occurs during the asynchronous insertion, an error
- completion entry is returned (see [`fi_eq`(3)](fi_eq.3.html) for a
- discussion of the fi_eq_err_entry error completion struct). The
- context field of the error completion will be the context that was
- specified in the insert call; the data field will contain the index
- of the failed address. There will be one error completion returned
- for each address that fails to insert into the AV.
-
- If an AV is opened with FI_EVENT, any insertions attempted before an
- EQ is bound to the AV will fail with -FI_ENOEQ.
-
- Error completions for failed insertions will contain the index of
- the failed address in the index field of the error completion entry.
-
- Note that the order of delivery of insert completions may not match
- the order in which the calls to fi_av_insert were made. The only
- guarantee is that all error completions for a given call to
- fi_av_insert will precede the single associated non-error
- completion.
-
-- *FI_READ*
-: Opens an AV for read-only access. An AV opened for read-only access
- must be named (name attribute specified), and the AV must exist.
-
-- *FI_SYMMETRIC*
-: Indicates that each node will be associated with the same number of
- endpoints, the same transport addresses will be allocated on each
- node, and the transport addresses will be sequential. This feature
- targets distributed applications on large fabrics and allows for
- highly-optimized storage of remote endpoint addressing.
-
-## fi_close
-
-The fi_close call is used to release all resources associated with an
-address vector. Note that any events queued on an event queue referencing
-the AV are left untouched. It is recommended that callers retrieve all
-events associated with the AV before closing it.
-
-When closing the address vector, there must be no opened endpoints associated
-with the AV. If resources are still associated with the AV when attempting to
-close, the call will return -FI_EBUSY.
-
-## fi_av_bind
-
-Associates an event queue with the AV. If an AV has been opened with
-`FI_EVENT`, then an event queue must be bound to the AV before any
-insertion calls are attempted. Any calls to insert addresses before
-an event queue has been bound will fail with `-FI_ENOEQ`.
-
-## fi_av_insert
-
-The fi_av_insert call inserts zero or more addresses into an AV. The
-number of addresses is specified through the count parameter. The
-addr parameter references an array of addresses to insert into the AV.
-Addresses inserted into an address vector must be in the same format
-as specified in struct fi_info:addr_format for the corresponding
-domain.
-
-For AV's of type FI_AV_MAP, once inserted addresses have been mapped,
-the mapped values are written into the buffer referenced by fi_addr.
-The fi_addr buffer must remain valid until the AV insertion has
-completed and an event has been generated to an associated event
-queue. The value of the returned fi_addr should be considered opaque
-by the application for AVs of type FI_AV_MAP. The returned value may
-point to an internal structure or a provider specific encoding of
-low-level addressing data, for example. In the latter case, use of
-FI_AV_MAP may be able to avoid memory references during data transfer
-operations.
-
-For AV's of type FI_AV_TABLE, addresses are placed into the table in
-order. An address is inserted at the lowest index that corresponds to
-an unused table location, with indices starting at 0. That is, the
-first address inserted may be referenced at index 0, the second at index
-1, and so forth. When addresses are inserted into an AV table,
-the assigned fi_addr values will be simple indices
-corresponding to the entry into the table where the address was
-inserted. Index values accumulate across successive insert calls
-in the order the calls are made, not necessarily in the order the
-insertions complete.
-
-Because insertions occur at a pre-determined index, the fi_addr
-parameter may be NULL. If fi_addr is non-NULL, it must reference
-an array of fi_addr_t, and the buffer must remain valid until the
-insertion operation completes. Note that if
-fi_addr is NULL and synchronous operation is requested, individual
-insertion failures cannot be reported and the application must use
-other calls, such as `fi_av_lookup` to learn which specific addresses
-failed to insert.
-
-*flags*
-: The following flag may be passed to fi_av_insert
-
-- *FI_MORE*
-: In order to allow optimized address insertion, the application may
- specify the FI_MORE flag to the insert call to give a hint to the
- provider that more insertion requests will follow, allowing the
- provider to aggregate insertion requests if desired. Providers are
- free to ignore FI_MORE.
-
-## fi_av_insertsvc
-
-The fi_av_insertsvc call behaves similar to fi_av_insert, but allows the
-application to specify the node and service names, similar to the
-fi_getinfo inputs, rather than an encoded address. The node and service
-parameters are defined the same as fi_getinfo(3). Node should be a string
-that corresponds to a hostname or network address. The service string
-corresponds to a textual representation of a transport address.
-
-## fi_av_insertsym
-
-fi_av_insertsym performs a symmetric insert that inserts a sequential
-range of nodes and/or service addresses into an AV. The svccnt
-parameter indicates the number of transport (endpoint) addresses to
-insert into the AV for each node address, with the service parameter
-specifying the starting transport address. Inserted transport
-addresses will be of the range {service, service + svccnt - 1},
-inclusive. All service addresses for a node will be inserted before
-the next node is inserted.
-
-The nodecnt parameter indicates the number of node (network) addresses
-to insert into the AV, with the node parameter specifying the starting
-node address. Inserted node addresses will be of the range {node,
-node + nodecnt - 1}, inclusive. If node is a non-numeric string, such
-as a hostname, it must contain a numeric suffix if nodecnt > 1.
-
-As an example, if node = "10.1.1.1", nodecnt = 2, service = "5000",
-and svccnt = 2, the following addresses will be inserted into the AV
-in the order shown: 10.1.1.1:5000, 10.1.1.1:5001, 10.1.1.2:5000,
-10.1.1.2:5001. If node were replaced by the hostname "host10", the
-addresses would be: host10:5000, host10:5001, host11:5000,
-host11:5001.
-
-The total number of inserted addresses will be nodecnt x svccnt.
-
-## fi_av_remove
-
-fi_av_remove removes a set of addresses from an address vector. All
-resources associated with the indicated addresses are released, and no
-future references to either the mapped address (in the case of
-FI_AV_MAP) or index (FI_AV_TABLE) are allowed.
-
-The use of fi_av_remove is an optimization that applications may use
-to free memory allocated with addresses that will no longer be
-accessed. Inserted addresses are not required to be removed.
-fi_av_close will automatically cleanup any resources associated with
-addresses remaining in the AV when it is invoked.
-
-## fi_av_lookup
-
-This call returns the address stored in the address vector that
-corresponds to the given fi_addr. The returned address is the same
-format as those stored by the AV. On input, the addrlen parameter
-should indicate the size of the addr buffer. If the actual address is
-larger than what can fit into the buffer, it will be truncated. On
-output, addrlen is set to the size of the buffer needed to store the
-address, which may be larger than the input value.
-
-## fi_rx_addr
-
-This function is used to convert an endpoint address, returned by
-fi_av_insert, into an address that specifies a target receive context.
-The specified fi_addr parameter must either be a value returned from
-fi_av_insert, in the case of FI_AV_MAP, or an index, in the case of
-FI_AV_TABLE. The value for rx_ctx_bits must match that specified in
-the AV attributes for the given address.
-
-Connected endpoints that support multiple receive contexts, but are
-not associated with address vectors should specify FI_ADDR_NOTAVAIL
-for the fi_addr parameter.
-
-## fi_av_straddr
-
-The fi_av_straddr function converts the provided address into a
-printable string. The specified address must be of the same format as
-those stored by the AV, though the address itself is not required to
-have been inserted. On input, the len parameter should specify the
-size of the buffer referenced by buf. On output, addrlen is set to the
-size of the buffer needed to store the address. This size may be
-larger than the input len. If the provided buffer is too small, the
-results will be truncated. fi_av_straddr returns a pointer to buf.
-
-# NOTES
-
-Providers may implement AV's using a variety of mechanisms.
-Specifically, a provider may begin resolving inserted addresses as
-soon as they have been added to an AV, even if asynchronous operation
-has been specified. Similarly, a provider may lazily release
-resources from removed entries.
-
-# RETURN VALUES
-
-Insertion calls for an AV opened for synchronous operation will return
-the number of addresses that were successfully inserted. In the case of
-failure, the return value will be less than the number of addresses that
-were specified.
-
-Insertion calls for an AV opened for asynchronous operation (with FI_EVENT
-flag specified) will return 0 if the operation was successfully initiated.
-In the case of failure, a negative fabric errno will be returned.
-
-All other calls return 0 on success, or a negative value corresponding to
-fabric errno on error.
-Fabric errno values are defined in
-`rdma/fi_errno.h`.
-
-# ERRORS
-
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_eq`(3)](fi_eq.3.html)
diff --git a/man/fi_cm.3.md b/man/fi_cm.3.md
deleted file mode 100644
index 144b890..0000000
--- a/man/fi_cm.3.md
+++ /dev/null
@@ -1,219 +0,0 @@
----
-layout: page
-title: fi_cm(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_cm - Connection management operations
-
-fi_connect / fi_listen / fi_accept / fi_reject / fi_shutdown
-: Manage endpoint connection state.
-
-fi_setname / fi_getname / fi_getpeer
-: Set local, or return local or peer endpoint address.
-
-# SYNOPSIS
-
-```c
-#include <rdma/fi_cm.h>
-
-int fi_connect(struct fid_ep *ep, const void *addr,
- const void *param, size_t paramlen);
-
-int fi_listen(struct fid_pep *pep);
-
-int fi_accept(struct fid_ep *ep, const void *param, size_t paramlen);
-
-int fi_reject(struct fid_pep *pep, fid_t handle,
- const void *param, size_t paramlen);
-
-int fi_shutdown(struct fid_ep *ep, uint64_t flags);
-
-int fi_setname(fid_t fid, void *addr, size_t addrlen);
-
-int fi_getname(fid_t fid, void *addr, size_t *addrlen);
-
-int fi_getpeer(struct fid_ep *ep, void *addr, size_t *addrlen);
-```
-
-# ARGUMENTS
-
-*ep / pep*
-: Fabric endpoint on which to change connection state.
-
-*fid*
-Active or passive endpoint to get/set address.
-
-*addr*
-: Buffer to address. On a set call, the endpoint will be assigned the
- specified address. On a get, the local address will be copied into the
- buffer, up to the space provided. For connect, this parameter indicates
- the peer address to connect to. The address must be in the same format as
- that specified using fi_info: addr_format when the endpoint was created.
-
-*addrlen*
-: On input, specifies size of addr buffer. On output, stores number
- of bytes written to addr buffer.
-
-*param*
-: User-specified data exchanged as part of the connection exchange.
-
-*paramlen*
-: Size of param buffer.
-
-*info*
-: Fabric information associated with a connection request.
-
-*flags*
-: Additional flags for controlling connection operation.
-
-*context*
-: User context associated with the request.
-
-# DESCRIPTION
-
-Connection management functions are used to connect an
-connection-oriented endpoint to a peer endpoint.
-
-## fi_listen
-
-The fi_listen call indicates that the specified endpoint should be
-transitioned into a passive connection state, allowing it to accept
-incoming connection requests. Connection requests against a listening
-endpoint are reported asynchronously to the user through a bound CM
-event queue using the FI_CONNREQ event type. The number of outstanding
-connection requests that can be queued at an endpoint is limited by the
-listening endpoint's backlog parameter. The backlog is initialized
-based on administrative configuration values, but may be adjusted
-through the fi_control call.
-
-## fi_connect
-
-The fi_connect call initiates a connection request on a
-connection-oriented endpoint to the destination address.
-
-## fi_accept / fi_reject
-
-The fi_accept and fi_reject calls are used on the passive (listening)
-side of a connection to accept or reject a connection request,
-respectively. To accept a connection, the listening application first
-waits for a connection request event (FI_CONNREQ).
-After receiving such an event, the application
-allocates a new endpoint to accept the connection. This endpoint must
-be allocated using an fi_info structure referencing the handle from this
-FI_CONNREQ event. fi_accept is then invoked
-with the newly allocated endpoint. If
-the listening application wishes to reject a connection request, it calls
-fi_reject with the listening endpoint and
-a reference to the connection request.
-
-A successfully accepted connection request will result in the active
-(connecting) endpoint seeing an FI_CONNECTED event on its associated
-event queue. A rejected or failed connection request will generate an
-error event. The error entry will provide additional details describing
-the reason for the failed attempt.
-
-An FI_CONNECTED event will also be generated on the passive side for the
-accepting endpoint once the connection has been properly established.
-The fid of the FI_CONNECTED event will be that of the endpoint passed to
-fi_accept as opposed to the listening passive endpoint.
-Outbound data transfers cannot be initiated on a connection-oriented
-endpoint until an FI_CONNECTED event has been generated. However, receive
-buffers may be associated with an endpoint anytime.
-
-## fi_shutdown
-
-The fi_shutdown call is used to gracefully disconnect an endpoint from
-its peer. The flags parameter is reserved and must be 0.
-
-Outstanding operations posted to the endpoint when fi_shutdown is
-called will be canceled or discarded. Notification of canceled operations
-will be reported by the provider to the corresponding completion
-queue(s). Discarded operations will silently be dropped, with no
-completions generated. The choice of canceling, versus discarding
-operations, is provider dependent. However, all canceled completions
-will be written before fi_shutdown returns.
-
-When called, fi_shutdown does not affect completions already written to a
-completion queue. Any queued completions associated with asynchronous
-operations posted to the endpoint may still be retrieved from the
-corresponding completion queue(s) after an endpoint has been shutdown.
-
-An FI_SHUTDOWN event will be generated for an endpoint when the remote
-peer issues a disconnect using fi_shutdown or abruptly closes the endpoint.
-Note that in the abrupt close case, an FI_SHUTDOWN event will only be
-generated if the peer system is reachable and a service or kernel agent
-on the peer system is able to notify the local endpoint that the connection
-has been aborted.
-
-## fi_setname
-
-The fi_setname call may be used to modify or assign the address of the
-local endpoint. It is conceptually similar to the socket bind operation.
-An endpoint may be assigned an address on its creation,
-through the fi_info structure. The fi_setname call allows an endpoint to
-be created without being associated with a specific service (e.g., port
-number) and/or node (e.g., network) address, with the addressing assigned
-dynamically. The format of the specified addressing data must match that
-specified through the fi_info structure when the endpoint was created.
-
-If no service address is specified and a service address
-has not yet been assigned to the endpoint, then the provider will allocate
-a service address and assign it to the endpoint. If a node or service
-address is specified, then, upon successful completion of fi_setname,
-the endpoint will be assigned the given addressing. If an address cannot
-be assigned, or the endpoint address cannot be modified, an appropriate
-fabric error number is returned.
-
-## fi_getname / fi_getpeer
-
-The fi_getname and fi_getpeer calls may be used to retrieve the local or
-peer endpoint address, respectively. On input, the addrlen parameter should
-indicate the size of the addr buffer. If the actual address is larger than
-what can fit into the buffer, it will be truncated and -FI_ETOOSMALL will
-be returned. On output, addrlen is set to the size of the buffer needed
-to store the address, which may be larger than the input value.
-
-fi_getname is not guaranteed to return a valid source address until after the
-specified endpoint has been enabled or has had an address assigned. An
-endpoint may be enabled explicitly through fi_enable, or implicitly, such as
-through fi_connect or fi_listen. An address may be assigned using fi_setname.
-fi_getpeer is not guaranteed to return a valid peer address until an endpoint
-has been completely connected -- an FI_CONNECTED event has been generated.
-
-# FLAGS
-
-Flag values are reserved and must be 0.
-
-# RETURN VALUE
-
-Returns 0 on success. On error, a negative value corresponding to fabric
-errno is returned. Fabric errno values are defined in
-`rdma/fi_errno.h`.
-
-# ERRORS
-
-
-# NOTES
-
-For connection-oriented endpoints, the buffer referenced by param
-will be sent as part of the connection request or response, subject
-to the constraints of the underlying connection protocol.
-Applications may use fi_getopt with the FI_OPT_CM_DATA_SIZE endpoint
-option to determine the size of application data that may be exchanged as
-part of a connection request or response. The fi_connect, fi_accept, and
-fi_reject calls will silently truncate any application data which cannot
-fit into underlying protocol messages. User data exchanged as part of
-the connection process is available as part of the fi_eq_cm_entry
-structure, for FI_CONNREQ and FI_CONNECTED events, or as additional
-err_data to fi_eq_err_entry, in the case of a rejected connection.
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_eq`(3)](fi_eq.3.html)
diff --git a/man/fi_cntr.3.md b/man/fi_cntr.3.md
deleted file mode 100644
index cbadf98..0000000
--- a/man/fi_cntr.3.md
+++ /dev/null
@@ -1,250 +0,0 @@
----
-layout: page
-title: fi_cntr(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_cntr \- Completion and event counter operations
-
-fi_cntr_open / fi_close
-: Allocate/free a counter
-
-fi_cntr_read
-: Read the current value of a counter
-
-fi_cntr_readerr
-: Reads the number of operations which have completed in error.
-
-fi_cntr_add
-: Increment a counter by a specified value
-
-fi_cntr_set
-: Set a counter to a specified value
-
-fi_cntr_wait
-: Wait for a counter to be greater or equal to a threshold value
-
-# SYNOPSIS
-
-```c
-#include <rdma/fi_domain.h>
-
-int fi_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
- struct fid_cntr **cntr, void *context);
-
-int fi_close(struct fid *cntr);
-
-uint64_t fi_cntr_read(struct fid_cntr *cntr);
-
-uint64_t fi_cntr_readerr(struct fid_cntr *cntr);
-
-int fi_cntr_add(struct fid_cntr *cntr, uint64_t value);
-
-int fi_cntr_set(struct fid_cntr *cntr, uint64_t value);
-
-int fi_cntr_wait(struct fid_cntr *cntr, uint64_t threshold,
- int timeout);
-```
-
-# ARGUMENTS
-
-*domain*
-: Fabric domain
-
-*cntr*
-: Fabric counter
-
-*attr*
-: Counter attributes
-
-*context*
-: User specified context associated with the counter
-
-*value*
-: Value to increment or set counter
-
-*threshold*
-: Value to compare counter against
-
-*timeout*
-: Time in milliseconds to wait. A negative value indicates infinite
- timeout.
-
-# DESCRIPTION
-
-Counters record the number of requested operations that have
-completed. Counters can provide a light-weight completion mechanism
-by suppressing the generation of a full completion event. They are
-useful for applications that only need to know the number of requests
-that have completed, and not details about each request. For example,
-counters may be useful for implementing credit based flow control or
-tracking the number of remote processes which have responded to a
-request.
-
-Counters typically only count successful completions. However, if an
-operation completes in error, it may increment an associated error
-value.
-
-## fi_cntr_open
-
-fi_cntr_open allocates a new fabric counter. The properties and
-behavior of the counter are defined by `struct fi_cntr_attr`.
-
-```c
-struct fi_cntr_attr {
- enum fi_cntr_events events; /* type of events to count */
- enum fi_wait_obj wait_obj; /* requested wait object */
- struct fid_wait *wait_set; /* optional wait set */
- uint64_t flags; /* operation flags */
-};
-```
-
-*events*
-: A counter captures different types of events. The specific type
- which is to counted are one of the following:
-
-- *FI_CNTR_EVENTS_COMP*
-: The counter increments for every successful completion that occurs
- on an associated bound endpoint. The type of completions -- sends
- and/or receives -- which are counted may be restricted using control
- flags when binding the counter an the endpoint. Counters increment
- on all successful completions, separately from whether the operation
- generates an entry in an event queue.
-
-*wait_obj*
-: Counters may be associated with a specific wait object. Wait
- objects allow applications to block until the wait object is
- signaled, indicating that a counter has reached a specific
- threshold. Users may use fi_control to retrieve the underlying wait
- object associated with a counter, in order to use it in other system
- calls. The following values may be used to specify the type of wait
- object associated with a counter: FI_WAIT_NONE, FI_WAIT_UNSPEC,
- FI_WAIT_SET, FI_WAIT_FD, and FI_WAIT_MUTEX_COND.
-
-- *FI_WAIT_NONE*
-: Used to indicate that the user will not block (wait) for events on
- the counter.
-
-- *FI_WAIT_UNSPEC*
-: Specifies that the user will only wait on the counter using fabric
- interface calls, such as fi_cntr_wait. In this case, the
- underlying provider may select the most appropriate or highest
- performing wait object available, including custom wait mechanisms.
- Applications that select FI_WAIT_UNSPEC are not guaranteed to
- retrieve the underlying wait object.
-
-- *FI_WAIT_SET*
-: Indicates that the event counter should use a wait set object to
- wait for events. If specified, the wait_set field must reference an
- existing wait set object.
-
-- *FI_WAIT_FD*
-: Indicates that the counter should use a file descriptor as its wait
- mechanism. A file descriptor wait object must be usable in select,
- poll, and epoll routines. However, a provider may signal an FD wait
- object by marking it as readable, writable, or with an error.
-
-- *FI_WAIT_MUTEX_COND*
-: Specifies that the counter should use a pthread mutex and cond
- variable as a wait object.
-
-*wait_set*
-: If wait_obj is FI_WAIT_SET, this field references a wait object to
- which the event counter should attach. When an event is added to
- the event counter, the corresponding wait set will be signaled if
- all necessary conditions are met. The use of a wait_set enables an
- optimized method of waiting for events across multiple event
- counters. This field is ignored if wait_obj is not FI_WAIT_SET.
-
-*flags*
-: Flags are reserved for future use, and must be set to 0.
-
-## fi_close
-
-The fi_close call releases all resources associated with a counter. When
-closing the counter, there must be no opened endpoints, transmit contexts,
-receive contexts or memory regions associated with the counter. If resources
-are still associated with the counter when attempting to close, the call will
-return -FI_EBUSY.
-
-
-## fi_cntr_control
-
-The fi_cntr_control call is used to access provider or implementation
-specific details of the counter. Access to the counter should be
-serialized across all calls when fi_cntr_control is invoked, as it may
-redirect the implementation of counter operations. The following
-control commands are usable with a counter:
-
-*FI_GETOPSFLAG (uint64_t \*)*
-: Returns the current default operational flags associated with the counter.
-
-*FI_SETOPSFLAG (uint64_t \*)*
-: Modifies the current default operational flags associated with the
- counter.
-
-*FI_GETWAIT (void \*\*)*
-: This command allows the user to retrieve the low-level wait object
- associated with the counter. The format of the wait-object is
- specified during counter creation, through the counter attributes.
- See fi_eq.3 for addition details using control with FI_GETWAIT.
-
-## fi_cntr_read
-
-The fi_cntr_read call returns the current value of the counter.
-
-## fi_cntr_readerr
-
-The read error call returns the number of operations that completed in
-error and were unable to update the counter.
-
-## fi_cntr_add
-
-This adds the user-specified value to the counter.
-
-## fi_cntr_set
-
-This sets the counter to the specified value.
-
-## fi_cntr_wait
-
-This call may be used to wait until the counter reaches the specified
-threshold, or until an error or timeout occurs. Upon successful
-return from this call, the counter will be greater than or equal to
-the input threshold value.
-
-If an operation associated with the counter encounters an error, it
-will increment the error value associated with the counter. Any
-change in a counter's error value will unblock any thread inside
-fi_cntr_wait.
-
-If the call returns due to timeout, -FI_ETIMEDOUT will be returned.
-The error value associated with the counter remains unchanged.
-
-It is invalid for applications to call this function if the counter
-has been configured with a wait object of FI_WAIT_NONE or FI_WAIT_SET.
-
-# RETURN VALUES
-
-Returns 0 on success. On error, a negative value corresponding to
-fabric errno is returned.
-
-fi_cntr_read / fi_cntr_readerr
-: Returns the current value of the counter.
-
-Fabric errno values are defined in
-`rdma/fi_errno.h`.
-
-# NOTES
-
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_eq`(3)](fi_eq.3.html),
-[`fi_poll`(3)](fi_poll.3.html)
diff --git a/man/fi_control.3.md b/man/fi_control.3.md
deleted file mode 100644
index 61f41a5..0000000
--- a/man/fi_control.3.md
+++ /dev/null
@@ -1,47 +0,0 @@
----
-layout: page
-title: fi_control(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_control \- Perform an operation on a fabric resource.
-
-# SYNOPSIS
-
-```c
-#include <rdma/fabric.h>
-
-int fi_control(struct fid *fid, int command, void *arg);
-```
-
-
-# ARGUMENTS
-
-*fid*
-: Fabric resource
-
-*command*
-: Operation to perform
-
-*arg*
-: Optional argument to the command
-
-# DESCRIPTION
-
-The fi_control operation is used to perform one or more operations on a
-fabric resource. Conceptually, fi_control is similar to the POSIX fcntl
-routine. The exact behavior of using fi_control depends on the fabric
-resource being operated on, the specified command, and any provided
-arguments for the command. For specific details, see the fabric resource
-specific help pages noted below.
-
-# SEE ALSO
-
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_cm`(3)](fi_cm.3.html),
-[`fi_cntr`(3)](fi_cntr.3.html),
-[`fi_cq`(3)](fi_cq.3.html),
-[`fi_eq`(3)](fi_eq.3.html),
diff --git a/man/fi_cq.3.md b/man/fi_cq.3.md
deleted file mode 100644
index 92cbde8..0000000
--- a/man/fi_cq.3.md
+++ /dev/null
@@ -1,553 +0,0 @@
----
-layout: page
-title: fi_cq(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_cq \- Completion queue operations
-
-fi_cq_open / fi_close
-: Open/close a completion queue
-
-fi_control
-: Control CQ operation or attributes.
-
-fi_cq_read / fi_cq_readfrom / fi_cq_readerr
-: Read a completion from a completion queue
-
-fi_cq_sread / fi_cq_sreadfrom
-: A synchronous (blocking) read that waits until a specified condition
- has been met before reading a completion from a completion queue.
-
-fi_cq_signal
-: Unblock any thread waiting in fi_cq_sread or fi_cq_sreadfrom.
-
-fi_cq_strerror
-: Converts provider specific error information into a printable string
-
-# SYNOPSIS
-
-```c
-#include <rdma/fi_domain.h>
-
-int fi_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
- struct fid_cq **cq, void *context);
-
-int fi_close(struct fid *cq);
-
-int fi_control(struct fid *cq, int command, void *arg);
-
-ssize_t fi_cq_read(struct fid_cq *cq, void *buf, size_t count);
-
-ssize_t fi_cq_readfrom(struct fid_cq *cq, void *buf, size_t count,
- fi_addr_t *src_addr);
-
-ssize_t fi_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf,
- uint64_t flags);
-
-ssize_t fi_cq_sread(struct fid_cq *cq, void *buf, size_t count,
- const void *cond, int timeout);
-
-ssize_t fi_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count,
- fi_addr_t *src_addr, const void *cond, int timeout);
-
-int fi_cq_signal(struct fid_cq *cq);
-
-const char * fi_cq_strerror(struct fid_cq *cq, int prov_errno,
- const void *err_data, char *buf, size_t len);
-```
-
-# ARGUMENTS
-
-*domain*
-: Open resource domain
-
-*cq*
-: Completion queue
-
-*attr*
-: Completion queue attributes
-
-*context*
-: User specified context associated with the completion queue.
-
-*buf*
-: For read calls, the data buffer to write completions into. For
- write calls, a completion to insert into the completion queue. For
- fi_cq_strerror, an optional buffer that receives printable error
- information.
-
-*count*
-: Number of CQ entries.
-
-*len*
-: Length of data buffer
-
-*src_addr*
-: Source address of a completed receive operation
-
-*flags*
-: Additional flags to apply to the operation
-
-*command*
-: Command of control operation to perform on CQ.
-
-*arg*
-: Optional control argument
-
-*cond*
-: Condition that must be met before a completion is generated
-
-*timeout*
-: Time in milliseconds to wait. A negative value indicates infinite
- timeout.
-
-*prov_errno*
-: Provider specific error value
-
-*err_data*
-: Provider specific error data related to a completion
-
-# DESCRIPTION
-
-Completion queues are used to report events associated with data
-transfers. They are associated with message sends and receives, RMA,
-atomic, tagged messages, and triggered events. Reported events are
-usually associated with a fabric endpoint, but may also refer to
-memory regions used as the target of an RMA or atomic operation.
-
-## fi_cq_open
-
-fi_cq_open allocates a new completion queue. Unlike event queues,
-completion queues are associated with a resource domain and may be
-offloaded entirely in provider hardware.
-
-The properties and behavior of a completion queue are defined by
-`struct fi_cq_attr`.
-
-```c
-struct fi_cq_attr {
- size_t size; /* # entries for CQ */
- uint64_t flags; /* operation flags */
- enum fi_cq_format format; /* completion format */
- enum fi_wait_obj wait_obj; /* requested wait object */
- int signaling_vector; /* interrupt affinity */
- enum fi_cq_wait_cond wait_cond; /* wait condition format */
- struct fid_wait *wait_set; /* optional wait set */
-};
-```
-
-*size*
-: Specifies the minimum size of a completion queue. A value of 0 indicates that
- the provider may choose a default value.
-
-*flags*
-: Flags that control the configuration of the CQ. This field must be 0.
-
-*format*
-: Completion queues allow the application to select the amount of
- detail that it must store and report. The format attribute allows
- the application to select one of several completion formats,
- indicating the structure of the data that the completion queue
- should return when read. Supported formats and the structures that
- correspond to each are listed below. The meaning of the CQ entry
- fields are defined in the _Completion Fields_ section.
-
-- *FI_CQ_FORMAT_UNSPEC*
-: If an unspecified format is requested, then the CQ will use a
- provider selected default format.
-
-- *FI_CQ_FORMAT_CONTEXT*
-: Provides only user specified context that was associated with the
- completion.
-
-```c
-struct fi_cq_entry {
- void *op_context; /* operation context */
-};
-```
-
-- *FI_CQ_FORMAT_MSG*
-: Provides minimal data for processing completions, with expanded
- support for reporting information about received messages.
-
-```c
-struct fi_cq_msg_entry {
- void *op_context; /* operation context */
- uint64_t flags; /* completion flags */
- size_t len; /* size of received data */
-};
-```
-
-- *FI_CQ_FORMAT_DATA*
-: Provides data associated with a completion. Includes support for
- received message length, remote EQ data, and multi-receive buffers.
-
-```c
-struct fi_cq_data_entry {
- void *op_context; /* operation context */
- uint64_t flags; /* completion flags */
- size_t len; /* size of received data */
- void *buf; /* receive data buffer */
- uint64_t data; /* completion data */
-};
-```
-
-- *FI_CQ_FORMAT_TAGGED*
-: Expands completion data to include support for the tagged message
- interfaces.
-
-```c
-struct fi_cq_tagged_entry {
- void *op_context; /* operation context */
- uint64_t flags; /* completion flags */
- size_t len; /* size of received data */
- void *buf; /* receive data buffer */
- uint64_t data; /* completion data */
- uint64_t tag; /* received tag */
-};
-```
-
-*wait_obj*
-: CQ's may be associated with a specific wait object. Wait objects
- allow applications to block until the wait object is signaled,
- indicating that a completion is available to be read. Users may use
- fi_control to retrieve the underlying wait object associated with an
- CQ, in order to use it in other system calls. The following values
- may be used to specify the type of wait object associated with an
- CQ: FI_WAIT_NONE, FI_WAIT_UNSPEC, FI_WAIT_SET, FI_WAIT_FD, and
- FI_WAIT_MUTEX_COND.
-
-- *FI_WAIT_NONE*
-: Used to indicate that the user will not block (wait) for completions
- on the CQ. When FI_WAIT_NONE is specified, the application may not
- call fi_cq_sread or fi_cq_sreadfrom.
-
-- *FI_WAIT_UNSPEC*
-: Specifies that the user will only wait on the CQ using fabric
- interface calls, such as fi_cq_sread or fi_cq_sreadfrom. In this
- case, the underlying provider may select the most appropriate or
- highest performing wait object available, including custom wait
- mechanisms. Applications that select FI_WAIT_UNSPEC are not
- guaranteed to retrieve the underlying wait object.
-
-- *FI_WAIT_SET*
-: Indicates that the completion queue should use a wait set object to
- wait for completions. If specified, the wait_set field must
- reference an existing wait set object.
-
-- *FI_WAIT_FD*
-: Indicates that the CQ should use a file descriptor as its wait
- mechanism. A file descriptor wait object must be usable in select,
- poll, and epoll routines. However, a provider may signal an FD wait
- object by marking it as readable, writable, or with an error.
-
-- *FI_WAIT_MUTEX_COND*
-: Specifies that the CQ should use a pthread mutex and cond variable
- as a wait object.
-
-*signaling_vector*
-: Indicates which processor core interrupts associated with the EQ should
- target.
-
-*wait_cond*
-: By default, when a completion is inserted into an CQ that supports
- blocking reads (fi_cq_sread/fi_cq_sreadfrom), the corresponding wait
- object is signaled. Users may specify a condition that must first
- be met before the wait is satisfied. This field indicates how the
- provider should interpret the cond field, which describes the
- condition needed to signal the wait object.
-
- A wait condition should be treated as an optimization. Providers
- are not required to meet the requirements of the condition before
- signaling the wait object. Applications should not rely on the
- condition necessarily being true when a blocking read call returns.
-
- If wait_cond is set to FI_CQ_COND_NONE, then no additional
- conditions are applied to the signaling of the CQ wait object, and
- the insertion of any new entry will trigger the wait condition. If
- wait_cond is set to FI_CQ_COND_THRESHOLD, then the cond field is
- interpreted as a size_t threshold value. The threshold indicates
- the number of entries that are to be queued before at the CQ before
- the wait is satisfied.
-
- This field is ignored if wait_obj is set to FI_WAIT_NONE.
-
-*wait_set*
-: If wait_obj is FI_WAIT_SET, this field references a wait object to
- which the completion queue should attach. When an event is inserted
- into the completion queue, the corresponding wait set will be
- signaled if all necessary conditions are met. The use of a wait_set
- enables an optimized method of waiting for events across multiple
- event and completion queues. This field is ignored if wait_obj is
- not FI_WAIT_SET.
-
-## fi_close
-
-The fi_close call releases all resources associated with a completion
-queue. Any completions which remain on the CQ when it is closed are
-lost.
-
-When closing the CQ, there must be no opened endpoints, transmit contexts, or
-receive contexts associated with the CQ. If resources are still associated
-with the CQ when attempting to close, the call will return -FI_EBUSY.
-
-## fi_control
-
-The fi_control call is used to access provider or implementation
-specific details of the completion queue. Access to the CQ should be
-serialized across all calls when fi_control is invoked, as it may
-redirect the implementation of CQ operations. The following control
-commands are usable with an CQ.
-
-*FI_GETWAIT (void \*\*)*
-: This command allows the user to retrieve the low-level wait object
- associated with the CQ. The format of the wait-object is specified
- during CQ creation, through the CQ attributes. The fi_control arg
- parameter should be an address where a pointer to the returned wait
- object will be written. See fi_eq.3 for addition details using
- fi_control with FI_GETWAIT.
-
-## fi_cq_read / fi_cq_readfrom
-
-The fi_cq_read and fi_cq_readfrom operations perform a non-blocking
-read of completion data from the CQ. The format of the completion
-event is determined using the fi_cq_format option that was specified
-when the CQ was opened. Multiple completions may be retrieved from a
-CQ in a single call. The maximum number of entries to return is
-limited to the specified count parameter, with the number of entries
-successfully read from the CQ returned by the call.
-
-The fi_cq_readfrom call allows the CQ to return source address
-information to the user for any received data. Source address data is
-only available for those endpoints configured with FI_SOURCE
-capability. If fi_cq_readfrom is called on an endpoint for which
-source addressing data is not available, the source address will be
-set to FI_ADDR_NOTAVAIL. The number of input src_addr entries must
-the the same as the count parameter.
-
-CQs are optimized to report operations which have completed
-successfully. Operations which fail are reported 'out of band'. Such
-operations are retrieved using the fi_cq_readerr function. When an
-operation that completes with an unexpected error is inserted into an
-CQ, it is placed into a temporary error queue. Attempting to read
-from an CQ while an item is in the error queue results in an FI_EAVAIL
-failure. Applications may use this return code to determine when to
-call fi_cq_readerr.
-
-## fi_cq_sread / fi_cq_sreadfrom
-
-The fi_cq_sread and fi_cq_sreadfrom calls are the blocking equivalent
-operations to fi_cq_read and fi_cq_readfrom. Their behavior is
-similar to the non-blocking calls, with the exception that the calls
-will not return until either a completion has been read from the CQ or
-an error or timeout occurs.
-
-It is invalid for applications to call these functions if the CQ
-has been configured with a wait object of FI_WAIT_NONE or FI_WAIT_SET.
-
-## fi_cq_readerr
-
-The read error function, fi_cq_readerr, retrieves information
-regarding any asynchronous operation which has completed with an
-unexpected error. fi_cq_readerr is a non-blocking call, returning
-immediately whether an error completion was found or not.
-
-Error information is reported to the user through `struct
-fi_cq_err_entry`. The format of this structure is defined below.
-
-```c
-struct fi_cq_err_entry {
- void *op_context; /* operation context */
- uint64_t flags; /* completion flags */
- size_t len; /* size of received data */
- void *buf; /* receive data buffer */
- uint64_t data; /* completion data */
- uint64_t tag; /* message tag */
- size_t olen; /* overflow length */
- int err; /* positive error code */
- int prov_errno; /* provider error code */
- void *err_data; /* error data */
-};
-```
-
-The general reason for the error is provided through the err field.
-Provider specific error information may also be available through the
-prov_errno and err_data fields. The err_data field, if set, will
-reference an internal buffer owned by the provider. The contents of
-the buffer will remain valid until a subsequent read call against the
-CQ. Users may call fi_cq_strerror to convert provider specific error
-information into a printable string for debugging purposes.
-
-## fi_cq_signal
-
-The fi_cq_signal call will unblock any thread waiting in fi_cq_sread
-or fi_cq_sreadfrom. This may be used to wake-up a thread
-that is blocked waiting to read a completion operation. The fi_cq_signal
-operation is only available if the CQ was configured with a wait object.
-
-# COMPLETION FIELDS
-
-The CQ entry data structures share many of the same fields. The meanings
-of these fields are the same for all CQ entry structure formats.
-
-*op_context*
-: The operation context is the application specified context value that
- was provided with an asynchronous operation. The op_context field is
- valid for all completions.
-
-*flags*
-: This specifies flags associated with the completed operation. The
- _Completion Flags_ section below lists valid flag values. Flags are
- set for all relevant completions.
-
-*len*
-: This len field only applies to completed receive operations. It indicates
- the size of received message data -- i.e. how many data bytes were
- actually placed into the associated receive buffer. If an endpoint has
- been configured with the FI_MSG_PREFIX mode, the len also reflects the size
- of the prefix buffer.
-
-*buf*
-: The buf field is only valid for completed receive operations, and only
- applies when the receive buffer was posted with the FI_MULTI_RECV flag.
- In this case, buf points to the starting location where the receive
- data was placed.
-
-*data*
-: The data field is only valid if the FI_REMOTE_CQ_DATA completion flag
- is set, and only applies to receive completions. If FI_REMOTE_CQ_DATA
- is set, this field will contain the completion data provided by the peer
- as part of their transmit request. The completion data will be given
- in host byte order.
-
-*tag*
-: A tag applies only to received messages that occur using the tagged
- interfaces. This field contains the tag that was included with the
- received message. The tag will be in host byte order.
-
-*olen*
-: The olen field applies to received messages. It is used to indicate
- that a received message has overrun the available buffer space and
- has been truncated. The olen specifies the amount of data that did
- not fit into the available receive buffer and was discarded.
-
-*err*
-: This err code is a positive fabric errno associated with a completion.
- The err value indicates the general reason for an error, if one occurred.
- See fi_errno.3 for a list of possible error codes.
-
-*prov_errno*
-: On an error, prov_errno may contain a provider specific error code. The
- use of this field and its meaning is provider specific. It is intended
- to be used as a debugging aid. See fi_cq_strerror for additional details
- on converting this error value into a human readable string.
-
-*err_data*
-: On an error, err_data may reference a provider specific amount of data
- associated with an error. The use of this field and its meaning is
- provider specific. It is intended to be used as a debugging aid. See
- fi_cq_strerror for additional details on converting this error data into
- a human readable string. Providers are allowed to reuse a single internal
- buffer to store additional error information. As a result, error data
- is only guaranteed to be available until the next time the CQ is read.
-
-# COMPLETION FLAGS
-
-Completion flags provide additional details regarding the completed
-operation. The following completion flags are defined.
-
-*FI_SEND*
-: Indicates that the completion was for a send operation. This flag
- may be combined with an FI_MSG or FI_TAGGED flag.
-
-*FI_RECV*
-: Indicates that the completion was for a receive operation. This flag
- may be combined with an FI_MSG or FI_TAGGED flag.
-
-*FI_RMA*
-: Indicates that an RMA operation completed. This flag may be combined
- with an FI_READ, FI_WRITE, FI_REMOTE_READ, or FI_REMOTE_WRITE flag.
-
-*FI_ATOMIC*
-: Indicates that an atomic operation completed. This flag may be combined
- with an FI_READ, FI_WRITE, FI_REMOTE_READ, or FI_REMOTE_WRITE flag.
-
-*FI_MSG*
-: Indicates that a message-based operation completed. This flag may be
- combined with an FI_SEND or FI_RECV flag.
-
-*FI_TAGGED*
-: Indicates that a tagged message operation completed. This flag may be
- combined with an FI_SEND or FI_RECV flag.
-
-*FI_READ*
-: Indicates that a locally initiated RMA or atomic read operation has
- completed. This flag may be combined with an FI_RMA or FI_ATOMIC flag.
-
-*FI_WRITE*
-: Indicates that a locally initiated RMA or atomic write operation has
- completed. This flag may be combined with an FI_RMA or FI_ATOMIC flag.
-
-*FI_REMOTE_READ*
-: Indicates that a remotely initiated RMA or atomic read operation has
- completed. This flag may be combined with an FI_RMA or FI_ATOMIC flag.
-
-*FI_REMOTE_WRITE*
-: Indicates that a remotely initiated RMA or atomic read operation has
- completed. This flag may be combined with an FI_RMA or FI_ATOMIC flag.
-
-*FI_REMOTE_CQ_DATA*
-: This indicates that remote CQ data is available as part of the
- completion.
-
-*FI_MULTI_RECV*
-: This flag applies to receive buffers that were posted with the
- FI_MULTI_RECV flag set. This completion flag indicates that the
- original receive buffer referenced by the completion has been
- consumed and was released by the provider. Providers may set
- this flag on the last message that is received into the multi-
- recv buffer, or may generate a separate completion that indicates
- that the buffer has been freed.
-
- Applications can distinguish between these two cases by examining
- the completion entry flags field. If additional flags, such as
- FI_RECV, are set, the completion is associated with a received message. In
- this case, the buf field will reference the location where the received
- message was placed into the multi-recv buffer. Other fields in the
- completion entry will be determined based on the received message.
- If other flag bits are zero, the provider is reporting that the multi-recv
- buffer has been freed, and the completion entry is not associated
- with a received message.
-
-# RETURN VALUES
-
-fi_cq_open / fi_cq_signal
-: Returns 0 on success. On error, a negative value corresponding to
- fabric errno is returned.
-
-fi_cq_read / fi_cq_readfrom / fi_cq_readerr
-fi_cq_sread / fi_cq_sreadfrom
-: On success, returns the number of completion events retrieved from
- the completion queue. On error, a negative value corresponding to
- fabric errno is returned. If no completions are available to
- return from the CQ, -FI_EAGAIN will be returned.
-
-fi_cq_strerror
-: Returns a character string interpretation of the provider specific
- error returned with a completion.
-
-Fabric errno values are defined in
-`rdma/fi_errno.h`.
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_eq`(3)](fi_eq.3.html),
-[`fi_cntr`(3)](fi_cntr.3.html),
-[`fi_poll`(3)](fi_poll.3.html)
diff --git a/man/fi_direct.7.md b/man/fi_direct.7.md
deleted file mode 100644
index d76d521..0000000
--- a/man/fi_direct.7.md
+++ /dev/null
@@ -1,75 +0,0 @@
----
-layout: page
-title: fi_direct(7)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-Direct fabric provider access
-
-# SYNOPSIS
-
-```c
--DFABRIC_DIRECT
-
-#define FABRIC_DIRECT
-```
-
-Fabric direct provides a mechanism for applications to compile against
-a specific fabric providers without going through the libfabric
-framework or function vector tables. This allows for extreme
-optimization via function inlining at the cost of supporting multiple
-providers or different versions of the same provider.
-
-# DESCRIPTION
-
-The use of fabric direct is intended only for applications that
-require the absolute minimum software latency, and are willing to
-re-compile for specific fabric hardware. Providers that support
-fabric direct implement their own versions of the static inline calls
-which are define in the libfabric header files, define selected enum
-values, and provide defines for compile-time optimizations.
-Applications can then code against the standard libfabric calls, but
-link directly against the provider calls by defining FABRIC_DIRECT as
-part of their build.
-
-In general, the use of fabric direct does not require application
-source code changes, and, instead, is limited to the build process.
-
-Providers supporting fabric direct must install 'direct' versions of
-all libfabric header files. For convenience, the libfabric sources
-contain sample header files that may be modified by a provider. The
-'direct' header file names have 'fi_direct' as their prefix:
-fi_direct.h, fi_direct_endpoint.h, etc.
-
-Direct providers are prohibited from overriding or modifying existing
-data structures. However, provider specific extensions are still
-available. In addition to provider direct function calls to provider
-code, a fabric direct provider may define zero of more of the
-following capability definitions. Applications can check for these
-capabilities in order to optimize code paths at compile time, versus
-relying on run-time checks.
-
-# CAPABILITY DEFINITIONS
-
-In order that application code may be optimized during compile time,
-direct providers must provide definitions for various capabilities and
-modes, if those capabilities are supported. The following #define
-values may be used by an application to test for provider support of
-supported features.
-
-*FI_DIRECT_CONTEXT*
-: The provider sets FI_CONTEXT for fi_info:mode. See fi_getinfo
- for additional details.
-
-*FI_DIRECT_LOCAL_MR*
-: The provider sets FI_LOCAL_MR for fi_info:mode. See fi_getinfo
- for additional details.
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_domain`(3)](fi_domain.3.html)
diff --git a/man/fi_domain.3.md b/man/fi_domain.3.md
deleted file mode 100644
index 8402a88..0000000
--- a/man/fi_domain.3.md
+++ /dev/null
@@ -1,516 +0,0 @@
----
-layout: page
-title: fi_domain(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_domain \- Open a fabric access domain
-
-# SYNOPSIS
-
-```c
-#include <rdma/fabric.h>
-
-#include <rdma/fi_domain.h>
-
-int fi_domain(struct fid_fabric *fabric, struct fi_info *info,
- struct fid_domain **domain, void *context);
-
-int fi_close(struct fid *domain);
-
-int fi_domain_bind(struct fid_domain *domain, struct fid *eq,
- uint64_t flags);
-
-int fi_open_ops(struct fid *domain, const char *name, uint64_t flags,
- void **ops, void *context);
-```
-
-# ARGUMENTS
-
-*fabric*
-: Fabric domain
-
-*info*
-: Fabric information, including domain capabilities and attributes.
-
-*domain*
-: An opened access domain.
-
-*context*
-: User specified context associated with the domain. This context is
- returned as part of any asynchronous event associated with the
- domain.
-
-*eq*
-: Event queue for asynchronous operations initiated on the domain.
-
-*name*
-: Name associated with an interface.
-
-*ops*
-: Fabric interface operations.
-
-# DESCRIPTION
-
-An access domain typically refers to a physical or virtual NIC or
-hardware port; however, a domain may span across multiple hardware
-components for fail-over or data striping purposes. A domain defines
-the boundary for associating different resources together. Fabric
-resources belonging to the same domain may share resources.
-
-## fi_domain
-
-Opens a fabric access domain, also referred to as a resource domain.
-Fabric domains are identified by a name. The properties of the opened
-domain are specified using the info parameter.
-
-## fi_open_ops
-
-fi_open_ops is used to open provider specific interfaces. Provider
-interfaces may be used to access low-level resources and operations
-that are specific to the opened resource domain. The details of
-domain interfaces are outside the scope of this documentation.
-
-## fi_domain_bind
-
-Associates an event queue with the domain. An event queue bound to a
-domain will be the default EQ associated with asynchronous control
-events that occur on the domain or active endpoints allocated on a
-domain. This includes CM events. Endpoints may direct their control
-events to alternate EQs by binding directly with the EQ.
-
-Binding an event queue to a domain with the FI_REG_MR flag indicates
-that the provider should perform all memory registration operations
-asynchronously, with the completion reported through the event queue.
-If an event queue is not bound to the domain with the FI_REG_MR flag,
-then memory registration requests complete synchronously.
-
-See [`fi_av_bind`(3)](fi_av_bind.3.html),
-[`fi_ep_bind`(3)](fi_ep_bind.3.html),
-[`fi_mr_bind`(3)](fi_mr_bind.3.html),
-[`fi_pep_bind`(3)](fi_pep_bind.3.html), and
-[`fi_scalable_ep_bind`(3)](fi_scalable_ep_bind.3.html) for more
-information.
-
-## fi_close
-
-The fi_close call is used to release all resources associated with a domain or
-interface. All objects associated with the opened domain must be released
-prior to calling fi_close, otherwise the call will return -FI_EBUSY.
-
-# DOMAIN ATTRIBUTES
-
-The `fi_domain_attr` structure defines the set of attributes associated
-with a domain.
-
-```c
-struct fi_domain_attr {
- struct fid_domain *domain;
- char *name;
- enum fi_threading threading;
- enum fi_progress control_progress;
- enum fi_progress data_progress;
- enum fi_resource_mgmt resource_mgmt;
- enum fi_av_type av_type;
- enum fi_mr_mode mr_mode;
- size_t mr_key_size;
- size_t cq_data_size;
- size_t cq_cnt;
- size_t ep_cnt;
- size_t tx_ctx_cnt;
- size_t rx_ctx_cnt;
- size_t max_ep_tx_ctx;
- size_t max_ep_rx_ctx;
- size_t max_ep_stx_ctx;
- size_t max_ep_srx_ctx;
-};
-```
-
-## domain
-
-On input to fi_getinfo, a user may set this to an opened domain
-instance to restrict output to the given domain. On output from
-fi_getinfo, if no domain was specified, but the user has an opened
-instance of the named domain, this will reference the first opened
-instance. If no instance has been opened, this field will be NULL.
-
-## Name
-
-The name of the access domain.
-
-## Multi-threading Support (threading)
-
-The threading model specifies the level of serialization required of
-an application when using the libfabric data transfer interfaces.
-Control interfaces are always considered thread safe, and may be
-accessed by multiple threads. Applications which can guarantee
-serialization in their access of provider allocated resources and
-interfaces enables a provider to eliminate lower-level locks.
-
-*FI_THREAD_UNSPEC*
-: This value indicates that no threading model has been defined. It
- may be used on input hints to the fi_getinfo call. When specified,
- providers will return a threading model that allows for the greatest
- level of parallelism.
-
-*FI_THREAD_SAFE*
-: A thread safe serialization model allows a multi-threaded
- application to access any allocated resources through any interface
- without restriction. All providers are required to support
- FI_THREAD_SAFE.
-
-*FI_THREAD_FID*
-: A fabric descriptor (FID) serialization model requires applications
- to serialize access to individual fabric resources associated with
- data transfer operations and completions. Multiple threads must
- be serialized when accessing the same endpoint, transmit context,
- receive context, completion queue, counter, wait set, or poll set.
- Serialization is required only by threads accessing the same object.
-
- For example, one thread may be initiating a data transfer on an
- endpoint, while another thread reads from a completion queue
- associated with the endpoint.
-
- Serialization to endpoint access is only required when accessing
- the same endpoint data flow. Multiple threads may initiate transfers
- on different transmit contexts of the same endpoint without serializing,
- and no serialization is required between the submission of data
- transmit requests and data receive operations.
-
- In general, FI_THREAD_FID allows the provider to be implemented
- without needing internal locking when handling data transfers.
- Conceptually, FI_THREAD_FID maps well to providers that implement
- fabric services in hardware and provide separate command queues to
- different data flows.
-
-*FI_THREAD_ENDPOINT*
-: The endpoint threading model is similar to FI_THREAD_FID, but with
- the added restriction that serialization is required when accessing
- the same endpoint, even if multiple transmit and receive contexts are
- used. Conceptually, FI_THREAD_ENDPOINT maps well to providers that
- implement fabric services in hardware but use a single command
- queue to access different data flows.
-
-*FI_THREAD_COMPLETION*
-: The completion threading model is intended for providers that make use
- of manual progress. Applications must serialize access to all objects
- that are associated through the use of having a shared completion
- structure. This includes endpoint, completion queue, counter, wait set,
- and poll set objects.
-
- For example, threads must serialize access to an endpoint and its
- bound completion queue(s) and/or counters. Access to endpoints that
- share the same completion queue must also be serialized.
-
- The use of FI_THREAD_COMPLETION can increase parallelism over
- FI_THREAD_SAFE, but requires the use of isolated resources.
-
-*FI_THREAD_DOMAIN*
-: A domain serialization model requires applications to serialize
- access to all objects belonging to a domain.
-
-## Progress Models (control_progress / data_progress)
-
-Progress is the ability of the underlying implementation to complete
-processing of an asynchronous request. In many cases, the processing
-of an asynchronous request requires the use of the host processor.
-For example, a received message may need to be matched with the
-correct buffer, or a timed out request may need to be retransmitted.
-For performance reasons, it may be undesirable for the provider to
-allocate a thread for this purpose, which will compete with the
-application threads.
-
-Control progress indicates the method that the provider uses to make
-progress on asynchronous control operations. Control operations are
-functions which do not directly involve the transfer of application
-data between endpoints. They include address vector, memory
-registration, and connection management routines.
-
-Data progress indicates the method that the provider uses to make
-progress on data transfer operations. This includes message queue,
-RMA, tagged messaging, and atomic operations, along with their
-completion processing.
-
-To balance between performance and ease of use, two progress models
-are defined.
-
-*FI_PROGRESS_UNSPEC*
-: This value indicates that no progress model has been defined. It
- may be used on input hints to the fi_getinfo call.
-
-*FI_PROGRESS_AUTO*
-: This progress model indicates that the provider will make forward
- progress on an asynchronous operation without further intervention
- by the application. When FI_PROGRESS_AUTO is provided as output to
- fi_getinfo in the absence of any progress hints, it often indicates
- that the desired functionality is implemented by the provider
- hardware or is a standard service of the operating system.
-
- All providers are required to support FI_PROGRESS_AUTO. However, if
- a provider does not natively support automatic progress, forcing the
- use of FI_PROGRESS_AUTO may result in threads being allocated below
- the fabric interfaces.
-
-*FI_PROGRESS_MANUAL*
-: This progress model indicates that the provider requires the use of
- an application thread to complete an asynchronous request. When
- manual progress is set, the provider will attempt to advance an
- asynchronous operation forward when the application attempts to
- wait on or read an event queue, completion queue, or counter
- where the completed operation will be reported. Progress also
- occurs when the application processes a poll or wait set that
- has been associated with the event or completion queue.
-
- Only wait operations defined by the fabric interface will result in
- an operation progressing. Operating system or external wait
- functions, such as select, poll, or pthread routines, cannot.
-
-## Resource Management (resource_mgmt)
-
-Resource management (RM) is provider and protocol support to protect
-against overrunning local and remote resources. This includes
-local and remote transmit contexts, receive contexts, completion
-queues, and source and target data buffers.
-
-When enabled, applications are given some level of protection against
-overrunning provider queues and local and remote data buffers. Such
-support may be built directly into the hardware and/or network
-protocol, but may also require that checks be enabled in the provider
-software. By disabling resource management, an application assumes
-all responsibility for preventing queue and buffer overruns, but doing
-so may allow a provider to eliminate internal synchronization calls,
-such as atomic variables or locks.
-
-It should be noted that even if resource management is disabled, the
-provider implementation and protocol may still provide some level of
-protection against overruns. However, such protection is not guaranteed.
-The following values for resource management are defined.
-
-*FI_RM_UNSPEC*
-: This value indicates that no resource management model has been defined.
- It may be used on input hints to the fi_getinfo call.
-
-*FI_RM_DISABLED*
-: The provider is free to select an implementation and protocol that does
- not protect against resource overruns. The application is responsible
- for resource protection.
-
-*FI_RM_ENABLED*
-: Resource management is enabled for this provider domain.
-
-The behavior of the various resource management options depends on whether
-the endpoint is reliable or unreliable, as well as provider and protocol
-specific implementation details, as shown in the following tables.
-
-| Resource | Unrel EP-RM Disabled| Unrel EP-RM Enabled | Rel EP-RM Disabled | Rel EP-RM Enabled |
-|:--------:|:-------------------:|:-------------------:|:------------------:|:-----------------:|
-| Tx | error | EAGAIN | error | EAGAIN |
-| Rx | error | EAGAIN | error | EAGAIN |
-| Tx CQ | error | EAGAIN | error | EAGAIN |
-| Rx CQ | error | EAGAIN or drop | error | EAGAIN or retry |
-| Unmatched Recv | buffered or drop | buffered or drop | buffered or error | buffered or retry |
-| Recv Overrun | truncate or drop | truncate or drop | truncate or error | truncate or error |
-| Unmatched RMA | not applicable | not applicable | error | error |
-| RMA Overrun | not applicable | not applicable | error | error |
-
-The resource column indicates the resource being accessed by a data
-transfer operation. Tx refers to the transmit context when a data
-transfer operation posted. Rx refers to the receive context when
-receive data buffers are posted. When RM is enabled, the
-provider will ensure that space is available to accept the operation.
-If space is not available, the operation will fail with -FI_EAGAIN.
-If resource management is disabled, the application is responsible for
-ensuring that there is space available before attempting to queue an
-operation.
-
-Tx CQ and Rx CQ refer to the completion queues associated with the
-transmit and receive contexts, respectively. When RM is disabled,
-applications must take care to ensure that completion queues do not
-get overrun. This can be accomplished by sizing the CQs appropriately
-or by deferring the posting of a data transfer operation unless CQ space
-is available to store its completion. When RM is enabled, providers
-may use different mechanisms to prevent CQ overruns. This includes
-failing (returning -FI_EAGAIN) the posting of operations that could
-result in CQ overruns, dropping received messages, or forcing requests
-to be retried.
-
-Unmatched receives and receive overruns deal with the processing of
-messages that consume a receive buffers. Unmatched receives references
-incoming messages that are received by an endpoint, but do not have an
-application data buffer to consume. No buffers may be available at the
-receive side, or buffers may available, but restricted from accepting
-the received message (such as being associated with different tags).
-Unmatched receives may be handled by protocol flow control, resulting
-in the message being retried. For unreliable endpoints, unmatched
-messages are usually dropped, unless the provider can internally buffer
-the data. An error will usually occur on a reliable endpoint if received
-data cannot be placed if RM is disabled, or the data cannot be received
-with RM enabled after retries have been exhausted.
-
-In some cases, buffering on the receive side may be available, but
-insufficient space may have been provided to receive the full message
-that was sent. This is considered an error, however, rather than
-failing the operation, a provider may instead truncate the message and
-report the truncation to the app.
-
-Unmatched RMA and RMA overruns deal with the processing of RMA and
-atomic operations that access registered memory buffers directly.
-RMA operations are not defined for unreliable endpoints. For reliable
-endpoints, unmatched RMA and RMA overruns are both treated as errors.
-
-When a resource management error occurs on an endpoint, the endpoint is
-transitioned into a disabled state. Any operations which have not
-already completed will fail and be discarded. For unconnected endpoints,
-the endpoint must be re-enabled before it will accept new data transfer
-operations. For connected endpoints, the connection is torn down and
-must be re-established.
-
-## AV Type (av_type)
-
-Specifies the type of address vectors that are usable with this domain.
-For additional details on AV type, see [`fi_av`(3)](fi_av.3.html).
-The following values may be specified.
-
-*FI_AV_UNSPEC*
-: Any address vector format is requested and supported.
-
-*FI_AV_MAP*
-: Only address vectors of type AV map are requested or supported.
-
-*FI_AV_TABLE*
-: Only address vectors of type AV index are requested or supported.
-
-Address vectors are only used by connectionless endpoints. Applications
-that require the use of a specific type of address vector should set the
-domain attribute av_type to the necessary value when calling fi_getinfo.
-The value FI_AV_UNSPEC may be used to indicate that the provider can support
-either address vector format. In this case, a provider may return
-FI_AV_UNSPEC to indicate that either format is supportable, or may return
-another AV type to indicate the optimal AV type supported by this domain.
-
-## Memory Registration Mode (mr_mode)
-
-Specifies the method of memory registration that is used with this domain.
-For additional details on MR mode, see [`fi_mr`(3)](fi_mr.3.html).
-The following values may be specified.
-
-*FI_MR_UNSPEC*
-: Any memory registration mode is requested and supported.
-
-*FI_MR_BASIC*
-: Only basic memory registration operations are requested or supported.
-
-*FI_MR_SCALABLE*
-: Only scalable memory registration operations are requested or supported.
-
-Buffers used in data transfer operations may require notifying the provider
-of their use before a data transfer can occur. The mr_mode field indicates
-the type of memory registration that is required, and when registration is
-necessary. Applications that require the use of a specific registration mode
-should set the domain attribute mr_mode to the necessary value when calling
-fi_getinfo. The value FI_MR_UNSPEC may be used to indicate support for any
-registration mode.
-
-## MR Key Size (mr_key_size)
-
-Size of the memory region remote access key, in bytes. Applications
-that request their own MR key must select a value within the range
-specified by this value.
-
-## CQ Data Size (cq_data_size)
-
-Applications may include a small message with a data transfer that
-is placed directly into a remote completion queue as part of a completion
-event. This is referred to as remote CQ data (sometimes referred to
-as immediate data). This field indicates the number of bytes that
-the provider supports for remote CQ data. If supported (non-zero
-value is returned), the minimum size of remote CQ data must be
-at least 4-bytes.
-
-## Completion Queue Count (cq_cnt)
-
-The optimal number of completion queues supported by the domain, relative
-to any specified or default CQ attributes. The cq_cnt value may be a
-fixed value of the maximum number of CQs supported by the
-underlying provider, or may be a dynamic value, based on the default
-attributes of an allocated CQ, such as the CQ size and data format.
-
-## Endpoint Count (ep_cnt)
-
-The total number of endpoints supported by the domain, relative to any
-specified or default endpoint attributes. The ep_cnt value may be a
-fixed value of the maximum number of endpoints supported by the
-underlying provider, or may be a dynamic value, based on the default
-attributes of an allocated endpoint, such as the endpoint capabilities
-and size. The endpoint count is the number of addressable endpoints
-supported by the provider.
-
-## Transmit Context Count (tx_ctx_cnt)
-
-The number of outbound command queues optimally supported by the
-provider. For a low-level provider, this represents the number of
-command queues to the hardware and/or the number of parallel transmit
-engines effectively supported by the hardware and caches.
-Applications which allocate more transmit contexts than this value
-will end up sharing underlying resources. By default, there is a
-single transmit context associated with each endpoint, but in an
-advanced usage model, an endpoint may be configured with multiple
-transmit contexts.
-
-## Receive Context Count (rx_ctx_cnt)
-
-The number of inbound processing queues optimally supported by the
-provider. For a low-level provider, this represents the number
-hardware queues that can be effectively utilized for processing
-incoming packets. Applications which allocate more receive contexts
-than this value will end up sharing underlying resources. By default,
-a single receive context is associated with each endpoint, but in an
-advanced usage model, an endpoint may be configured with multiple
-receive contexts.
-
-## Maximum Endpoint Transmit Context (max_ep_tx_ctx)
-
-The maximum number of transmit contexts that may be associated with an
-endpoint.
-
-## Maximum Endpoint Receive Context (max_ep_rx_ctx)
-
-The maximum number of receive contexts that may be associated with an
-endpoint.
-
-## Maximum Sharing of Transmit Context (max_ep_stx_ctx)
-
-The maximum number of endpoints that may be associated with a
-shared transmit context.
-
-## Maximum Sharing of Receive Context (max_ep_srx_ctx)
-
-The maximum number of endpoints that may be associated with a
-shared receive context.
-
-# RETURN VALUE
-
-Returns 0 on success. On error, a negative value corresponding to fabric
-errno is returned. Fabric errno values are defined in
-`rdma/fi_errno.h`.
-
-# NOTES
-
-Users should call fi_close to release all resources allocated to the
-fabric domain.
-
-The following fabric resources are associated with access domains:
-active endpoints, memory regions, completion event queues, and address
-vectors.
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_av`(3)](fi_av.3.html),
-[`fi_ep`(3)](fi_ep.3.html),
-[`fi_eq`(3)](fi_eq.3.html),
-[`fi_mr`(3)](fi_mr.3.html)
diff --git a/man/fi_endpoint.3.md b/man/fi_endpoint.3.md
deleted file mode 100644
index 2693909..0000000
--- a/man/fi_endpoint.3.md
+++ /dev/null
@@ -1,1226 +0,0 @@
----
-layout: page
-title: fi_endpoint(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_endpoint \- Fabric endpoint operations
-
-fi_endpoint / fi_scalable_ep / fi_passive_ep / fi_close
-: Allocate or close an endpoint.
-
-fi_ep_bind
-: Associate an endpoint with an event queue, completion queue,
- counter, address vector, or memory region
-
-fi_scalable_ep_bind
-: Associate a scalable endpoint with an address vector
-
-fi_pep_bind
-: Associate a passive endpoint with an event queue
-
-fi_enable
-: Transitions an endpoint into an active state.
-
-fi_cancel
-: Cancel a pending asynchronous data transfer
-
-fi_alias
-: Create an alias to the endpoint
-
-fi_control
-: Control endpoint operation.
-
-fi_getopt / fi_setopt
-: Get or set endpoint options.
-
-fi_rx_context / fi_tx_context / fi_srx_context / fi_stx_context
-: Open a transmit or receive context.
-
-fi_rx_size_left / fi_tx_size_left
-: Query the lower bound on how many RX/TX operations may be posted without
- an operation returning -FI_EAGAIN.
-
-# SYNOPSIS
-
-```c
-#include <rdma/fabric.h>
-
-#include <rdma/fi_endpoint.h>
-
-int fi_endpoint(struct fid_domain *domain, struct fi_info *info,
- struct fid_ep **ep, void *context);
-
-int fi_scalable_ep(struct fid_domain *domain, struct fi_info *info,
- struct fid_ep **sep, void *context);
-
-int fi_passive_ep(struct fi_fabric *fabric, struct fi_info *info,
- struct fid_pep **pep, void *context);
-
-int fi_tx_context(struct fid_ep *sep, int index,
- struct fi_tx_attr *attr, struct fid_ep **tx_ep,
- void *context);
-
-int fi_rx_context(struct fid_ep *sep, int index,
- struct fi_rx_attr *attr, struct fid_ep **rx_ep,
- void *context);
-
-int fi_stx_context(struct fid_domain *domain,
- struct fi_tx_attr *attr, struct fid_stx **stx,
- void *context);
-
-int fi_srx_context(struct fid_domain *domain,
- struct fi_rx_attr *attr, struct fid_ep **rx_ep,
- void *context);
-
-int fi_close(struct fid *ep);
-
-int fi_ep_bind(struct fid_ep *ep, struct fid *fid, uint64_t flags);
-
-int fi_scalable_ep_bind(struct fid_ep *sep, struct fid *fid, uint64_t flags);
-
-int fi_pep_bind(struct fid_pep *pep, struct fid *fid, uint64_t flags);
-
-int fi_enable(struct fid_ep *ep);
-
-int fi_cancel(struct fid_ep *ep, void *context);
-
-int fi_alias(struct fid_ep *ep, fid_t *alias_ep, uint64_t flags);
-
-int fi_control(struct fid *ep, int command, void *arg);
-
-int fi_getopt(struct fid *ep, int level, int optname,
- void *optval, size_t *optlen);
-
-int fi_setopt(struct fid *ep, int level, int optname,
- const void *optval, size_t optlen);
-
-ssize_t fi_rx_size_left(struct fid_ep *ep);
-
-ssize_t fi_tx_size_left(struct fid_ep *ep);
-```
-
-# ARGUMENTS
-
-*fid*
-: On creation, specifies a fabric or access domain. On bind,
- identifies the event queue, completion queue, counter, or address vector to
- bind to the endpoint. In other cases, it's a fabric identifier of an
- associated resource.
-
-*info*
-: Details about the fabric interface endpoint to be opened, obtained
- from fi_getinfo.
-
-*ep*
-: A fabric endpoint.
-
-*sep*
-: A scalable fabric endpoint.
-
-*pep*
-: A passive fabric endpoint.
-
-*context*
-: Context associated with the endpoint or asynchronous operation.
-
-*index*
-: Index to retrieve a specific transmit/receive context.
-
-*attr*
-: Transmit or receive context attributes.
-
-*flags*
-: Additional flags to apply to the operation.
-
-*command*
-: Command of control operation to perform on endpoint.
-
-*arg*
-: Optional control argument.
-
-*level*
-: Protocol level at which the desired option resides.
-
-*optname*
-: The protocol option to read or set.
-
-*optval*
-: The option value that was read or to set.
-
-*optlen*
-: The size of the optval buffer.
-
-# DESCRIPTION
-
-Endpoints are transport level communication portals. There are two
-types of endpoints: active and passive. Passive endpoints belong to a
-fabric domain and are most often used to listen for incoming connection
-requests. However, a passive endpoint may be used to reserve a fabric address
-that can be granted to an active endpoint. Active endpoints belong to access
-domains and can perform data transfers.
-
-Active endpoints may be connection-oriented or connectionless, and may
-provide data reliability. The data transfer interfaces -- messages (fi_msg),
-tagged messages (fi_tagged), RMA (fi_rma), and atomics (fi_atomic) --
-are associated with active endpoints. In basic configurations, an
-active endpoint has transmit and receive queues. In general, operations
-that generate traffic on the fabric are posted to the transmit queue.
-This includes all RMA and atomic operations, along with sent messages and
-sent tagged messages. Operations that post buffers for receiving incoming
-data are submitted to the receive queue.
-
-Active endpoints are created in the disabled state. They must
-transition into an enabled state before accepting data transfer
-operations, including posting of receive buffers. The fi_enable call
-is used to transition an endpoint into an active enabled state. The
-fi_connect and fi_accept calls will also transition an endpoint into
-the enabled state, if it is not already active.
-
-In order to transition an endpoint into an enabled state, it must be
-bound to one or more fabric resources. An endpoint that will generate
-asynchronous completions, either through data transfer operations or
-communication establishment events, must be bound to the appropriate
-completion queues or event queues before being enabled.
-
-Once an endpoint has been activated, it may be associated with memory
-regions and address vectors. Receive buffers may be posted to it, and
-calls may be made to connection establishment routines.
-Connectionless endpoints may also perform data transfers.
-
-The behavior of an endpoint may be adjusted by setting its control
-data and protocol options. This allows the underlying provider to
-redirect function calls to implementations optimized to meet the
-desired application behavior.
-
-If an endpoint experiences a critical error, it will transition back
-into a disabled state. Critical errors are reported through the
-event queue associated with the EP. In certain cases, a disabled endpoint may
-be re-enabled. The ability to transition back into an enabled
-state is provider specific and depends on the type of error that
-the endpoint experienced. When an endpoint is disabled as a result
-of a critical error, all pending operations are discarded.
-
-## fi_endpoint / fi_passive_ep / fi_scalable_ep
-
-fi_endpoint allocates a new active endpoint. fi_passive_ep allocates a
-new passive endpoint. fi_scalable_ep allocates a scalable endpoint.
-The properties and behavior of the endpoint are defined based on the
-provided struct fi_info. See fi_getinfo for additional details on
-fi_info. fi_info flags that control the operation of an endpoint are
-defined below. See section SCALABLE ENDPOINTS.
-
-If an active endpoint is allocated in order to accept a connection request,
-the fi_info parameter must be the same as the fi_info structure provided with
-the connection request (FI_CONNREQ) event.
-
-An active endpoint may acquire the properties of a passive endpoint by setting
-the fi_info handle field to the passive endpoint fabric descriptor. This is
-useful for applications that need to reserve the fabric address of an
-endpoint prior to knowing if the endpoint will be used on the active or passive
-side of a connection. For example, this feature is useful for simulating
-socket semantics. Once an active endpoint acquires the properties of a passive
-endpoint, the passive endpoint is no longer bound to any fabric resources and
-must no longer be used. The user is expected to close the passive endpoint
-after opening the active endpoint in order to free up any lingering resources
-that had been used.
-
-## fi_close
-
-Closes an endpoint and release all resources associated with it.
-
-When closing a scalable endpoint, there must be no opened transmit contexts, or
-receive contexts associated with the scalable endpoint. If resources are still
-associated with the scalable endpoint when attempting to close, the call will
-return -FI_EBUSY.
-
-Outstanding operations posted to the endpoint when fi_close is
-called will be discarded. Discarded operations will silently be dropped,
-with no completions reported. Additionally, a provider may discard previously
-completed operations from the associated completion queue(s). The
-behavior to discard completed operations is provider specific.
-
-## fi_ep_bind
-
-fi_ep_bind is used to associate an endpoint with hardware resources.
-The common use of fi_ep_bind is to direct asynchronous operations
-associated with an endpoint to a completion queue. An endpoint must
-be bound with CQs capable of reporting completions for any
-asynchronous operation initiated on the endpoint. This is true even
-for endpoints which are configured to suppress successful completions,
-in order that operations that complete in error may be reported to the
-user. For passive endpoints, this requires binding the endpoint with
-an EQ that supports the communication management (CM) domain.
-
-An active endpoint may direct asynchronous completions to different
-CQs, based on the type of operation. This is specified using
-fi_ep_bind flags. The following flags may be used separately or OR'ed
-together when binding an endpoint to a completion domain CQ.
-
-*FI_TRANSMIT*
-: Directs the completion of outbound data transfer requests to the
- specified completion queue. This includes send message, RMA, and
- atomic operations. The FI_SEND flag may be used interchangeably.
- This binding automatically includes FI_READ and FI_WRITE, if
- applicable to the endpoint.
-
-*FI_RECV*
-: Directs the notification of inbound data transfers to the specified
- completion queue. This includes received messages. This binding
- automatically includes FI_REMOTE_WRITE, if applicable to the
- endpoint.
-
-*FI_SELECTIVE_COMPLETION*
-: By default, data transfer operations generate completion entries
- into a completion queue after they have successfully completed.
- Applications can use this bind flag to selectively enable when
- completions are generated. If FI_SELECTIVE_COMPLETION is specified,
- data transfer operations will not generate entries for successful
- completions unless FI_COMPLETION is set as an operational flag for the
- given operation. FI_SELECTIVE_COMPLETION must be OR'ed with FI_SEND
- and/or FI_RECV flags.
-
- When FI_SELECTIVE_COMPLETION is set, the user must determine when a
- request that does NOT have FI_COMPLETION set has completed indirectly,
- usually based on the completion of a subsequent operation. Use of
- this flag may improve performance by allowing the provider to avoid
- writing a completion entry for every operation.
-
- Example: An application can selectively generate send completions by
- using the following general approach:
-
-```c
- fi_tx_attr::op_flags = 0; // default - no completion
- fi_ep_bind(ep, cq, FI_SEND | FI_SELECTIVE_COMPLETION);
- fi_send(ep, ...); // no completion
- fi_sendv(ep, ...); // no completion
- fi_sendmsg(ep, ..., FI_COMPLETION); // completion!
- fi_inject(ep, ...); // no completion
-```
-
- Example: An application can selectively disable send completions by
- modifying the operational flags:
-
-```c
- fi_tx_attr::op_flags = FI_COMPLETION; // default - completion
- fi_ep_bind(ep, cq, FI_SEND | FI_SELECTIVE_COMPLETION);
- fi_send(ep, ...); // completion
- fi_sendv(ep, ...); // completion
- fi_sendmsg(ep, ..., 0); // no completion!
- fi_inject(ep, ...); // no completion!
-```
-
- Example: Omitting FI_SELECTIVE_COMPLETION when binding will generate
- completions for all non-fi_inject calls:
-
-```c
- fi_tx_attr::op_flags = 0;
- fi_ep_bind(ep, cq, FI_SEND); // default - completion
- fi_send(ep, ...); // completion
- fi_sendv(ep, ...); // completion
- fi_sendmsg(ep, ..., 0); // completion!
- fi_sendmsg(ep, ..., FI_COMPLETION); // completion
- fi_sendmsg(ep, ..., FI_INJECT|FI_COMPLETION); // completion!
- fi_inject(ep, ...); // no completion!
-```
-
-An endpoint may also, or instead, be bound to a fabric counter. When
-binding an endpoint to a counter, the following flags may be specified.
-
-*FI_SEND*
-: Increments the specified counter whenever a successful message is
- transferred over the endpoint. Sent messages include both tagged
- and normal message operations.
-
-*FI_RECV*
-: Increments the specified counter whenever a successful message is
- received over the endpoint. Received messages include both tagged
- and normal message operations.
-
-*FI_READ*
-: Increments the specified counter whenever a successful RMA read or
- atomic fetch operation is initiated from the endpoint.
-
-*FI_WRITE*
-: Increments the specified counter whenever a successful RMA write or
- atomic operation is initiated from the endpoint.
-
-*FI_REMOTE_READ*
-: Increments the specified counter whenever a successful RMA read or
- atomic fetch operation is initiated from a remote endpoint that
- targets the given endpoint. Use of this flag requires that the
- endpoint be created using FI_RMA_EVENT.
-
-*FI_REMOTE_WRITE*
-: Increments the specified counter whenever a successful RMA write or
- atomic operation is initiated from a remote endpoint that targets
- the given endpoint. Use of this flag requires that the
- endpoint be created using FI_RMA_EVENT.
-
-Connectionless endpoints must be bound to a single address vector.
-If an endpoint is using a shared transmit and/or receive context, the
-shared contexts must be bound to the endpoint. CQs, counters, AV, and
-shared contexts must be bound to endpoints before they are enabled.
-
-## fi_scalable_ep_bind
-
-fi_scalable_ep_bind is used to associate a scalable endpoint with an
-address vector. See section on SCALABLE ENDPOINTS. A scalable
-endpoint has a single transport level address and can support multiple
-transmit and receive contexts. The transmit and receive contexts share
-the transport-level address. Address vectors that are bound to
-scalable endpoints are implicitly bound to any transmit or receive
-contexts created using the scalable endpoint.
-
-## fi_enable
-
-This call transitions the endpoint into an enabled state. An endpoint
-must be enabled before it may be used to perform data transfers.
-Enabling an endpoint typically results in hardware resources being
-assigned to it.
-
-Calling connect or accept on an endpoint will implicitly enable an
-endpoint if it has not already been enabled.
-
-fi_enable may also be used to re-enable an endpoint that has been
-disabled as a result of experiencing a critical error. Applications
-should check the return value from fi_enable to see if a disabled
-endpoint has successfully be re-enabled.
-
-## fi_cancel
-
-fi_cancel attempts to cancel an outstanding asynchronous operation.
-Canceling an operation causes the fabric provider to search for the
-operation and, if it is still pending, complete it as having been
-canceled. If multiple outstanding operations match the context
-parameter, only one will be canceled. In this case, the operation
-which is canceled is provider specific. The cancel operation is
-asynchronous, but will complete within a bounded period of time.
-
-## fi_alias
-
-This call creates an alias to the specified endpoint. Conceptually,
-an endpoint alias provides an alternate software path from the
-application to the underlying provider hardware. Applications
-configure an alias endpoint with data transfer flags, specified
-through the fi_alias call. Typically, the data transfer flags will be
-different than those assigned to the actual endpoint. The alias
-mechanism allows a single endpoint to have multiple optimized software
-interfaces. All allocated aliases must be closed for the underlying
-endpoint to be released.
-
-## fi_control
-
-The control operation is used to adjust the default behavior of an
-endpoint. It allows the underlying provider to redirect function
-calls to implementations optimized to meet the desired application
-behavior. As a result, calls to fi_ep_control must be serialized
-against all other calls to an endpoint.
-
-The base operation of an endpoint is selected during creation using
-struct fi_info. The following control commands and arguments may be
-assigned to an endpoint.
-
-**FI_GETOPSFLAG -- uint64_t *flags**
-: Used to retrieve the current value of flags associated with data
- transfer operations initiated on the endpoint. See below for a list
- of control flags.
-
-**FI_SETOPSFLAG -- uint64_t *flags**
-: Used to change the data transfer operation flags associated with an
- endpoint. The FI_READ, FI_WRITE, FI_SEND, FI_RECV flags indicate
- the type of data transfer that the flags should apply to, with other
- flags OR'ed in. Valid control flags are defined below.
-
-**FI_BACKLOG - int *value**
-: This option only applies to passive endpoints. It is used to set the
- connection request backlog for listening endpoints.
-
-## fi_getopt / fi_setopt
-
-Endpoint protocol operations may be retrieved using fi_getopt or set
-using fi_setopt. Applications specify the level that a desired option
-exists, identify the option, and provide input/output buffers to get
-or set the option. fi_setopt provides an application a way to adjust
-low-level protocol and implementation specific details of an endpoint.
-
-The following option levels and option names and parameters are defined.
-
-*FI_OPT_ENDPOINT*
-
-- *FI_OPT_MIN_MULTI_RECV - size_t*
-: Defines the minimum receive buffer space available when the receive
- buffer is automatically freed (see FI_MULTI_RECV). Modifying this
- value is only guaranteed to set the minimum buffer space needed on
- receives posted after the value has been changed. It is recommended
- that applications that want to override the default MIN_MULTI_RECV
- value set this option before enabling the corresponding endpoint.
-
-- *FI_OPT_CM_DATA_SIZE - size_t*
-: Defines the size of available space in CM messages for user-defined
- data. This value limits the amount of data that applications can
- exchange between peer endpoints using the fi_connect, fi_accept,
- and fi_reject operations. This option is read only.
-
-## fi_rx_size_left
-
-The fi_rx_size_left call returns a lower bound on the number of receive
-operations that may be posted to the given endpoint without that operation
-returning -FI_EAGAIN. Depending on the specific details of the subsequently
-posted receive operations (e.g., number of iov entries, which receive function
-is called, etc.), it may be possible to post more receive operations than
-originally indicated by fi_rx_size_left.
-
-## fi_tx_size_left
-
-The fi_tx_size_left call returns a lower bound on the number of transmit
-operations that may be posted to the given endpoint without that operation
-returning -FI_EAGAIN. Depending on the specific details of the subsequently
-posted transmit operations (e.g., number of iov entries, which transmit
-function is called, etc.), it may be possible to post more transmit operations
-than originally indicated by fi_tx_size_left.
-
-# ENDPOINT ATTRIBUTES
-
-The fi_ep_attr structure defines the set of attributes associated with
-an endpoint. Endpoint attributes may be further refined using the transmit
-and receive context attributes as shown below.
-
-{% highlight c %}
-struct fi_ep_attr {
- enum fi_ep_type type;
- uint32_t protocol;
- uint32_t protocol_version;
- size_t max_msg_size;
- size_t msg_prefix_size;
- size_t max_order_raw_size;
- size_t max_order_war_size;
- size_t max_order_waw_size;
- uint64_t mem_tag_format;
- size_t tx_ctx_cnt;
- size_t rx_ctx_cnt;
-};
-{% endhighlight %}
-
-## type - Endpoint Type
-
-If specified, indicates the type of fabric interface communication
-desired. Supported types are:
-
-*FI_EP_UNSPEC*
-: The type of endpoint is not specified. This is usually provided as
- input, with other attributes of the endpoint or the provider
- selecting the type.
-
-*FI_EP_MSG*
-: Provides a reliable, connection-oriented data transfer service with
- flow control that maintains message boundaries.
-
-*FI_EP_DGRAM*
-: Supports a connectionless, unreliable datagram communication.
- Message boundaries are maintained, but the maximum message size may
- be limited to the fabric MTU. Flow control is not guaranteed.
-
-*FI_EP_RDM*
-: Reliable datagram message. Provides a reliable, unconnected data
- transfer service with flow control that maintains message
- boundaries.
-
-## Protocol
-
-Specifies the low-level end to end protocol employed by the provider.
-A matching protocol must be used by communicating endpoints to ensure
-interoperability. The following protocol values are defined.
-Provider specific protocols are also allowed. Provider specific
-protocols will be indicated by having the upper bit of the
-protocol value set to one.
-
-*FI_PROTO_UNSPEC*
-: The protocol is not specified. This is usually provided as input,
- with other attributes of the socket or the provider selecting the
- actual protocol.
-
-*FI_PROTO_RDMA_CM_IB_RC*
-: The protocol runs over Infiniband reliable-connected queue pairs,
- using the RDMA CM protocol for connection establishment.
-
-*FI_PROTO_IWARP*
-: The protocol runs over the Internet wide area RDMA protocol transport.
-
-*FI_PROTO_IB_UD*
-: The protocol runs over Infiniband unreliable datagram queue pairs.
-
-*FI_PROTO_PSMX*
-: The protocol is based on an Intel proprietary protocol known as PSM,
- performance scaled messaging. PSMX is an extended version of the
- PSM protocol to support the libfabric interfaces.
-
-*FI_PROTO_UDP*
-: The protocol sends and receives UDP datagrams. For example, an
- endpoint using *FI_PROTO_UDP* will be able to communicate with a
- remote peer that is using Berkeley *SOCK_DGRAM* sockets using
- *IPPROTO_UDP*.
-
-*FI_PROTO_SOCK_TCP*
-: The protocol is layered over TCP packets.
-
-## protocol_version - Protocol Version
-
-Identifies which version of the protocol is employed by the provider.
-The protocol version allows providers to extend an existing protocol,
-by adding support for additional features or functionality for example,
-in a backward compatible manner. Providers that support different versions
-of the same protocol should inter-operate, but only when using the
-capabilities defined for the lesser version.
-
-## max_msg_size - Max Message Size
-
-Defines the maximum size for an application data transfer as a single
-operation.
-
-## msg_prefix_size - Message Prefix Size
-
-Specifies the size of any required message prefix buffer space. This
-field will be 0 unless the FI_MSG_PREFIX mode is enabled. If
-msg_prefix_size is > 0 the specified value will be a multiple of
-8-bytes.
-
-## Max RMA Ordered Size
-
-The maximum ordered size specifies the delivery order of transport
-data into target memory for RMA and atomic operations. Data ordering
-is separate, but dependent on message ordering (defined below). Data
-ordering is unspecified where message order is not defined.
-
-Data ordering refers to the access of target memory by subsequent
-operations. When back to back RMA read or write operations access the
-same registered memory location, data ordering indicates whether the
-second operation reads or writes the target memory after the first
-operation has completed. Because RMA ordering applies between two
-operations, and not within a single data transfer, ordering is defined
-per byte-addressable memory location. I.e. ordering specifies
-whether location X is accessed by the second operation after the first
-operation. Nothing is implied about the completion of the first
-operation before the second operation is initiated.
-
-In order to support large data transfers being broken into multiple packets
-and sent using multiple paths through the fabric, data ordering may be
-limited to transfers of a specific size or less. Providers specify when
-data ordering is maintained through the following values. Note that even
-if data ordering is not maintained, message ordering may be.
-
-*max_order_raw_size*
-: Read after write size. If set, an RMA or atomic read operation
- issued after an RMA or atomic write operation, both of which are
- smaller than the size, will be ordered. Where the target memory
- locations overlap, the RMA or atomic read operation will see the
- results of the previous RMA or atomic write.
-
-*max_order_war_size*
-: Write after read size. If set, an RMA or atomic write operation
- issued after an RMA or atomic read operation, both of which are
- smaller than the size, will be ordered. The RMA or atomic read
- operation will see the initial value of the target memory location
- before a subsequent RMA or atomic write updates the value.
-
-*max_order_waw_size*
-: Write after write size. If set, an RMA or atomic write operation
- issued after an RMA or atomic write operation, both of which are
- smaller than the size, will be ordered. The target memory location
- will reflect the results of the second RMA or atomic write.
-
-An order size value of 0 indicates that ordering is not guaranteed.
-A value of -1 guarantees ordering for any data size.
-
-## mem_tag_format - Memory Tag Format
-
-The memory tag format is a bit array used to convey the number of
-tagged bits supported by a provider. Additionally, it may be used to
-divide the bit array into separate fields. The mem_tag_format
-optionally begins with a series of bits set to 0, to signify bits
-which are ignored by the provider. Following the initial prefix of
-ignored bits, the array will consist of alternating groups of bits set
-to all 1's or all 0's. Each group of bits corresponds to a tagged
-field. The implication of defining a tagged field is that when a mask
-is applied to the tagged bit array, all bits belonging to a single
-field will either be set to 1 or 0, collectively.
-
-For example, a mem_tag_format of 0x30FF indicates support for 14
-tagged bits, separated into 3 fields. The first field consists of
-2-bits, the second field 4-bits, and the final field 8-bits. Valid
-masks for such a tagged field would be a bitwise OR'ing of zero or
-more of the following values: 0x3000, 0x0F00, and 0x00FF.
-
-By identifying fields within a tag, a provider may be able to optimize
-their search routines. An application which requests tag fields must
-provide tag masks that either set all mask bits corresponding to a
-field to all 0 or all 1. When negotiating tag fields, an application
-can request a specific number of fields of a given size. A provider
-must return a tag format that supports the requested number of fields,
-with each field being at least the size requested, or fail the
-request. A provider may increase the size of the fields. When reporting
-completions (see FI_CQ_FORMAT_TAGGED), the provider must provide the
-exact value of the recieved tag, clearing out any unsupported tag bits.
-
-It is recommended that field sizes be ordered from smallest to
-largest. A generic, unstructured tag and mask can be achieved by
-requesting a bit array consisting of alternating 1's and 0's.
-
-## tx_ctx_cnt - Transmit Context Count
-
-Number of transmit contexts to associate with the endpoint. If not
-specified (0), 1 context will be assigned if the endpoint supports
-outbound transfers. Transmit contexts are independent transmit queues
-that may be separately configured. Each transmit context may be bound
-to a separate CQ, and no ordering is defined between contexts.
-Additionally, no synchronization is needed when accessing contexts in
-parallel.
-
-If the count is set to the value FI_SHARED_CONTEXT, the endpoint will
-be configured to use a shared transmit context, if supported by the
-provider. Providers that do not support shared transmit contexts will
-fail the request.
-
-See the scalable endpoint and shared contexts sections for additional
-details.
-
-## rx_ctx_cnt - Receive Context Count
-
-Number of receive contexts to associate with the endpoint. If not
-specified, 1 context will be assigned if the endpoint supports inbound
-transfers. Receive contexts are independent processing queues that
-may be separately configured. Each receive context may be bound to a
-separate CQ, and no ordering is defined between contexts.
-Additionally, no synchronization is needed when accessing contexts in
-parallel.
-
-If the count is set to the value FI_SHARED_CONTEXT, the endpoint will
-be configured to use a shared receive context, if supported by the
-provider. Providers that do not support shared receive contexts will
-fail the request.
-
-See the scalable endpoint and shared contexts sections for additional
-details.
-
-# TRANSMIT CONTEXT ATTRIBUTES
-
-Attributes specific to the transmit capabilities of an endpoint are
-specified using struct fi_tx_attr.
-
-{% highlight c %}
-struct fi_tx_attr {
- uint64_t caps;
- uint64_t mode;
- uint64_t op_flags;
- uint64_t msg_order;
- uint64_t comp_order;
- size_t inject_size;
- size_t size;
- size_t iov_limit;
- size_t rma_iov_limit;
-};
-{% endhighlight %}
-
-## caps - Capabilities
-
-The requested capabilities of the context. The capabilities must be
-a subset of those requested of the associated endpoint. See the
-CAPABILITIES section of fi_getinfo(3) for capability details. If
-the caps field is 0 on input to fi_getinfo(3), the caps value from the
-fi_info structure will be used.
-
-## mode
-
-The operational mode bits of the context. The mode bits will be a
-subset of those associated with the endpoint. See the MODE section
-of fi_getinfo(3) for details. A mode value of 0 will be ignored on
-input to fi_getinfo(3), with the mode value of the fi_info structure
-used instead. On return from fi_getinfo(3), the mode will be set
-only to those constraints specific to transmit operations.
-
-## op_flags - Default transmit operation flags
-
-Flags that control the operation of operations submitted against the
-context. Applicable flags are listed in the Operation Flags
-section.
-
-## msg_order - Message Ordering
-
-Message ordering refers to the order in which transport layer headers
-(as viewed by the application) are processed. Relaxed message order
-enables data transfers to be sent and received out of order, which may
-improve performance by utilizing multiple paths through the fabric
-from the initiating endpoint to a target endpoint. Message order
-applies only between a single source and destination endpoint pair.
-Ordering between different target endpoints is not defined.
-
-Message order is determined using a set of ordering bits. Each set
-bit indicates that ordering is maintained between data transfers of
-the specified type. Message order is defined for [read | write |
-send] operations submitted by an application after [read | write |
-send] operations.
-
-Message ordering only applies to the end to end transmission of transport
-headers. Message ordering is necessary, but does not guarantee, the order in
-which message data is sent or received by the transport layer. Message
-ordering requires matching ordering semantics on the receiving side of a data
-transfer operation in order to guarantee that ordering is met.
-
-*FI_ORDER_NONE*
-: No ordering is specified. This value may be used as input in order
- to obtain the default message order supported by the provider.
-
-*FI_ORDER_RAR*
-: Read after read. If set, RMA and atomic read operations are
- transmitted in the order submitted relative to other
- RMA and atomic read operations. If not set, RMA and atomic reads
- may be transmitted out of order from their submission.
-
-*FI_ORDER_RAW*
-: Read after write. If set, RMA and atomic read operations are
- transmitted in the order submitted relative to RMA and atomic write
- operations. If not set, RMA and atomic reads may be transmitted ahead
- of RMA and atomic writes.
-
-*FI_ORDER_RAS*
-: Read after send. If set, RMA and atomic read operations are
- transmitted in the order submitted relative to message send
- operations, including tagged sends. If not set, RMA and atomic
- reads may be transmitted ahead of sends.
-
-*FI_ORDER_WAR*
-: Write after read. If set, RMA and atomic write operations are
- transmitted in the order submitted relative to RMA and atomic read
- operations. If not set, RMA and atomic writes may be transmitted
- ahead of RMA and atomic reads.
-
-*FI_ORDER_WAW*
-: Write after write. If set, RMA and atomic write operations are
- transmitted in the order submitted relative to other RMA and atomic
- write operations. If not set, RMA and atomic writes may be
- transmitted out of order from their submission.
-
-*FI_ORDER_WAS*
-: Write after send. If set, RMA and atomic write operations are
- transmitted in the order submitted relative to message send
- operations, including tagged sends. If not set, RMA and atomic
- writes may be transmitted ahead of sends.
-
-*FI_ORDER_SAR*
-: Send after read. If set, message send operations, including tagged
- sends, are transmitted in order submitted relative to RMA and atomic
- read operations. If not set, message sends may be transmitted ahead
- of RMA and atomic reads.
-
-*FI_ORDER_SAW*
-: Send after write. If set, message send operations, including tagged
- sends, are transmitted in order submitted relative to RMA and atomic
- write operations. If not set, message sends may be transmitted ahead
- of RMA and atomic writes.
-
-*FI_ORDER_SAS*
-: Send after send. If set, message send operations, including tagged
- sends, are transmitted in the order submitted relative to other
- message send. If not set, message sends may be transmitted out of
- order from their submission.
-
-## comp_order - Completion Ordering
-
-Completion ordering refers to the order in which completed requests are
-written into the completion queue. Completion ordering is similar to
-message order. Relaxed completion order may enable faster reporting of
-completed transfers, allow acknowledgments to be sent over different
-fabric paths, and support more sophisticated retry mechanisms.
-This can result in lower-latency completions, particularly when
-using unconnected endpoints. Strict completion ordering may require
-that providers queue completed operations or limit available optimizations.
-
-For transmit requests, completion ordering depends on the endpoint
-communication type. For unreliable communication, completion ordering
-applies to all data transfer requests submitted to an endpoint.
-For reliable communication, completion ordering only applies to requests
-that target a single destination endpoint. Completion ordering of
-requests that target different endpoints over a reliable transport
-is not defined.
-
-Applications should specify the completion ordering that they support
-or require. Providers should return the completion order that they
-actually provide, with the constraint that the returned ordering is
-stricter than that specified by the application. Supported completion
-order values are:
-
-*FI_ORDER_NONE*
-: No ordering is defined for completed operations. Requests submitted
- to the transmit context may complete in any order.
-
-*FI_ORDER_STRICT*
-: Requests complete in the order in which they are submitted to the
- transmit context.
-
-## inject_size
-
-The requested inject operation size (see the FI_INJECT flag) that
-the context will support. This is the maximum size data transfer that
-can be associated with an inject operation (such as fi_inject) or may
-be used with the FI_INJECT data transfer flag.
-
-## size
-
-The size of the context. The size is specified as the minimum number
-of transmit operations that may be posted to the endpoint without the
-operation returning -FI_EAGAIN.
-
-## iov_limit
-
-This is the maximum number of IO vectors (scatter-gather elements)
-that a single posted operation may reference.
-
-## rma_iov_limit
-
-This is the maximum number of RMA IO vectors (scatter-gather elements)
-that an RMA or atomic operation may reference. The rma_iov_limit
-corresponds to the rma_iov_count values in RMA and atomic operations.
-See struct fi_msg_rma and struct fi_msg_atomic in fi_rma.3 and
-fi_atomic.3, for additional details. This limit applies to both the
-number of RMA IO vectors that may be specified when initiating an
-operation from the local endpoint, as well as the maximum number of
-IO vectors that may be carried in a single request from a remote endpoint.
-
-# RECEIVE CONTEXT ATTRIBUTES
-
-Attributes specific to the receive capabilities of an endpoint are
-specified using struct fi_rx_attr.
-
-{% highlight c %}
-struct fi_rx_attr {
- uint64_t caps;
- uint64_t mode;
- uint64_t op_flags;
- uint64_t msg_order;
- uint64_t comp_order;
- size_t total_buffered_recv;
- size_t size;
- size_t iov_limit;
-};
-{% endhighlight %}
-
-## caps - Capabilities
-
-The requested capabilities of the context. The capabilities must be
-a subset of those requested of the associated endpoint. See the
-CAPABILITIES section if fi_getinfo(3) for capability details. If
-the caps field is 0 on input to fi_getinfo(3), the caps value from the
-fi_info structure will be used.
-
-## mode
-
-The operational mode bits of the context. The mode bits will be a
-subset of those associated with the endpoint. See the MODE section
-of fi_getinfo(3) for details. A mode value of 0 will be ignored on
-input to fi_getinfo(3), with the mode value of the fi_info structure
-used instead. On return from fi_getinfo(3), the mode will be set
-only to those constraints specific to receive operations.
-
-## op_flags - Default receive operation flags
-
-Flags that control the operation of operations submitted against the
-context. Applicable flags are listed in the Operation Flags
-section.
-
-## msg_order - Message Ordering
-
-For a description of message ordering, see the msg_order field in
-the _Transmit Context Attribute_ section. Receive context message
-ordering defines the order in which received transport message headers
-are processed when received by an endpoint.
-
-The following ordering flags, as defined for transmit ordering, also
-apply to the processing of received operations: FI_ORDER_NONE,
-FI_ORDER_RAR, FI_ORDER_RAW, FI_ORDER_RAS, FI_ORDER_WAR, FI_ORDER_WAW,
-FI_ORDER_WAS, FI_ORDER_SAR, FI_ORDER_SAW, and FI_ORDER_SAS.
-
-## comp_order - Completion Ordering
-
-For a description of completion ordering, see the comp_order field in
-the _Transmit Context Attribute_ section.
-
-*FI_ORDER_NONE*
-: No ordering is defined for completed operations. Receive operations may
- complete in any order, regardless of their submission order.
-
-*FI_ORDER_STRICT*
-: Receive operations complete in the order in which they are processed by
- the receive context, based on the receive side msg_order attribute.
-
-*FI_ORDER_DATA*
-: When set, this bit indicates that received data is written into memory
- in order. Data ordering applies to memory accessed as part of a single
- operation and between operations if message ordering is guaranteed.
-
-## total_buffered_recv
-
-Defines the total available space allocated by the provider to buffer messages
-that are received for which there is no matching receive operation. If set to
-0, and the domain does not support FI_RM_ENABLED, any messages that arrive
-before a receive buffer has been posted are lost. When the domain supports
-FI_RM_ENABLED, the actual amount of buffering provided may exceed the value
-specified in total_buffered_recv.
-
-## size
-
-The size of the context. The size is specified as the minimum number
-of receive operations that may be posted to the endpoint without the
-operation returning -FI_EAGAIN.
-
-## iov_limit
-
-This is the maximum number of IO vectors (scatter-gather elements)
-that a single posted operating may reference.
-
-# SCALABLE ENDPOINTS
-
-A scalable endpoint is a communication portal that supports multiple
-transmit and receive contexts. Scalable endpoints are loosely modeled
-after the networking concept of transmit/receive side scaling, also
-known as multi-queue. Support for scalable endpoints is domain
-specific. Scalable endpoints may improve the performance of
-multi-threaded and parallel applications, by allowing threads to
-access independent transmit and receive queues. A scalable endpoint
-has a single transport level address, which can reduce the memory
-requirements needed to store remote addressing data, versus using
-standard endpoints. Scalable endpoints cannot be used directly for
-communication operations, and require the application to explicitly
-create transmit and receive contexts as described below.
-
-## fi_tx_context
-
-Transmit contexts are independent transmit queues. Ordering and
-synchronization between contexts are not defined. Conceptually a
-transmit context behaves similar to a send-only endpoint. A transmit
-context may be configured with relaxed capabilities, and has its own
-completion queue. The number of transmit contexts associated with an
-endpoint is specified during endpoint creation.
-
-The fi_tx_context call is used to retrieve a specific context,
-identified by an index. Providers may dynamically allocate contexts
-when fi_tx_context is called, or may statically create all contexts
-when fi_endpoint is invoked. By default, a transmit context inherits
-the properties of its associated endpoint. However, applications may
-request context specific attributes through the attr parameter.
-Support for per transmit context attributes is provider specific and
-not guaranteed. Providers will return the actual attributes assigned
-to the context through the attr parameter, if provided.
-
-## fi_rx_context
-
-Receive contexts are independent receive queues for receiving incoming
-data. Ordering and synchronization between contexts are not
-guaranteed. Conceptually a receive context behaves similar to a
-receive-only endpoint. A receive context may be configured with
-relaxed endpoint capabilities, and has its own completion queue. The
-number of receive contexts associated with an endpoint is specified
-during endpoint creation.
-
-Receive contexts are often associated with steering flows, that
-specify which incoming packets targeting a scalable endpoint to
-process. However, receive contexts may be targeted directly by the
-initiator, if supported by the underlying protocol. Such contexts are
-referred to as 'named'. Support for named contexts must be indicated
-by setting the caps FI_NAMED_RX_CTX capability when the corresponding
-endpoint is created. Support for named receive contexts is
-coordinated with address vectors. See fi_av(3) and fi_rx_addr(3).
-
-The fi_rx_context call is used to retrieve a specific context,
-identified by an index. Providers may dynamically allocate contexts
-when fi_rx_context is called, or may statically create all contexts
-when fi_endpoint is invoked. By default, a receive context inherits
-the properties of its associated endpoint. However, applications may
-request context specific attributes through the attr parameter.
-Support for per receive context attributes is provider specific and
-not guaranteed. Providers will return the actual attributes assigned
-to the context through the attr parameter, if provided.
-
-# SHARED CONTEXTS
-
-Shared contexts are transmit and receive contexts explicitly shared
-among one or more endpoints. A shareable context allows an application
-to use a single dedicated provider resource among multiple transport
-addressable endpoints. This can greatly reduce the resources needed
-to manage communication over multiple endpoints by multiplexing
-transmit and/or receive processing, with the potential cost of
-serializing access across multiple endpoints. Support for shareable
-contexts is domain specific.
-
-Conceptually, shareable transmit contexts are transmit queues that may be
-accessed by many endpoints. The use of a shared transmit context is
-mostly opaque to an application. Applications must allocate and bind
-shared transmit contexts to endpoints, but operations are posted
-directly to the endpoint. Shared transmit contexts are not associated
-with completion queues or counters. Completed operations are posted
-to the CQs bound to the endpoint. An endpoint may only
-be associated with a single shared transmit context.
-
-Unlike shared transmit contexts, applications interact directly with
-shared receive contexts. Users post receive buffers directly to a
-shared receive context, with the buffers usable by any endpoint bound
-to the shared receive context. Shared receive contexts are not
-associated with completion queues or counters. Completed receive
-operations are posted to the CQs bound to the endpoint. An endpoint
-may only be associated with a single receive context, and all
-connectionless endpoints associated with a shared receive context must
-also share the same address vector.
-
-Endpoints associated with a shared transmit context may use dedicated
-receive contexts, and vice-versa. Or an endpoint may use shared
-transmit and receive contexts. And there is no requirement that the
-same group of endpoints sharing a context of one type also share the
-context of an alternate type. Furthermore, an endpoint may use a
-shared context of one type, but a scalable set of contexts of the
-alternate type.
-
-## fi_stx_context
-
-This call is used to open a shareable transmit context. See
-fi_tx_context call under the SCALABLE ENDPOINTS section for details on
-the transit context attributes. The exception is that endpoints
-attached to a shared transmit context must use a subset of the
-transmit context attributes. This is opposite of the requirement for
-scalable endpoints.
-
-## fi_srx_context
-
-This allocates a shareable receive context. See fi_rx_context call
-under SCALABLE ENDPOINTS section for details on the receive context
-attributes. The exception is that endpoints attached to a shared
-receive context must use a subset of the receive context attributes.
-This is opposite of the requirement for scalable endpoints.
-
-# OPERATION FLAGS
-
-Operation flags are obtained by OR-ing the following flags together.
-Operation flags define the default flags applied to an endpoint's data
-transfer operations, where a flags parameter is not available. Data
-transfer operations that take flags as input override the op_flags
-value of an endpoint.
-
-*FI_INJECT*
-: Indicates that all outbound data buffers should be returned to the
- user's control immediately after a data transfer call returns, even
- if the operation is handled asynchronously. This may require that
- the provider copy the data into a local buffer and transfer out of
- that buffer. A provider may limit the total amount of send data
- that may be buffered and/or the size of a single send.
-
-*FI_MULTI_RECV*
-: Applies to posted receive operations. This flag allows the user to
- post a single buffer that will receive multiple incoming messages.
- Received messages will be packed into the receive buffer until the
- buffer has been consumed. Use of this flag may cause a single
- posted receive operation to generate multiple completions as
- messages are placed into the buffer. The placement of received data
- into the buffer may be subjected to provider specific alignment
- restrictions. The buffer will be returned to the application's
- control, and an *FI_MULTI_RECV* completion will be generated, when a
- message is received that cannot fit into the remaining free buffer
- space.
-
-*FI_COMPLETION*
-: Indicates that a completion entry should be generated for data
- transfer operations.
-
-*FI_INJECT_COMPLETE*
-: Indicates that a completion should be generated when the
- source buffer(s) may be reused. A completion guarantees that
- the buffers will not be read from again and the application may
- reclaim them. No other guarantees are made with respect to the
- state of the operation.
-
- Note: This flag is used to control when a completion entry is inserted
- into a completion queue. It does not apply to operations that do not
- generate a completion queue entry, such as the fi_inject operation.
-
-*FI_TRANSMIT_COMPLETE*
-: Indicates that a completion should be generated when the transmit
- operation has completed relative to the local provider. The exact
- behavior is dependent on the endpoint type.
-
- For reliable endpoints:
-
- Indicates that a completion should be generated when the operation has
- been delivered to the peer endpoint. A completion guarantees that the
- operation is no longer dependent on the fabric or local resources. The
- state of the operation at the peer endpoint is not defined.
-
- For unreliable endpoints:
-
- Indicates that a completion should be generated when the operation has
- been delivered to the fabric. A completion guarantees that the
- operation is no longer dependent on local resources. The state of the
- operation within the fabric is not defined.
-
-*FI_DELIVERY_COMPLETE*
-: Indicates that a completion should not be generated until an operation
- has been processed by the destination endpoint(s). A completion
- guarantees that the result of the operation is available.
-
- This completion mode applies only to reliable endpoints. For operations
- that return data to the initiator, such as RMA read or atomic-fetch,
- the source endpoint is also considered a destination endpoint. This is the
- default completion mode for such operations.
-
-# NOTES
-
-Users should call fi_close to release all resources allocated to the
-fabric endpoint.
-
-Endpoints allocated with the FI_CONTEXT mode set must typically
-provide struct fi_context as their per operation context parameter.
-(See fi_getinfo.3 for details.) However, when FI_SELECTIVE_COMPLETION is
-enabled to suppress completion entries, and an operation is initiated
-without FI_COMPLETION flag set, then the context parameter is ignored.
-An application does not need to pass in a valid struct fi_context into
-such data transfers.
-
-Operations that complete in error that are not associated with valid
-operational context will use the endpoint context in any error
-reporting structures.
-
-Although applications typically associate individual completions with
-either completion queues or counters, an endpoint can be attached to
-both a counter and completion queue. When combined with using
-selective completions, this allows an application to use counters to
-track successful completions, with a CQ used to report errors.
-Operations that complete with an error increment the error counter
-and generate a completion event. The generation of entries going to
-the CQ can then be controlled using FI_SELECTIVE_COMPLETION.
-
-# RETURN VALUES
-
-Returns 0 on success. On error, a negative value corresponding to
-fabric errno is returned. For fi_cancel, a return value of 0
-indicates that the cancel request was submitted for processing.
-
-Fabric errno values are defined in `rdma/fi_errno.h`.
-
-# ERRORS
-
-*-FI_EDOMAIN*
-: A resource domain was not bound to the endpoint or an attempt was
- made to bind multiple domains.
-
-*-FI_ENOCQ*
-: The endpoint has not been configured with necessary event queue.
-
-*-FI_EOPBADSTATE*
-: The endpoint's state does not permit the requested operation.
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_msg`(3)](fi_msg.3.html),
-[`fi_tagged`(3)](fi_tagged.3.html),
-[`fi_rma`(3)](fi_rma.3.html)
diff --git a/man/fi_eq.3.md b/man/fi_eq.3.md
deleted file mode 100644
index 9a86f9d..0000000
--- a/man/fi_eq.3.md
+++ /dev/null
@@ -1,433 +0,0 @@
----
-layout: page
-title: fi_eq(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_eq \- Event queue operations
-
-fi_eq_open / fi_close
-: Open/close an event queue
-
-fi_control
-: Control operation of EQ
-
-fi_eq_read / fi_eq_readerr
-: Read an event from an event queue
-
-fi_eq_write
-: Writes an event to an event queue
-
-fi_eq_sread
-: A synchronous (blocking) read of an event queue
-
-fi_eq_strerror
-: Converts provider specific error information into a printable string
-
-# SYNOPSIS
-
-```c
-#include <rdma/fi_domain.h>
-
-int fi_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr,
- struct fid_eq **eq, void *context);
-
-int fi_close(struct fid *eq);
-
-int fi_control(struct fid *eq, int command, void *arg);
-
-ssize_t fi_eq_read(struct fid_eq *eq, uint32_t *event,
- void *buf, size_t len, uint64_t flags);
-
-ssize_t fi_eq_readerr(struct fid_eq *eq, struct fi_eq_err_entry *buf,
- uint64_t flags);
-
-ssize_t fi_eq_write(struct fid_eq *eq, uint32_t event,
- const void *buf, size_t len, uint64_t flags);
-
-ssize_t fi_eq_sread(struct fid_eq *eq, uint32_t *event,
- void *buf, size_t len, int timeout, uint64_t flags);
-
-const char * fi_eq_strerror(struct fid_eq *eq, int prov_errno,
- const void *err_data, char *buf, size_t len);
-```
-
-# ARGUMENTS
-
-*fabric*
-: Opened fabric descriptor
-
-*eq*
-: Event queue
-
-*attr*
-: Event queue attributes
-
-*context*
-: User specified context associated with the event queue.
-
-*event*
-: Reported event
-
-*buf*
-: For read calls, the data buffer to write events into. For write
- calls, an event to insert into the event queue. For fi_eq_strerror,
- an optional buffer that receives printable error information.
-
-*len*
-: Length of data buffer
-
-*flags*
-: Additional flags to apply to the operation
-
-*command*
-: Command of control operation to perform on EQ.
-
-*arg*
-: Optional control argument
-
-*prov_errno*
-: Provider specific error value
-
-*err_data*
-: Provider specific error data related to a completion
-
-*timeout*
-: Timeout specified in milliseconds
-
-# DESCRIPTION
-
-Event queues are used to report events associated with control
-operations. They are associated with memory registration, address
-vectors, connection management, and fabric and domain level events.
-Reported events are either associated with a requested operation or
-affiliated with a call that registers for specific types of events,
-such as listening for connection requests.
-
-## fi_eq_open
-
-fi_eq_open allocates a new event queue.
-
-The properties and behavior of an event queue are defined by `struct
-fi_eq_attr`.
-
-```c
-struct fi_eq_attr {
- size_t size; /* # entries for EQ */
- uint64_t flags; /* operation flags */
- enum fi_wait_obj wait_obj; /* requested wait object */
- int signaling_vector; /* interrupt affinity */
- struct fid_wait *wait_set; /* optional wait set */
-};
-```
-
-*size*
-: Specifies the minimum size of an event queue.
-
-*flags*
-: Flags that control the configuration of the EQ.
-
-- *FI_WRITE*
-: Indicates that the application requires support for inserting user
- events into the EQ. If this flag is set, then the fi_eq_write
- operation must be supported by the provider. If the FI_WRITE flag
- is not set, then the application may not invoke fi_eq_write.
-
-*wait_obj*
-: EQ's may be associated with a specific wait object. Wait objects
- allow applications to block until the wait object is signaled,
- indicating that an event is available to be read. Users may use
- fi_control to retrieve the underlying wait object associated with an
- EQ, in order to use it in other system calls. The following values
- may be used to specify the type of wait object associated with an
- EQ:
-
-- *FI_WAIT_NONE*
-: Used to indicate that the user will not block (wait) for events on
- the EQ. When FI_WAIT_NONE is specified, the application may not
- call fi_eq_sread.
-
-- *FI_WAIT_UNSPEC*
-: Specifies that the user will only wait on the EQ using fabric
- interface calls, such as fi_eq_sread. In this case, the underlying
- provider may select the most appropriate or highest performing wait
- object available, including custom wait mechanisms. Applications
- that select FI_WAIT_UNSPEC are not guaranteed to retrieve the
- underlying wait object.
-
-- *FI_WAIT_SET*
-: Indicates that the event queue should use a wait set object to wait
- for events. If specified, the wait_set field must reference an
- existing wait set object.
-
-- *FI_WAIT_FD*
-: Indicates that the EQ should use a file descriptor as its wait
- mechanism. A file descriptor wait object must be usable in select,
- poll, and epoll routines. However, a provider may signal an FD wait
- object by marking it as readable or with an error.
-
-- *FI_WAIT_MUTEX_COND*
-: Specifies that the EQ should use a pthread mutex and cond variable
- as a wait object.
-
-*signaling_vector*
-: Indicates which processor core interrupts associated with the EQ
- should target.
-
-*wait_set*
-: If wait_obj is FI_WAIT_SET, this field references a wait object to
- which the event queue should attach. When an event is inserted into
- the event queue, the corresponding wait set will be signaled if all
- necessary conditions are met. The use of a wait_set enables an
- optimized method of waiting for events across multiple event queues.
- This field is ignored if wait_obj is not FI_WAIT_SET.
-
-## fi_close
-
-The fi_close call releases all resources associated with an event queue. Any
-events which remain on the EQ when it is closed are lost.
-
-The EQ must not be bound to any other objects prior to being closed, otherwise
-the call will return -FI_EBUSY.
-
-## fi_control
-
-The fi_control call is used to access provider or implementation
-specific details of the event queue. Access to the EQ should be
-serialized across all calls when fi_control is invoked, as it may
-redirect the implementation of EQ operations. The following control
-commands are usable with an EQ.
-
-*FI_GETWAIT (void \*\*)*
-: This command allows the user to retrieve the low-level wait object
- associated with the EQ. The format of the wait-object is specified
- during EQ creation, through the EQ attributes. The fi_control arg
- parameter should be an address where a pointer to the returned wait
- object will be written. This should be an 'int *' for FI_WAIT_FD,
- or 'struct fi_mutex_cond' for FI_WAIT_MUTEX_COND.
-
-```c
-struct fi_mutex_cond {
- pthread_mutex_t *mutex;
- pthread_cond_t *cond;
-};
-```
-
-## fi_eq_read
-
-The fi_eq_read operations performs a non-blocking read of event data
-from the EQ. The format of the event data is based on the type of
-event retrieved from the EQ, with all events starting with a struct
-fi_eq_entry header. At most one event will be returned per EQ read
-operation. The number of bytes successfully read from the EQ is
-returned from the read. The FI_PEEK flag may be used to indicate that
-event data should be read from the EQ without being consumed. A
-subsequent read without the FI_PEEK flag would then remove the event
-from the EQ.
-
-The following types of events may be reported to an EQ, along with
-information regarding the format associated with each event.
-
-*Asynchronous Control Operations*
-: Asynchronous control operations are basic requests that simply need
- to generate an event to indicate that they have completed. These
- include the following types of events: memory registration and address
- vector resolution.
-
- Control requests report their completion by inserting a `struct
- fi_eq_entry` into the EQ. The format of this structure is:
-
-```c
-struct fi_eq_entry {
- fid_t fid; /* fid associated with request */
- void *context; /* operation context */
- uint64_t data; /* completion-specific data */
-};
-```
-
- For the completion of basic asynchronous control operations, the
- returned event will indicate the operation that has completed, and
- the fid will reference the fabric descriptor associated with
- the event. For memory registration, this will be an FI_MR_COMPLETE
- event and the fid_mr; address resolution will reference an
- FI_AV_COMPLETE event and fid_av. The context field will be set
- to the context specified as part of the operation, if available,
- otherwise the context will be associated with the fabric descriptor.
- The data field will be set as described in the man page for the
- corresponding object type (e.g., see [`fi_av`(3)](fi_av.3.html) for
- a description of how asynchronous address vector insertions are
- completed).
-
-*Connection Notification*
-: Connection notifications are connection management notifications
- used to setup or tear down connections between endpoints. There are
- three connection notification events: FI_CONNREQ, FI_CONNECTED, and
- FI_SHUTDOWN. Connection notifications are reported using `struct
- fi_eq_cm_entry`:
-
-```c
-struct fi_eq_cm_entry {
- fid_t fid; /* fid associated with request */
- struct fi_info *info; /* endpoint information */
- uint8_t data[]; /* app connection data */
-};
-```
-
- A connection request (FI_CONNREQ) event indicates that
- a remote endpoint wishes to establish a new connection to a listening,
- or passive, endpoint. The fid is the passive endpoint.
- Information regarding the requested, active endpoint's
- capabilities and attributes are available from the info field. The
- application is responsible for freeing this structure by calling
- fi_freeinfo when it is no longer needed. The fi_info connreq field
- will reference the connection request associated with this event.
- To accept a connection, an endpoint must first be created by passing
- an fi_info structure referencing this connreq field to fi_endpoint().
- This endpoint is then passed to fi_accept() to complete the acceptance
- of the connection attempt.
- Creating the endpoint is most easily accomplished by
- passing the fi_info returned as part of the CM event into
- fi_endpoint(). If the connection is to be rejected, the connreq is
- passed to fi_reject().
-
- Any application data exchanged as part of the connection request is
- placed beyond the fi_eq_cm_entry structure. The amount of data
- available is application dependent and limited to the buffer space
- provided by the application when fi_eq_read is called. The amount
- of returned data may be calculated using the return value to
- fi_eq_read. Note that the amount of returned data is limited by the
- underlying connection protocol, and the length of any data returned
- may include protocol padding. As a result, the returned length may
- be larger than that specified by the connecting peer.
-
- If a connection request has been accepted, an FI_CONNECTED event will
- be generated on both sides of the connection. The active side -- one
- that called fi_connect() -- may receive user data as part of the
- FI_CONNECTED event. The user data is passed to the connection
- manager on the passive side through the fi_accept call. User data is
- not provided with an FI_CONNECTED event on the listening side of the
- connection.
-
- Notification that a remote peer has disconnected from an active
- endpoint is done through the FI_SHUTDOWN event. Shutdown
- notification uses struct fi_eq_cm_entry as declared above. The fid
- field for a shutdown notification refers to the active endpoint's
- fid_ep.
-
-*Asynchronous Error Notification*
-: Asynchronous errors are used to report problems with fabric resources.
- Reported errors may be fatal or transient, based on the error, and
- result in the resource becoming disabled. Disabled resources will fail
- operations submitted against them until they are explicitly re-enabled
- by the application.
-
- Asynchronous errors may be reported for completion queues and endpoints
- of all types. CQ errors can result when resource management has been
- disabled, and the provider has detected a queue overrun. Endpoint
- errors may be result of numerous actions, but are often associated with
- a failed operation. Operations may fail because of buffer overruns,
- invalid permissions, incorrect memory access keys, network routing
- failures, network reach-ability issues, etc.
-
- Asynchronous errors are reported using struct fi_eq_err_entry, as defined
- below. The fabric descriptor (fid) associated with the error is provided
- as part of the error data. An error code is also available to determine
- the cause of the error.
-
-## fi_eq_sread
-
-The fi_eq_sread call is the blocking (or synchronous) equivalent to
-fi_eq_read. It behaves is similar to the non-blocking call, with the
-exception that the calls will not return until either an event has
-been read from the EQ or an error or timeout occurs. Specifying a
-negative timeout means an infinite timeout.
-
-It is invalid for applications to call this function if the EQ
-has been configured with a wait object of FI_WAIT_NONE or FI_WAIT_SET.
-
-## fi_eq_readerr
-
-The read error function, fi_eq_readerr, retrieves information
-regarding any asynchronous operation which has completed with an
-unexpected error. fi_eq_readerr is a non-blocking call, returning
-immediately whether an error completion was found or not.
-
-EQs are optimized to report operations which have completed
-successfully. Operations which fail are reported 'out of band'. Such
-operations are retrieved using the fi_eq_readerr function. When an
-operation that completes with an unexpected error is inserted into an
-EQ, it is placed into a temporary error queue. Attempting to read
-from an EQ while an item is in the error queue results in an FI_EAVAIL
-failure. Applications may use this return code to determine when to
-call fi_eq_readerr.
-
-Error information is reported to the user through struct
-fi_eq_err_entry. The format of this structure is defined below.
-
-```c
-struct fi_eq_err_entry {
- fid_t fid; /* fid associated with error */
- void *context; /* operation context */
- uint64_t data; /* completion-specific data */
- int err; /* positive error code */
- int prov_errno; /* provider error code */
- void *err_data; /* additional error data */
- size_t err_data_size; /* size of err_data */
-};
-```
-
-The fid will reference the fabric descriptor associated with the
-event. For memory registration, this will be the fid_mr, address
-resolution will reference a fid_av, and CM events will refer to a
-fid_ep. The context field will be set to the context specified as
-part of the operation.
-
-The data field will be set as described in the man page for the
-corresponding object type (e.g., see [`fi_av`(3)](fi_av.3.html) for a
-description of how asynchronous address vector insertions are
-completed).
-
-The general reason for the error is provided through the err field.
-Provider or operational specific error information may also be available
-through the prov_errno and err_data fields. Users may call fi_eq_strerror to
-convert provider specific error information into a printable string
-for debugging purposes.
-
-If err_data_size is > 0, then the buffer referenced by err_data is
-directly user-accessible. Applications which read the err_data buffer
-must ensure that they do not read past the end of the referenced buffer.
-
-# RETURN VALUES
-
-fi_eq_open
-: Returns 0 on success. On error, a negative value corresponding to
- fabric errno is returned.
-
-fi_eq_read / fi_eq_readerr / fi_eq_sread
-: On success, returns the number of bytes read from the
- event queue. On error, a negative value corresponding to fabric
- errno is returned. If no data is available to be read from the
- event queue, -FI_EAGAIN is returned.
-
-fi_eq_write
-: On success, returns the number of bytes written to the
- event queue. On error, a negative value corresponding to fabric
- errno is returned.
-
-fi_eq_strerror
-: Returns a character string interpretation of the provider specific
- error returned with a completion.
-
-Fabric errno values are defined in
-`rdma/fi_errno.h`.
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_cntr`(3)](fi_cntr.3.html),
-[`fi_poll`(3)](fi_poll.3.html)
diff --git a/man/fi_errno.3.md b/man/fi_errno.3.md
deleted file mode 100644
index 4e3ca7d..0000000
--- a/man/fi_errno.3.md
+++ /dev/null
@@ -1,159 +0,0 @@
----
-layout: page
-title: fi_errno(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_errno \- fabric errors
-
-fi_strerror \- Convert fabric error into a printable string
-
-# SYNOPSIS
-
-```c
-#include <rdma/fi_errno.h>
-
-const char *fi_strerror(int errno);
-```
-
-
-# ERRORS
-
-*FI_ENOENT*
-: No such file or directory.
-
-*FI_EIO*
-: I/O error
-
-*FI_E2BIG*
-: Argument list too long.
-
-*FI_EBADF*
-: Bad file number.
-
-*FI_EAGAIN*
-: Try again.
-
-*FI_ENOMEM*
-: Out of memory.
-
-*FI_EACCES*
-: Permission denied.
-
-*FI_EBUSY*
-: Device or resource busy
-
-*FI_ENODEV*
-: No such device
-
-*FI_EINVAL*
-: Invalid argument
-
-*FI_EMFILE*
-: Too many open files
-
-*FI_ENOSPC*
-: No space left on device
-
-*FI_ENOSYS*
-: Function not implemented
-
-*FI_ENOMSG*
-: No message of desired type
-
-*FI_ENODATA*
-: No data available
-
-*FI_EMSGSIZE*
-: Message too long
-
-*FI_ENOPROTOOPT*
-: Protocol not available
-
-*FI_EOPNOTSUPP*
-: Operation not supported on transport endpoint
-
-*FI_EADDRINUSE*
-: Address already in use
-
-*FI_EADDRNOTAVAIL*
-: Cannot assign requested address
-
-*FI_ENETDOWN*
-: Network is down
-
-*FI_ENETUNREACH*
-: Network is unreachable
-
-*FI_ECONNABORTED*
-: Software caused connection abort
-
-*FI_ECONNRESET*
-: Connection reset by peer
-
-*FI_EISCONN*
-: Transport endpoint is already connected
-
-*FI_ENOTCONN*
-: Transport endpoint is not connected
-
-*FI_ESHUTDOWN*
-: Cannot send after transport endpoint shutdown
-
-*FI_ETIMEDOUT*
-: Operation timed out
-
-*FI_ECONNREFUSED*
-: Connection refused
-
-*FI_EHOSTUNREACH*
-: No route to host
-
-*FI_EALREADY*
-: Operation already in progress
-
-*FI_EINPROGRESS*
-: Operation now in progress
-
-*FI_EREMOTEIO*
-: Remote I/O error
-
-*FI_ECANCELED*
-: Operation Canceled
-
-*FI_ENOKEY*
-: Required key not available
-
-*FI_EKEYREJECTED*
-: Key was rejected by service
-
-*FI_EOTHER*
-: Unspecified error
-
-*FI_ETOOSMALL*
-: Provided buffer is too small
-
-*FI_EOPBADSTATE*
-: Operation not permitted in current state
-
-*FI_EAVAIL*
-: Error available
-
-*FI_EBADFLAGS*
-: Flags not supported
-
-*FI_ENOEQ*
-: Missing or unavailable event queue
-
-*FI_EDOMAIN*
-: Invalid resource domain
-
-*FI_ENOCQ*
-: Missing or unavailable completion queue
-
-# SEE ALSO
-
-[`fabric`(7)](fabric.7.html)
diff --git a/man/fi_fabric.3.md b/man/fi_fabric.3.md
deleted file mode 100644
index 366b68a..0000000
--- a/man/fi_fabric.3.md
+++ /dev/null
@@ -1,189 +0,0 @@
----
-layout: page
-title: fi_fabric(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_fabric \- Fabric domain operations
-
-fi_fabric / fi_close
-: Open / close a fabric domain
-
-fi_tostr
-: Convert fabric attributes, flags, and capabilities to printable string
-
-# SYNOPSIS
-
-```c
-#include <rdma/fabric.h>
-
-int fi_fabric(struct fi_fabric_attr *attr,
- struct fid_fabric **fabric, void *context);
-
-int fi_close(struct fid *fabric);
-
-char * fi_tostr(const void *data, enum fi_type datatype);
-```
-
-# ARGUMENTS
-
-*attr*
-: Attributes of fabric to open.
-
-*fabric*
-: Fabric domain
-
-*context*
-: User specified context associated with the opened object. This
- context is returned as part of any associated asynchronous event.
-
-# DESCRIPTION
-
-A fabric domain represents a collection of hardware and software
-resources that access a single physical or virtual network. All
-network ports on a system that can communicate with each other through
-their attached networks belong to the same fabric domain. A fabric
-domain shares network addresses and can span multiple providers.
-
-## fi_fabric
-
-Opens a fabric provider. The attributes of the fabric provider are
-specified through the open call, and may be obtained by calling
-fi_getinfo.
-
-## fi_close
-
-The fi_close call is used to release all resources associated with a
-fabric domain or interface. All items associated with the opened
-fabric must be released prior to calling fi_close.
-
-## fi_tostr
-
-Converts fabric interface attributes, capabilities, flags, and enum
-values into a printable string. The data parameter accepts a pointer
-to the attribute or value(s) to display, with the datatype parameter
-indicating the type of data referenced by the data parameter. Valid
-values for the datatype are listed below, along with the corresponding
-datatype or field value.
-
-*FI_TYPE_INFO*
-: struct fi_info
-
-*FI_TYPE_EP_TYPE*
-: struct fi_info::type field
-
-*FI_TYPE_EP_CAP*
-: struct fi_info::ep_cap field
-
-*FI_TYPE_OP_FLAGS*
-: struct fi_info::op_flags field, or general uint64_t flags
-
-*FI_TYPE_ADDR_FORMAT*
-: struct fi_info::addr_format field
-
-*FI_TYPE_TX_ATTR*
-: struct fi_tx_attr
-
-*FI_TYPE_RX_ATTR*
-: struct fi_rx_attr
-
-*FI_TYPE_EP_ATTR*
-: struct fi_ep_attr
-
-*FI_TYPE_DOMAIN_ATTR*
-: struct fi_domain_attr
-
-*FI_TYPE_FABRIC_ATTR*
-: struct fi_fabric_attr
-
-*FI_TYPE_DOMAIN_CAP*
-: struct fi_info::domain_cap field
-
-*FI_TYPE_THREADING*
-: enum fi_threading
-
-*FI_TYPE_PROGRESS*
-: enum fi_progress
-
-*FI_TYPE_PROTO*
-: struct fi_ep_attr::protocol field
-
-*FI_TYPE_MSG_ORDER*
-: struct fi_ep_attr::msg_order field
-
-*FI_TYPE_VERSION*
-: Returns the library version of libfabric in string form. The data
- parameter is ignored.
-
-fi_tostr() will return a pointer to an internal libfabric buffer that
-should not be modified, and will be overwritten the next time
-fi_tostr() is invoked. fi_tostr() is not thread safe.
-
-# NOTES
-
-The following resources are associated with fabric domains: access
-domains, passive endpoints, and CM event queues.
-
-# FABRIC ATTRIBUTES
-
-The fi_fabric_attr structure defines the set of attributes associated
-with a fabric and a fabric provider.
-
-```c
-struct fi_fabric_attr {
- struct fid_fabric *fabric;
- char *name;
- char *prov_name;
- uint32_t prov_version;
-};
-```
-
-## fabric
-
-On input to fi_getinfo, a user may set this to an opened fabric
-instance to restrict output to the given fabric. On output from
-fi_getinfo, if no fabric was specified, but the user has an opened
-instance of the named fabric, this will reference the first opened
-instance. If no instance has been opened, this field will be NULL.
-
-## name
-
-A fabric identifier.
-
-## prov_name
-
-The name of the underlying fabric provider.
-
-For debugging and administrative purposes, environment variables can be used
-to control which fabric providers will be registered with libfabric.
-Specifying "FI_PROVIDER=foo,bar" will allow any providers with the names "foo"
-or "bar" to be registered. Similarly, specifying "FI_PROVIDER=^foo,bar" will
-prevent any providers with the names "foo" or "bar" from being registered.
-Providers which are not registered will not appear in fi_getinfo results.
-Applications which need a specific set of providers should implement
-their own filtering of fi_getinfo's results rather than relying on these
-environment variables in a production setting.
-
-## prov_version
-
-Version information for the fabric provider.
-
-# RETURN VALUE
-
-Returns FI_SUCCESS on success. On error, a negative value corresponding to
-fabric errno is returned. Fabric errno values are defined in
-`rdma/fi_errno.h`.
-
-# ERRORS
-
-
-# SEE ALSO
-
-[`fabric`(7)](fabric.7.html),
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_eq`(3)](fi_eq.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html)
diff --git a/man/fi_getinfo.3.md b/man/fi_getinfo.3.md
deleted file mode 100644
index c52ae9f..0000000
--- a/man/fi_getinfo.3.md
+++ /dev/null
@@ -1,576 +0,0 @@
----
-layout: page
-title: fi_getinfo(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_getinfo / fi_freeinfo \- Obtain / free fabric interface information
-
-fi_allocinfo / fi_dupinfo \- Allocate / duplicate an fi_info structure
-
-# SYNOPSIS
-
-```c
-#include <rdma/fabric.h>
-
-int fi_getinfo(int version, const char *node, const char *service,
- uint64_t flags, struct fi_info *hints, struct fi_info **info);
-
-void fi_freeinfo(struct fi_info *info);
-
-struct fi_info *fi_allocinfo(void);
-
-struct fi_info *fi_dupinfo(const struct fi_info *info);
-```
-
-# ARGUMENTS
-
-*version*
-: Interface version requested by application.
-
-*node*
-: Optional, name or fabric address to resolve.
-
-*service*
-: Optional, service name or port number of address.
-
-*flags*
-: Operation flags for the fi_getinfo call.
-
-*hints*
-: Reference to an fi_info structure that specifies criteria for
- selecting the returned fabric information.
-
-*info*
-: A pointer to a linked list of fi_info structures containing response
- information.
-
-# DESCRIPTION
-
-fi_getinfo returns information about available fabric services for reaching
-specified node or service, subject to any provided hints. Callers
-may specify NULL for node, service, and hints in order to retrieve
-information about what providers are available and their optimal usage
-models. If no matching fabric information is available, info will
-be set to NULL and the call will return -FI_ENODATA.
-
-Based on the input hints, node, and service parameters, a list of
-fabric domains and endpoints will be returned. Each fi_info structure
-will describe an endpoint that meets the application's specified
-communication criteria. Each endpoint will be associated with a
-domain. Applications can restrict the number of returned endpoints by
-including additional criteria in their search hints. Relaxing or
-eliminating input hints will increase the number and type of endpoints
-that are available. Providers that return multiple endpoints to a
-single fi_getinfo call should return the endpoints that are highest
-performing first. Providers may indicate that an endpoint and domain can
-support additional capabilities than those requested by the user only
-if such support will not adversely affect application performance or security.
-
-The version parameter is used by the application to request the
-desired version of the interfaces. The version determines the format
-of all data structures used by any of the fabric interfaces.
-Applications should use the FI_VERSION(major, minor) macro to indicate
-the version, with hard-coded integer values for the major and minor
-values. The FI_MAJOR_VERSION and FI_MINOR_VERSION enum values defined
-in fabric.h specify the latest version of the installed library.
-However, it is recommended that the integer values for
-FI_MAJOR_VERSION and FI_MINOR_VERSION be used, rather than referencing
-the enum types in order to ensure compatibility with future versions
-of the library. This protects against the application being built
-from source against a newer version of the library that introduces new
-fields to data structures, which would not be initialized by the
-application.
-
-Node, service, or hints may be provided, with any combination
-being supported. If node is provided, fi_getinfo will attempt to
-resolve the fabric address to the given node. If node is not given,
-fi_getinfo will attempt to resolve the fabric addressing information
-based on the provided hints.
-
-The hints parameter, if provided, may be used to limit the resulting
-output as indicated below. As a general rule, specifying a non-zero
-value for input hints indicates that a provider must support the
-requested value or fail the operation with -FI_ENODATA. With the
-exception of mode bits, hints that are set to zero are treated as
-a wildcard. A zeroed hint value results in providers either returning
-a default value or a value that works best for their implementation.
-Mode bits that are set to zero indicate the application does not support
-any modes.
-
-The caller must call fi_freeinfo to release fi_info structures returned
-by this call.
-
-The fi_allocinfo call will allocate and zero an fi_info structure
-and all related substructures. The fi_dupinfo will duplicate
-a single fi_info structure and all the substructures within it.
-
-
-# FI_INFO
-
-```c
-struct fi_info {
- struct fi_info *next;
- uint64_t caps;
- uint64_t mode;
- uint32_t addr_format;
- size_t src_addrlen;
- size_t dest_addrlen;
- void *src_addr;
- void *dest_addr;
- fid_t handle;
- struct fi_tx_attr *tx_attr;
- struct fi_rx_attr *rx_attr;
- struct fi_ep_attr *ep_attr;
- struct fi_domain_attr *domain_attr;
- struct fi_fabric_attr *fabric_attr;
-};
-```
-
-*next*
-: Pointer to the next fi_info structure in the list. Will be NULL
- if no more structures exist.
-
-*caps - fabric interface capabilities*
-: If specified, indicates the desired capabilities of the fabric
- interfaces. Supported capabilities are listed in the _Capabilities_
- section below.
-
-*mode*
-: Operational modes supported by the application. See the _Mode_
- section below.
-
-*addr_format - address format*
-: If specified, indicates the format of addresses referenced by the
- fabric interfaces and data structures. Supported formats are listed
- in the _Addressing formats_ section below.
-
-*src_addrlen - source address length*
-: Indicates the length of the source address. This value must be > 0
- if *src_addr* is non-NULL. This field will be ignored in hints if
- FI_SOURCE is specified, or *src_addr* is NULL.
-
-*dest_addrlen - destination address length*
-: Indicates the length of the destination address. This value must be > 0
- if *dest_addr* is non-NULL. This field will be ignored in hints
- unless the node and service parameters are NULL or FI_SOURCE is
- specified, or if *dst_addr* is NULL.
-
-*src_addr - source address*
-: If specified, indicates the source address. This field will be
- ignored in hints if FI_SOURCE is specified. On output a provider shall
- return an address that corresponds to the indicated fabric or domain,
- with the format indicated by the returned *addr_format* field.
-
-*dest_addr - destination address*
-: If specified, indicates the destination address. This field will be
- ignored in hints unless the node and service parameters are NULL or
- FI_SOURCE is specified.
-
-*handle - provider context handle*
-: References a provider specific handle. The use of this field
- is operation specific. Unless its use is described for a given operation,
- the handle field must be NULL. It is commonly used by applications
- that make use of connection-oriented endpoints. For other applications,
- the field should usually be NULL.
-
- This field is used when processing connection requests and
- responses. It is also used to inherit endpoint's attributes.
- See fi_eq(3), fi_reject(3), and fi_endpoint(3) .
-
-*tx_attr - transmit context attributes*
-: Optionally supplied transmit context attributes. Transmit context
- attributes may be specified and returned as part of fi_getinfo.
- When provided as hints, requested values of struct fi_tx_ctx_attr
- should be set. On output, the actual transmit context attributes
- that can be provided will be returned. Output values will be
- greater than or equal to the requested input values.
-
-*rx_attr - receive context attributes*
-: Optionally supplied receive context attributes. Receive context
- attributes may be specified and returned as part of fi_getinfo.
- When provided as hints, requested values of struct fi_rx_ctx_attr
- should be set. On output, the actual receive context attributes
- that can be provided will be returned. Output values will be
- greater than or or equal to the requested input values.
-
-*ep_attr - endpoint attributes*
-: Optionally supplied endpoint attributes. Endpoint attributes may be
- specified and returned as part of fi_getinfo. When provided as
- hints, requested values of struct fi_ep_attr should be set. On
- output, the actual endpoint attributes that can be provided will be
- returned. Output values will be greater than or equal to requested
- input values. See fi_endpoint(3) for details.
-
-*domain_attr - domain attributes*
-: Optionally supplied domain attributes. Domain attributes may be
- specified and returned as part of fi_getinfo. When provided as
- hints, requested values of struct fi_domain_attr should be set. On
- output, the actual domain attributes that can be provided will be
- returned. Output values will be greater than or equal to requested
- input values. See fi_domain(3) for details.
-
-*fabric_attr - fabric attributes*
-: Optionally supplied fabric attributes. Fabric attributes may be
- specified and returned as part of fi_getinfo. When provided as
- hints, requested values of struct fi_fabric_attr should be set. On
- output, the actual fabric attributes that can be provided will be
- returned. See fi_fabric(3) for details.
-
-# CAPABILITIES
-
-Interface capabilities are obtained by OR-ing the following flags
-together. If capabilities in the hint parameter are set to 0, the
-underlying provider will return the set of capabilities which are
-supported. Otherwise, providers will return data matching the
-specified set of capabilities. Providers may indicate support for
-additional capabilities beyond those requested when the use of
-expanded capabilities will not adversely affect performance or expose
-the application to communication beyond that which was requested.
-Applications may use this feature to request a minimal set of
-requirements, then check the returned capabilities to enable
-additional optimizations.
-
-*FI_MSG*
-: Specifies that an endpoint should support sending and receiving
- messages or datagrams. Message capabilities imply support for send
- and/or receive queues. Endpoints supporting this capability support
- operations defined by struct fi_ops_msg.
-
- The caps may be used to specify or restrict the type of messaging
- operations that are supported. In the absence of any relevant
- flags, FI_MSG implies the ability to send and receive messages.
- Applications can use the FI_SEND and FI_RECV flags to optimize an
- endpoint as send-only or receive-only.
-
-*FI_RMA*
-: Specifies that the endpoint should support RMA read and write
- operations. Endpoints supporting this capability support operations
- defined by struct fi_ops_rma. In the absence of any relevant flags,
- FI_RMA implies the ability to initiate and be the target of remote
- memory reads and writes. Applications can use the FI_READ,
- FI_WRITE, FI_REMOTE_READ, and FI_REMOTE_WRITE flags to restrict the
- types of RMA operations supported by an endpoint.
-
-*FI_TAGGED*
-: Specifies that the endpoint should handle tagged message transfers.
- Tagged message transfers associate a user-specified key or tag with
- each message that is used for matching purposes at the remote side.
- Endpoints supporting this capability support operations defined by
- struct fi_ops_tagged. In the absence of any relevant flags,
- FI_TAGGED implies the ability to send and receive tagged messages.
- Applications can use the FI_SEND and FI_RECV flags to optimize an
- endpoint as send-only or receive-only.
-
-*FI_ATOMIC*
-: Specifies that the endpoint supports some set of atomic operations.
- Endpoints supporting this capability support operations defined by
- struct fi_ops_atomic. In the absence of any relevant flags,
- FI_ATOMIC implies the ability to initiate and be the target of
- remote atomic reads and writes. Applications can use the FI_READ,
- FI_WRITE, FI_REMOTE_READ, and FI_REMOTE_WRITE flags to restrict the
- types of atomic operations supported by an endpoint.
-
-*FI_NAMED_RX_CTX*
-: Requests that endpoints which support multiple receive contexts
- allow an initiator to target (or name) a specific receive context as
- part of a data transfer operation.
-
-*FI_DIRECTED_RECV*
-: Requests that the communication endpoint use the source address of
- an incoming message when matching it with a receive buffer. If this
- capability is not set, then the src_addr parameter for msg and tagged
- receive operations is ignored.
-
-*FI_MULTI_RECV*
-: Specifies that the endpoint must support the FI_MULTI_RECV flag when
- posting receive buffers.
-
-*FI_SOURCE*
-: Requests that the endpoint return source addressing data as part of
- its completion data. This capability only applies to connectionless
- endpoints. Note that returning source address information may
- require that the provider perform address translation and/or look-up
- based on data available in the underlying protocol in order to
- provide the requested data, which may adversely affect performance.
-
-*FI_READ*
-: Indicates that the user requires an endpoint capable of initiating
- reads against remote memory regions. This flag requires that FI_RMA
- and/or FI_ATOMIC be set.
-
-*FI_WRITE*
-: Indicates that the user requires an endpoint capable of initiating
- writes against remote memory regions. This flag requires that FI_RMA
- and/or FI_ATOMIC be set.
-
-*FI_SEND*
-: Indicates that the user requires an endpoint capable of sending
- message data transfers. Message transfers include base message
- operations as well as tagged message functionality.
-
-*FI_RECV*
-: Indicates that the user requires an endpoint capable of receiving
- message data transfers. Message transfers include base message
- operations as well as tagged message functionality.
-
-*FI_REMOTE_READ*
-: Indicates that the user requires an endpoint capable of receiving
- read memory operations from remote endpoints. This flag requires
- that FI_RMA and/or FI_ATOMIC be set.
-
-*FI_REMOTE_WRITE*
-: Indicates that the user requires an endpoint capable of receiving
- write memory operations from remote endpoints. This flag requires
- that FI_RMA and/or FI_ATOMIC be set.
-
-*FI_RMA_EVENT*
-: Requests that an endpoint support the generation of completion events
- when it is the target of an RMA and/or atomic operation. This
- flag requires that FI_REMOTE_READ and/or FI_REMOTE_WRITE be enabled on
- the endpoint.
-
-*FI_TRIGGER*
-: Indicates that the endpoint should support triggered operations.
- Endpoints support this capability must meet the usage model as
- described by fi_trigger.3.
-
-*FI_FENCE*
-: Indicates that the endpoint support the FI_FENCE flag on data
- transfer operations. Support requires tracking that all previous
- transmit requests to a specified remote endpoint complete prior
- to initiating the fenced operation. Fenced operations are often
- used to enforce ordering between operations that are not otherwise
- guaranteed by the underlying provider or protocol.
-
-Capabilities may be grouped into two general categories: primary and
-secondary. Primary capabilities must explicitly be requested by an
-application, and a provider must enable support for only those primary
-capabilities which were selected. Secondary capabilities may optionally
-be requested by an application. If requested, a provider must support
-the capability or fail the fi_getinfo request (FI_ENODATA). A provider
-may optionally report non-selected secondary capabilities if doing so
-would not compromise performance or security.
-
-Primary capabilities: FI_MSG, FI_RMA, FI_TAGGED, FI_ATOMIC, FI_NAMED_RX_CTX,
-FI_DIRECTED_RECV, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ,
-and FI_REMOTE_WRITE.
-
-Secondary capabilities: FI_MULTI_RECV, FI_SOURCE, FI_RMA_EVENT, FI_TRIGGER, FI_FENCE.
-
-# MODE
-
-The operational mode bits are used to convey requirements that an
-application must adhere to when using the fabric interfaces. Modes
-specify optimal ways of accessing the reported endpoint or domain.
-Applications that are designed to support a specific mode of operation
-may see improved performance when that mode is desired by the
-provider. It is recommended that providers support applications that
-disable any provider preferred modes.
-
-On input to fi_getinfo, applications set the mode bits that they
-support. On output, providers will clear mode bits that are not
-necessary to achieve high-performance. Mode bits that remain set
-indicate application requirements for using the fabric interfaces
-created using the returned fi_info. The set of modes are listed
-below. If a NULL hints structure is provided, then the provider's
-supported set of modes will be returned in the info structure(s).
-
-*FI_CONTEXT*
-: Specifies that the provider requires that applications use struct
- fi_context as their per operation context parameter. This structure
- should be treated as opaque to the application. For performance
- reasons, this structure must be allocated by the user, but may be
- used by the fabric provider to track the operation. Typically,
- users embed struct fi_context within their own context structure.
- The struct fi_context must remain valid until the corresponding
- operation completes or is successfully canceled. As such,
- fi_context should NOT be allocated on the stack. Doing so is likely
- to result in stack corruption that will be difficult to debug.
- Users should not update or interpret the fields in this structure,
- or reuse it until the original operation has completed. The
- structure is specified in rdma/fabric.h.
-
-*FI_LOCAL_MR*
-: The provider is optimized around having applications register memory
- for locally accessed data buffers. Data buffers used in send and
- receive operations and as the source buffer for RMA and atomic
- operations must be registered by the application for access domains
- opened with this capability.
-
-*FI_MSG_PREFIX*
-: Message prefix mode indicates that an application will provide
- buffer space in front of all message send and receive buffers for
- use by the provider. Typically, the provider uses this space to
- implement a protocol, with the protocol headers being written into
- the prefix area. The contents of the prefix space should be treated
- as opaque. The use of FI_MSG_PREFIX may improve application
- performance over certain providers by reducing the number of IO
- vectors referenced by underlying hardware and eliminating provider
- buffer allocation.
-
- FI_MSG_PREFIX only applies to send and receive operations, including
- tagged sends and receives. RMA and atomics do not require the
- application to provide prefix buffers. Prefix buffer space must be
- provided with all sends and receives, regardless of the size of the
- transfer or other transfer options. The ownership of prefix buffers
- is treated the same as the corresponding message buffers, but the
- size of the prefix buffer is not counted toward any message limits,
- including inject.
-
- Applications that support prefix mode must supply buffer space
- before their own message data. The size of space that must be
- provided is specified by the msg_prefix_size endpoint attribute.
- Providers are required to define a msg_prefix_size that is a
- multiple of 8 bytes. Additionally, applications may receive
- provider generated packets that do not contain application data.
- Such received messages will indicate a transfer size of that is
- equal to or smaller than msg_prefix_size.
-
- The buffer pointer given to all send and receive operations must point
- to the start of the prefix region of the buffer (as opposed to the
- payload). For scatter-gather send/recv operations, the prefix buffer
- must be a contiguous region, though it may or may not be directly
- adjacent to the payload portion of the buffer.
-
-*FI_ASYNC_IOV*
-: Applications can reference multiple data buffers as part of a single
- operation through the use of IO vectors (SGEs). Typically,
- the contents of an IO vector are copied by the provider into an
- internal buffer area, or directly to the underlying hardware.
- However, when a large number of IOV entries are supported,
- IOV buffering may have a negative impact on performance and memory
- consumption. The FI_ASYNC_IOV mode indicates that the application
- must provide the buffering needed for the IO vectors. When set,
- an application must not modify an IO vector, including any
- related memory descriptor array, until the associated
- operation has completed.
-
-*FI_RX_CQ_DATA*
-: This mode bit only applies to data transfers that set FI_REMOTE_CQ_DATA.
- When set, a data transfer that carries remote CQ data will consume a
- receive buffer at the target. This is true even for operations that would
- normally not consume posted receive buffers, such as RMA write operations.
-
-# ADDRESSING FORMATS
-
-Multiple fabric interfaces take as input either a source or
-destination address parameter. This includes struct fi_info (src_addr
-and dest_addr), CM calls (getname, getpeer, connect, join, and leave),
-and AV calls (insert, lookup, and straddr). The fi_info addr_format
-field indicates the expected address format for these operations.
-
-A provider may support one or more of the following addressing
-formats. In some cases, a selected addressing format may need to be
-translated or mapped into an address which is native to the
-fabric. See `fi_av`(3).
-
-*FI_FORMAT_UNSPEC*
-: FI_FORMAT_UNSPEC indicates that a provider specific address format
- should be selected. Provider specific addresses may be protocol
- specific or a vendor proprietary format. Applications that select
- FI_FORMAT_UNSPEC should be prepared to treat returned addressing
- data as opaque. FI_FORMAT_UNSPEC targets apps which make use of an
- out of band address exchange. Applications which use FI_FORMAT_UNSPEC
- may use fi_getname() to obtain a provider specific address assigned
- to an allocated endpoint.
-
-*FI_SOCKADDR*
-: Address is of type sockaddr. The specific socket address format
- will be determined at run time by interfaces examining the sa_family
- field.
-
-*FI_SOCKADDR_IN*
-: Address is of type sockaddr_in (IPv4).
-
-*FI_SOCKADDR_IN6*
-: Address is of type sockaddr_in6 (IPv6).
-
-*FI_SOCKADDR_IB*
-: Address is of type sockaddr_ib (defined in Linux kernel source)
-
-*FI_ADDR_PSMX*
-: Address is an Intel proprietary format that is used with their PSMX
- (extended performance scaled messaging) protocol.
-
-# FLAGS
-
-The operation of the fi_getinfo call may be controlled through the use of
-input flags. Valid flags include the following.
-
-*FI_NUMERICHOST*
-: Indicates that the node parameter is a numeric string representation
- of a fabric address, such as a dotted decimal IP address. Use of
- this flag will suppress any lengthy name resolution protocol.
-
-*FI_SOURCE*
-: Indicates that the node and service parameters specify the local
- source address to associate with an endpoint. If specified, either
- the node and/or service parameter must be non-NULL. This flag is
- often used with passive endpoints.
-
-*FI_PROV_ATTR_ONLY*
-: Indicates that the caller is only querying for what providers are
- potentially available. All providers will return exactly one
- fi_info struct, regardless of whether that provider is usable on the
- current platform or not. The returned fi_info struct will contain
- default values for all members, with the exception of fabric_attr.
- The fabric_attr member will have the prov_name and prov_version
- values filled in.
-
-# RETURN VALUE
-
-fi_getinfo() returns 0 on success. On error, fi_getinfo() returns a
-negative value corresponding to fabric errno. Fabric errno values are
-defined in `rdma/fi_errno.h`.
-
-fi_allocinfo() returns a pointer to a new fi_info structure on
-success, or NULL on error. fi_dupinfo() duplicates a single fi_info
-structure and all the substructures within it, returning a pointer to
-the new fi_info structure on success, or NULL on error.
-Both calls require that the returned fi_info structure be freed
-via fi_freeinfo().
-
-# ERRORS
-
-*FI_EBADFLAGS*
-: The specified endpoint or domain capability or operation flags are
- invalid.
-
-*FI_ENOMEM*
-: Indicates that there was insufficient memory to complete the operation.
-
-*FI_ENODATA*
-: Indicates that no providers could be found which support the requested
- fabric information.
-
-# NOTES
-
-If hints are provided, the operation will be controlled by the values
-that are supplied in the various fields (see section on _fi_info_).
-Applications that require specific communication interfaces, domains,
-capabilities or other requirements, can specify them using fields in
-_hints_. Libfabric returns a linked list in *info* that points to a
-list of matching interfaces. *info* is set to NULL if there are no
-communication interfaces or none match the input hints.
-
-If node is provided, fi_getinfo will attempt to resolve the fabric
-address to the given node. If node is not provided, fi_getinfo will
-attempt to resolve the fabric addressing information based on the
-provided hints. The caller must call fi_freeinfo to release fi_info
-structures returned by fi_getinfo.
-
-If neither node, service or hints are provided, then fi_getinfo simply
-returns the list all available communication interfaces.
-
-Multiple threads may call
-`fi_getinfo` simultaneously, without any requirement for serialization.
-
-# SEE ALSO
-
-[`fi_open`(3)](fi_open.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_domain`(3)](fi_domain.3.html)
diff --git a/man/fi_gni.7.md b/man/fi_gni.7.md
deleted file mode 100644
index e8c71b0..0000000
--- a/man/fi_gni.7.md
+++ /dev/null
@@ -1,197 +0,0 @@
----
-layout: page
-title: fi_gni(7)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-The GNI Fabric Provider
-
-# OVERVIEW
-
-The GNI provider runs on Cray XC (TM) systems utilizing the user-space
-Generic Network Interface (uGNI) which provides low-level access to
-the Aries interconnect. The Aries interconnect is designed for
-low-latency one-sided messaging and also includes direct hardware
-support for common atomic operations and optimized collectives.
-
-# REQUIREMENTS
-
-The GNI provider runs on Cray XC systems running CLE 5.2 UP04 or higher
-using gcc version 4.9 or higher.
-
-# SUPPORTED FEATURES
-
-The GNI provider supports the following features defined for the
-libfabric API:
-
-*Endpoint types*
-: The provider supports the *FI_EP_RDM* and *FI_EP_DGRAM* endpoint types.
-
-*Address vectors*
-: The provider implements both the *FI_AV_MAP* and *FI_AV_TABLE*
- address vector types.
-
-*Memory registration modes*
-: The provider implements the *FI_MR_BASIC* memory registration mode.
-
-*Data transfer operations*
-
-: The following data transfer interfaces are supported for all
- endpoint types: *FI_ATOMIC*, *FI_MSG*, *FI_RMA*, *FI_TAGGED*. See
- DATA TRANSFER OPERATIONS below for more details.
-
-*Completion events*
-: The GNI provider supports *FI_CQ_FORMAT_CONTEXT*, *FI_CQ_FORMAT_MSG*,
- *FI_CQ_FORMAT_DATA* and *FI_CQ_FORMAT_TAGGED* with wait objects of type
- *FI_WAIT_NONE*, *FI_WAIT_FD*, and *FI_WAIT_MUTEX_COND*.
-
-*Modes*
-: The GNI provider does not require any operation modes.
-
-*Progress*
-
-: For both control and data progress, the GNI provider supports both
- *FI_PROGRESS_AUTO* and *FI_PROGRESS_MANUAL*, with a default set to
- *FI_PROGRESS_AUTO*.
-
-*Additional Features*
-: The GNI provider also supports the following capabilities and features:
-- *FI_MULTI_RECV*
-- *FI_SOURCE*
-- *FI_FENCE*
-- *FI_RM_ENABLED*
-- *FABRIC_DIRECT* compilation mode
-
-# DATA TRANSFER OPERATIONS
-
-## FI_ATOMIC
-
-Currently, the GNI provider only supports atomic operations supported
-directly by the Aries NIC. These include operations on 32- and
-64-bit, signed and unsigned integer and floating point values.
-Specifically,
-
-### Basic (fi_atomic, etc.)
-- *FI_MIN*, *FI_MAX* (no unsigned)
-- *FI_SUM* (no 64-bit floating point)
-- *FI_BOR*, *FI_BAND*, *FI_BXOR* (no floating point)
-- *FI_ATOMIC_WRITE*
-
-### Fetching (fi_fetch_atomic, etc.)
-- All of the basic operations as above
-- FI_ATOMIC_READ
-
-### Comparison (fi_compare_atomic, etc.)
-- FI_CSWAP
-- FI_MSWAP
-
-## FI_MSG
-
-All *FI_MSG* operations are supported.
-
-## FI_RMA
-
-All *FI_RMA* operations are supported.
-
-## FI_TAGGED
-
-All *FI_TAGGED* operations are supported except `fi_tinjectdata`.
-
-# GNI EXTENSIONS
-
-The GNI provider exposes low-level tuning parameters via a domain and
-endpoint `fi_open_ops` interface named *FI_GNI_DOMAIN_OPS_1* and
-*FI_GNI_EP_OPS_1*. The flags parameter is currently ignored. The
-fi_open_ops function takes either a `struct fi_gni_ops_domain` or a
-`struct fi_gni_ops_ep` parameter and populates it with the following:
-
-```c
-struct fi_gni_ops_domain {
- int (*set_val)(struct fid *fid, dom_ops_val_t t, void *val);
- int (*get_val)(struct fid *fid, dom_ops_val_t t, void *val);
-};
-
-struct fi_gni_ops_ep {
- int (*set_val)(struct fid *fid, dom_ops_val_t t, void *val);
- int (*get_val)(struct fid *fid, dom_ops_val_t t, void *val);
-};
-```
-
-The `set_val` function sets the value of a given parameter; the
-`get_val` function returns the current value. For
-*FI_GNI_DOMAIN_OPS_1*, the currently supported values are:
-
-*GNI_MSG_RENDEZVOUS_THRESHOLD*
-: Threshold message size at which a rendezvous protocol is used for
- *FI_MSG* data transfers. The value is of type uint32_t.
-
-*GNI_RMA_RDMA_THRESHOLD*
-: Threshold message size at which RDMA is used for *FI_RMA* data
- transfers. The value is of type uint32_t.
-
-*GNI_CONN_TABLE_INITIAL_SIZE*
-: Initial size of the internal table data structure used to manage
- connections. The value is of type uint32_t.
-
-*GNI_CONN_TABLE_MAX_SIZE*
-: Maximum size of the internal table data structure used to manage
- connections. The value is of type uint32_t.
-
-*GNI_CONN_TABLE_STEP_SIZE*
-: Step size for increasing the size of the internal table data
- structure used to manage internal GNI connections. The value is of
- type uint32_t.
-
-*GNI_VC_ID_TABLE_CAPACITY*
-: Size of the virtual channel (VC) table used for managing remote
- connections. The value is of type uint32_t.
-
-*GNI_MBOX_PAGE_SIZE*
-: Page size for GNI SMSG mailbox allocations. The value is of type
- uint32_t.
-
-*GNI_MBOX_NUM_PER_SLAB*
-: Number of GNI SMSG mailboxes per allocation slab. The value is of
- type uint32_t.
-
-*GNI_MBOX_MAX_CREDIT*
-: Maximum number of credits per GNI SMSG mailbox. The value is of
- type uint32_t.
-
-*GNI_MBOX_MSG_MAX_SIZE*
-: Maximum size of GNI SMSG messages. The value is of type uint32_t.
-
-*GNI_RX_CQ_SIZE*
-: Recommended GNI receive CQ size. The value is of type uint32_t.
-
-*GNI_TX_CQ_SIZE*
-: Recommended GNI transmit CQ size. The value is of type uint32_t.
-
-*GNI_MAX_RETRANSMITS*
-: Maximum number of message retransmits before failure. The value is
- of type uint32_t.
-
-*GNI_MR_CACHE_LAZY_DEREG*
-: Enable or disable lazy deregistration of memory. The value is of
- type int32_t.
-
-For *FI_GNI_EP_OPS_1*, the currently supported values are:
-*GNI_HASH_TAG_IMPL*
-: Use a hashlist for the tag list implementation. The value is of type uint32_t.
-
-
-# SEE ALSO
-
-[`fabric`(7)](fabric.7.html),
-[`fi_open_ops`(3)](fi_open_ops.3.html),
-[`fi_provider`(7)](fi_provider.7.html),
-[`fi_getinfo`(3)](fi_getinfo.3.html)
-
-For more information on uGNI, see *Using the GNI and DMAPP APIs*
-(S-2446-3103, Cray Inc.). For more information on the GNI provider,
-see *An Implementation of OFI libfabric in Support of Multithreaded
-PGAS Solutions* (PGAS '15).
-
diff --git a/man/fi_mr.3.md b/man/fi_mr.3.md
deleted file mode 100644
index c984e26..0000000
--- a/man/fi_mr.3.md
+++ /dev/null
@@ -1,320 +0,0 @@
----
-layout: page
-title: fi_mr(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_mr \- Memory region operations
-
-fi_mr_reg / fi_mr_regv / fi_mr_regattr
-: Register local memory buffers for direct fabric access
-
-fi_close
-: Deregister registered memory buffers.
-
-fi_mr_desc
-: Return a local descriptor associated with a registered memory region
-
-fi_mr_key
-: Return the remote key needed to access a registered memory region
-
-fi_mr_bind
-: Associate a registered memory region with a completion counter.
-
-# SYNOPSIS
-
-```c
-#include <rdma/fi_domain.h>
-
-int fi_mr_reg(struct fid_domain *domain, const void *buf, size_t len,
- uint64_t access, uint64_t offset, uint64_t requested_key,
- uint64_t flags, struct fid_mr **mr, void *context);
-
-int fi_mr_regv(struct fid_domain *domain, const struct iovec * iov,
- size_t count, uint64_t access, uint64_t offset, uint64_t requested_key,
- uint64_t flags, struct fid_mr **mr, void *context);
-
-int fi_mr_regattr(struct fid_domain *domain, const struct fi_mr_attr *attr,
- uint64_t flags, struct fid_mr **mr);
-
-int fi_close(struct fid *mr);
-
-void * fi_mr_desc(struct fid_mr *mr);
-
-uint64_t fi_mr_key(struct fid_mr *mr);
-
-int fi_mr_bind(struct fid_mr *mr, struct fid *bfid, uint64_t flags);
-```
-
-# ARGUMENTS
-
-*domain*
-: Resource domain
-
-*mr*
-: Memory region
-
-*bfid*
-: Fabric identifier of an associated resource.
-
-*context*
-: User specified context associated with the memory region.
-
-*buf*
-: Memory buffer to register with the fabric hardware
-
-*len*
-: Length of memory buffer to register
-
-*iov*
-: Vectored memory buffer.
-
-*count*
-: Count of vectored buffer entries.
-
-*access*
-: Memory access permissions associated with registration
-
-*offset*
-: Optional specified offset for accessing specified registered buffers.
- This parameter is reserved for future use and must be 0.
-
-*requested_key*
-: Optional requested remote key associated with registered buffers.
-
-*attr*
-: Memory region attributes
-
-*flags*
-: Additional flags to apply to the operation.
-
-# DESCRIPTION
-
-Registered memory regions associate memory buffers with permissions
-granted for access by fabric resources. A memory buffer must be
-registered with a resource domain before it can be used as the target
-of a remote RMA or atomic data transfer. Additionally, a fabric
-provider may require that data buffers be registered before being used
-in local transfers.
-
-A provider may hide local registration requirements from applications
-by making use of an internal registration cache or similar mechanisms.
-Such mechanisms, however, may negatively impact performance for some
-applications, notably those which manage their own network buffers.
-In order to support as broad range of applications as possible,
-without unduly affecting their performance, applications that wish to
-manage their own local memory registrations may do so by using the
-memory registration calls. Applications may use the FI_LOCAL_MR
-domain mode bit as a guide.
-
-When the FI_LOCAL_MR mode bit is set, applications must register all
-data buffers that will be accessed by the local hardware and provide
-a valid mem_desc parameter into applicable data transfer operations.
-When FI_LOCAL_MR is zero, applications are not required to register
-data buffers before using them for local operations (e.g. send and
-receive data buffers), and the mem_desc parameter into data transfer
-operations is ignored.
-
-Further behavior of memory registration operations is
-controlled based on the mr_mode field in the domain attribute.
-
-*Basic Memory Registration Mode*
-: If the mr_mode field is set to FI_MR_BASIC, then memory registration
- operations are set to basic mode. In basic mode, registration occurs
- on allocated data buffers, and the MR attributes are selected by
- the provider.
-
- Basic mode uses provider assigned attributes for the registered buffers.
- The local memory descriptor and remote memory key are selected by
- the provider. The address used to access a buffer as the target of an
- RMA or atomic operation is the same as the virtual address of the buffer.
-
- Applications that support the basic registration mode will need to
- exchange MR parameters with remote peers for RMA and atomic
- operations. The exchanged data should include both the address of
- the memory region as well as the MR key.
-
-*Scalable Memory Registration Mode*
-: If the mr_mode field is set to FI_MR_SCALABLE, then memory registration
- operations are set to scalable mode. In scalable mode, registration
- occurs on memory address ranges, and the MR attributes are selected by
- the user.
-
- Memory regions registered as the target of RMA and atomic operations are
- associated with a MR key selected by the application. If local
- registrations are required (see FI_LOCAL_MR mode), the local descriptor
- will be the same as the remote key. The resulting memory
- region will be accessible by remote peers starting at a base address of 0.
- Because scalable registration mode refers to memory regions, versus data
- buffers, the address ranges given for a registration request do not need
- to map to data buffers allocated by the application at the time the
- registration call is made. That is, an application can register any
- range of addresses in their virtual address space, whether or not those
- addresses are backed by physical pages or have been allocated.
-
-The registrations functions -- fi_mr_reg, fi_mr_regv, and
-fi_mr_regattr -- are used to register one or more memory regions with
-fabric resources. The main difference between registration functions
-are the number and type of parameters that they accept as input.
-Otherwise, they perform the same general function.
-
-By default, memory registration completes synchronously. I.e. the
-registration call will not return until the registration has
-completed. Memory registration can complete asynchronous by binding
-the resource domain to an event queue using the FI_REG_MR flag. See
-fi_domain_bind. When memory registration is asynchronous, in order to
-avoid a race condition between the registration call returning and the
-corresponding reading of the event from the EQ, the mr output
-parameter will be written before any event associated with the
-operation may be read by the application. An asynchronous event will
-not be generated unless the registration call returns success (0).
-
-## fi_mr_reg
-
-The fi_mr_reg call registers the user-specified memory buffer with
-the resource domain. The buffer is enabled for access by the fabric
-hardware based on the provided access permissions. Supported access
-permissions are the bitwise OR of the following:
-
-*FI_SEND*
-: The memory buffer may be used in outgoing message data transfers. This
- includes fi_msg and fi_tagged operations.
-
-*FI_RECV*
-: The memory buffer may be used to receive inbound message transfers.
- This includes fi_msg and fi_tagged operations.
-
-*FI_READ*
-: The memory buffer may be used as the result buffer for RMA read
- and atomic operations on the initiator side.
-
-*FI_WRITE*
-: The memory buffer may be used as the source buffer for RMA write
- and atomic operations on the initiator side.
-
-*FI_REMOTE_READ*
-: The memory buffer may be used as the source buffer of an RMA read
- operation on the target side.
-
-*FI_REMOTE_WRITE*
-: The memory buffer may be used as the target buffer of an RMA write
- or atomic operation.
-
-Registered memory is associated with a local memory descriptor and,
-optionally, a remote memory key. A memory descriptor is a provider
-specific identifier associated with registered memory. Memory
-descriptors often map to hardware specific indices or keys associated
-with the memory region. Remote memory keys provide limited protection
-against unwanted access by a remote node. Remote accesses to a memory
-region must provide the key associated with the registration.
-
-Because MR keys must be provided by a remote process, an application
-can use the requested_key parameter to indicate that a specific key
-value be returned. Support for user requested keys is provider
-specific and is determined by the mr_mode domain attribute.
-
-Remote RMA and atomic operations indicate the location within a
-registered memory region by specifying an address. The location
-is referenced by adding the offset to either the base virtual address
-of the buffer or to 0, depending on the mr_mode.
-
-The offset parameter is reserved for future use and must be 0.
-
-For asynchronous memory registration requests, the result will be
-reported to the user through an event queue associated with the
-resource domain. If successful, the allocated memory region structure
-will be returned to the user through the mr parameter. The mr address
-must remain valid until the registration operation completes. The
-context specified with the registration request is returned with the
-completion event.
-
-## fi_mr_regv
-
-The fi_mr_regv call adds support for a scatter-gather list to
-fi_mr_reg. Multiple memory buffers are registered as a single memory
-region. Otherwise, the operation is the same.
-
-## fi_mr_regattr
-
-The fi_mr_regattr call is a more generic, extensible registration call
-that allows the user to specify the registration request using a
-struct fi_mr_attr.
-
-```c
-struct fi_mr_attr {
- const struct iovec *mr_iov; /* scatter-gather array */
- size_t iov_count; /* # elements in mr_iov */
- uint64_t access; /* access permission flags */
- uint64_t requested_key; /* requested remote key */
- void *context; /* user-defined context */
-};
-```
-
-## fi_close
-
-Fi_close is used to release all resources associated with a
-registering a memory region. Once unregistered, further access to the
-registered memory is not guaranteed.
-
-When closing the MR, there must be no opened endpoints or counters associated
-with the MR. If resources are still associated with the MR when attempting to
-close, the call will return -FI_EBUSY.
-
-## fi_mr_desc / fi_mr_key
-
-The local memory descriptor and remote protection key associated with
-a MR may be obtained by calling fi_mr_desc and fi_mr_key,
-respectively. The memory registration must have completed
-successfully before invoking these calls.
-
-## fi_mr_bind
-
-The fi_mr_bind function associates a memory region with a
-counter, for providers that support the generation of completions
-based on fabric operations. The type of events tracked against the
-memory region is based on the bitwise OR of the following flags.
-
-*FI_REMOTE_WRITE*
-: Generates an event whenever a remote RMA write or atomic operation
- modify the memory region.
-
-# FLAGS
-
-Flags are reserved for future use and must be 0.
-
-# RETURN VALUES
-
-Returns 0 on success. On error, a negative value corresponding to
-fabric errno is returned.
-
-Fabric errno values are defined in
-`rdma/fi_errno.h`.
-
-# ERRORS
-
-*-FI_ENOKEY*
-: The requested_key is already in use.
-
-*-FI_EKEYREJECTED*
-: The requested_key is not available. They key may be out of the
- range supported by the provider, or the provider may not support
- user-requested memory registration keys.
-
-*-FI_ENOSYS*
-: Returned by fi_mr_bind if the provider does not support reporting
- events based on access to registered memory regions.
-
-*-FI_EBADFLAGS*
-: Returned if the specified flags are not supported by the provider.
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_rma`(3)](fi_rma.3.html),
-[`fi_msg`(3)](fi_msg.3.html),
-[`fi_atomic`(3)](fi_atomic.3.html)
diff --git a/man/fi_msg.3.md b/man/fi_msg.3.md
deleted file mode 100644
index e020577..0000000
--- a/man/fi_msg.3.md
+++ /dev/null
@@ -1,294 +0,0 @@
----
-layout: page
-title: fi_msg(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_msg - Message data transfer operations
-
-fi_recv / fi_recvv / fi_recvmsg
-: Post a buffer to receive an incoming message
-
-fi_send / fi_sendv / fi_sendmsg
-fi_inject / fi_senddata
-: Initiate an operation to send a message
-
-# SYNOPSIS
-
-```c
-#include <rdma/fi_endpoint.h>
-
-ssize_t fi_recv(struct fid_ep *ep, void * buf, size_t len,
- void *desc, fi_addr_t src_addr, void *context);
-
-ssize_t fi_recvv(struct fid_ep *ep, const struct iovec *iov, void **desc,
- size_t count, fi_addr_t src_addr, void *context);
-
-ssize_t fi_recvmsg(struct fid_ep *ep, const struct fi_msg *msg,
- uint64_t flags);
-
-ssize_t fi_send(struct fid_ep *ep, void *buf, size_t len,
- void *desc, fi_addr_t dest_addr, void *context);
-
-ssize_t fi_sendv(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t dest_addr, void *context);
-
-ssize_t fi_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
- uint64_t flags);
-
-ssize_t fi_inject(struct fid_ep *ep, void *buf, size_t len,
- fi_addr_t dest_addr);
-
-ssize_t fi_senddata(struct fid_ep *ep, void *buf, size_t len,
- void *desc, uint64_t data, fi_addr_t dest_addr, void *context);
-
-ssize_t fi_injectdata(struct fid_ep *ep, void *buf, size_t len,
- uint64_t data, fi_addr_t dest_addr);
-```
-
-# ARGUMENTS
-
-*ep*
-: Fabric endpoint on which to initiate send or post receive buffer.
-
-*buf*
-: Data buffer to send or receive.
-
-*len*
-: Length of data buffer to send or receive, specified in bytes. Valid
- transfers are from 0 bytes up to the endpoint's max_msg_size.
-
-*iov*
-: Vectored data buffer.
-
-*count*
-: Count of vectored data entries.
-
-*desc*
-: Descriptor associated with the data buffer
-
-*data*
-: Remote CQ data to transfer with the sent message.
-
-*dest_addr*
-: Destination address for connectionless transfers. Ignored for
- connected endpoints.
-
-*src_addr*
-: Source address to receive from for connectionless transfers. Applies
- only to connectionless endpoints with the FI_DIRECTED_RECV capability
- enabled, otherwise this field is ignored. If set to FI_ADDR_UNSPEC,
- any source address may match.
-
-*msg*
-: Message descriptor for send and receive operations.
-
-*flags*
-: Additional flags to apply for the send or receive operation.
-
-*context*
-: User specified pointer to associate with the operation.
-
-# DESCRIPTION
-
-The send functions -- fi_send, fi_sendv, fi_sendmsg,
-fi_inject, and fi_senddata -- are used to
-transmit a message from one endpoint to another endpoint. The main
-difference between send functions are the number and type of
-parameters that they accept as input. Otherwise, they perform the
-same general function. Messages sent using fi_msg operations are
-received by a remote endpoint into a buffer posted to receive such
-messages.
-
-The receive functions -- fi_recv, fi_recvv, fi_recvmsg --
-post a data buffer to an endpoint to receive inbound messages.
-Similar to the send operations, receive operations operate
-asynchronously. Users should not touch the posted data buffer(s)
-until the receive operation has completed.
-
-Completed message operations are reported to the user through one or
-more event collectors associated with the endpoint. Users provide
-context which are associated with each operation, and is returned to
-the user as part of the event completion. See fi_cq for completion
-event details.
-
-## fi_send
-
-The call fi_send transfers the data contained in the user-specified
-data buffer to a remote endpoint, with message boundaries being
-maintained. The local endpoint must be connected to a remote endpoint
-or destination before fi_send is called. Unless the endpoint has been
-configured differently, the data buffer passed into fi_send must not
-be touched by the application until the fi_send call completes
-asynchronously.
-
-## fi_sendv
-
-The fi_sendv call adds support for a scatter-gather list to fi_send.
-The fi_sendv transfers the set of data buffers
-referenced by the iov parameter to a remote endpoint as a single
-message.
-
-## fi_sendmsg
-
-The fi_sendmsg call supports data transfers over both connected and
-unconnected endpoints, with the ability to control the send operation
-per call through the use of flags. The fi_sendmsg function takes a
-`struct fi_msg` as input.
-
-```c
-struct fi_msg {
- const struct iovec *msg_iov; /* scatter-gather array */
- void **desc; /* local request descriptors */
- size_t iov_count;/* # elements in iov */
- fi_addr_t addr; /* optional endpoint address */
- void *context; /* user-defined context */
- uint64_t data; /* optional message data */
-};
-```
-
-## fi_inject
-
-The send inject call is an optimized version of fi_send. The
-fi_inject function behaves as if the FI_INJECT transfer flag were
-set, and FI_COMPLETION were not. That is, the data buffer is
-available for reuse immediately on returning from from fi_inject, and
-no completion event will be generated for this send. The completion
-event will be suppressed even if the CQ was bound without
-FI_SELECTIVE_COMPLETION or the endpoint's op_flags contain
-FI_COMPLETION. See the flags discussion below for more details.
-
-## fi_senddata
-
-The send data call is similar to fi_send, but allows for the sending
-of remote CQ data (see FI_REMOTE_CQ_DATA flag) as part of the
-transfer.
-
-## fi_injectdata
-
-The inject data call is similar to fi_inject, but allows for the sending
-of remote CQ data (see FI_REMOTE_CQ_DATA flag) as part of the
-transfer.
-
-## fi_recv
-
-The fi_recv call posts a data buffer to the receive queue of the
-corresponding endpoint. Posted receives are searched in the order in
-which they were posted in order to match sends.
-Message boundaries are maintained. The order in which
-the receives complete is dependent on
-the endpoint type and protocol. For unconnected endpoints, the
-src_addr parameter can be used to indicate that a buffer should be
-posted to receive incoming data from a specific remote endpoint.
-
-## fi_recvv
-
-The fi_recvv call adds support for a scatter-gather list to fi_recv.
-The fi_recvv posts the set of data buffers referenced by the iov
-parameter to a receive incoming data.
-
-## fi_recvmsg
-
-The fi_recvmsg call supports posting buffers over both connected and
-unconnected endpoints, with the ability to control the receive
-operation per call through the use of flags. The fi_recvmsg function
-takes a struct fi_msg as input.
-
-# FLAGS
-
-The fi_recvmsg and fi_sendmsg calls allow the user to specify flags
-which can change the default message handling of the endpoint. Flags
-specified with fi_recvmsg / fi_sendmsg override most flags previously
-configured with the endpoint, except where noted (see fi_endpoint.3).
-The following list of flags are usable with fi_recvmsg and/or
-fi_sendmsg.
-
-*FI_REMOTE_CQ_DATA*
-: Applies to fi_sendmsg and fi_senddata. Indicates
- that remote CQ data is available and should be sent as part of the
- request. See fi_getinfo for additional details on
- FI_REMOTE_CQ_DATA.
-
-*FI_COMPLETION*
-: Indicates that a completion entry should be generated for the
- specified operation. The endpoint must be bound to a completion
- queue with FI_SELECTIVE_COMPLETION that corresponds to the
- specified operation, or this flag is ignored.
-
-*FI_MORE*
-: Indicates that the user has additional requests that will
- immediately be posted after the current call returns. Use of this
- flag may improve performance by enabling the provider to optimize
- its access to the fabric hardware.
-
-*FI_INJECT*
-: Applies to fi_sendmsg. Indicates that the outbound data buffer
- should be returned to user immediately after the send call returns,
- even if the operation is handled asynchronously. This may require
- that the underlying provider implementation copy the data into a
- local buffer and transfer out of that buffer.
-
-*FI_MULTI_RECV*
-: Applies to posted receive operations. This flag allows the user to
- post a single buffer that will receive multiple incoming messages.
- Received messages will be packed into the receive buffer until the
- buffer has been consumed. Use of this flag may cause a single
- posted receive operation to generate multiple events as messages are
- placed into the buffer. The placement of received data into the
- buffer may be subjected to provider specific alignment restrictions.
- The buffer will be freed from the endpoint when the available buffer
- space falls below the network's MTU size (see
- FI_OPT_MIN_MULTI_RECV).
-
-*FI_INJECT_COMPLETE*
-: Applies to fi_sendmsg. Indicates that a completion should be
- generated when the source buffer(s) may be reused.
-
-*FI_TRANSMIT_COMPLETE*
-: Applies to fi_sendmsg. Indicates that a completion should not be
- generated until the operation has been successfully transmitted and
- is no longer being tracked by the provider.
-
-*FI_DELIVERY_COMPLETE*
-: Applies to fi_sendmsg. Indicates that a completion should be
- generated when the operation has been processed by the destination.
-
-*FI_FENCE*
-: Applies to transmits. Indicates that the requested operation, also
- known as the fenced operation, be deferred until all previous operations
- targeting the same target endpoint have completed.
-
-# NOTES
-
-If an endpoint has been configured with FI_MSG_PREFIX, the application
-must include buffer space of size msg_prefix_size, as specified by the
-endpoint attributes. The prefix buffer must occur at the start of the
-data referenced by the buf parameter, or be referenced by the first IO vector.
-Message prefix space cannot be split between multiple IO vectors. The size
-of the prefix buffer should be included as part of the total buffer length.
-
-# RETURN VALUE
-
-Returns 0 on success. On error, a negative value corresponding to fabric
-errno is returned. Fabric errno values are defined in
-`rdma/fi_errno.h`.
-
-# ERRORS
-
-*-FI_EAGAIN*
-: Indicates that the underlying provider currently lacks the resources
- needed to initiate the requested operation. This may be the result
- of insufficient internal buffering, in the case of FI_INJECT,
- or processing queues are full. The operation may be retried after
- additional provider resources become available, usually through the
- completion of currently outstanding operations.
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_cq`(3)](fi_cq.3.html)
diff --git a/man/fi_mxm.7.md b/man/fi_mxm.7.md
deleted file mode 100644
index ab768d5..0000000
--- a/man/fi_mxm.7.md
+++ /dev/null
@@ -1,60 +0,0 @@
----
-layout: page
-title: fi_mxm(7)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-The MXM Fabric Provider
-
-# OVERVIEW
-
-The *mxm* provider runs over the MXM (Mellanox messaging) interface
-that is currently supported by the Mellanox infiniband fabrics.
-The *mxm* provider makes use of MXM tag matching API in order to
-implement a limited set of the libfabric data transfer APIs, namely,
-tagged message queue.
-
-# LIMITATIONS
-
-The *mxm* provider doesn't support all the features defined in the
-libfabric API. Here are some of the limitations:
-
-Endpoint types
-: Only supported type: *FI_RDM*
-
-Endpoint capabilities
-: Endpoints can support the only data transfer capability
- *FI_TAGGED*.
-
-
-Modes
-: *FI_CONTEXT* is required. That means, all the requests that generate
- completions must have a valid pointer to type *struct fi_context*
- passed as the operation context.
-
-Threading
-: The supported mode is FI_THREAD_DOMAIN, i.e. the *mxm* provider is not thread safe.
-
-
-Unsupported features
-: These features are unsupported: connection management, event queue,
- scalable endpoint, passive endpoint, shared receive context,
- rma, atomics.
-
-Mem tag format
-: MXM library provides a tag matching interface with just 32 bits wide tag.
- Another 16 bits are available for matching via specifying an MXM MQ. Hence
- total maximum available matching bits equal 48. This is why the only allowed
- mem_tag_format is 0xFFFF00000000.
-
-
-# RUNTIME PARAMETERS
-
-
-# SEE ALSO
-
-[`fabric`(7)](fabric.7.html),
-[`fi_provider`(7)](fi_provider.7.html),
diff --git a/man/fi_poll.3.md b/man/fi_poll.3.md
deleted file mode 100644
index eb93e75..0000000
--- a/man/fi_poll.3.md
+++ /dev/null
@@ -1,303 +0,0 @@
----
-layout: page
-title: fi_poll(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_poll \- Polling and wait set operations
-
-fi_poll_open / fi_close
-: Open/close a polling set
-
-fi_poll_add / fi_poll_del
-: Add/remove a completion queue or counter to/from a poll set.
-
-fi_poll
-: Poll for progress and events across multiple completion queues
- and counters.
-
-fi_wait_open / fi_close
-: Open/close a wait set
-
-fi_wait
-: Waits for one or more wait objects in a set to be signaled.
-
-fi_trywait
-: Indicate when it is safe to block on wait objects using native OS calls.
-
-fi_control
-: Control wait set operation or attributes.
-
-# SYNOPSIS
-
-```c
-#include <rdma/fi_domain.h>
-
-int fi_poll_open(struct fid_domain *domain, struct fi_poll_attr *attr,
- struct fid_poll **pollset);
-
-int fi_close(struct fid *pollset);
-
-int fi_poll_add(struct fid_poll *pollset, struct fid *event_fid,
- uint64_t flags);
-
-int fi_poll_del(struct fid_poll *pollset, struct fid *event_fid,
- uint64_t flags);
-
-int fi_poll(struct fid_poll *pollset, void **context, int count);
-
-int fi_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
- struct fid_wait **waitset);
-
-int fi_close(struct fid *waitset);
-
-int fi_wait(struct fid_wait *waitset, int timeout);
-
-int fi_trywait(struct fid_fabric *fabric, struct fid **fids, size_t count);
-
-int fi_control(struct fid *waitset, int command, void *arg);
-```
-
-# ARGUMENTS
-
-*fabric*
-: Fabric provider
-
-*domain*
-: Resource domain
-
-*pollset*
-: Event poll set
-
-*waitset*
-: Wait object set
-
-*attr*
-: Poll or wait set attributes
-
-*context*
-: On success, an array of user context values associated with
- completion queues or counters.
-
-*fids*
-: An array of fabric descriptors, each one associated with a native
- wait object.
-
-*count*
-: Number of entries in context or fids array.
-
-*timeout*
-: Time to wait for a signal, in milliseconds.
-
-*command*
-: Command of control operation to perform on the wait set.
-
-*arg*
-: Optional control argument.
-
-# DESCRIPTION
-
-
-## fi_poll_open
-
-fi_poll_open creates a new polling set. A poll set enables an
-optimized method for progressing asynchronous operations across
-multiple completion queues and counters and checking for their completions.
-
-A poll set is defined with the following attributes.
-
-```c
-struct fi_poll_attr {
- uint64_t flags; /* operation flags */
-};
-```
-
-*flags*
-: Flags that set the default operation of the poll set. The use of
- this field is reserved and must be set to 0 by the caller.
-
-## fi_close
-
-The fi_close call releases all resources associated with a poll set.
-The poll set must not be associated with any other resources prior to
-being closed, otherwise the call will return -FI_EBUSY.
-
-## fi_poll_add
-
-Associates a completion queue or counter with a poll set.
-
-## fi_poll_del
-
-Removes an completion queue or counter from a poll set.
-
-## fi_poll
-
-Progresses all completion queues and counters associated with a poll set
-and checks for events. If events have occurred, contexts associated
-with the completion queues and/or counters are returned. Completion
-queues will return their context if they are not empty. The context
-associated with a counter will be returned if the counter's success
-value or error value have changed since the last time fi_poll was
-called. The number of contexts is limited to the size of the context
-array, indicated by the count parameter.
-
-## fi_wait_open
-
-fi_wait_open allocates a new wait set. A wait set enables an
-optimized method of waiting for events across multiple completion queues
-and counters. Where possible, a wait set uses a single underlying
-wait object that is signaled when a specified condition occurs on an
-associated completion queue or counter.
-
-The properties and behavior of a wait set are defined by struct
-fi_wait_attr.
-
-```c
-struct fi_wait_attr {
- enum fi_wait_obj wait_obj; /* requested wait object */
- uint64_t flags; /* operation flags */
-};
-```
-
-*wait_obj*
-: Wait sets are associated with specific wait object(s). Wait objects
- allow applications to block until the wait object is signaled,
- indicating that an event is available to be read. The following
- values may be used to specify the type of wait object associated
- with a wait set: FI_WAIT_UNSPEC, FI_WAIT_FD, and FI_WAIT_MUTEX_COND.
-
-- *FI_WAIT_UNSPEC*
-: Specifies that the user will only wait on the wait set using
- fabric interface calls, such as fi_wait. In this case, the
- underlying provider may select the most appropriate or highest
- performing wait object available, including custom wait mechanisms.
- Applications that select FI_WAIT_UNSPEC are not guaranteed to
- retrieve the underlying wait object.
-
-- *FI_WAIT_FD*
-: Indicates that the wait set should use file descriptor(s) as its wait
- mechanism. It may not always be possible for a wait set to be implemented
- using a single underlying file descriptor, but all wait objects will be file
- descriptors. File descriptor wait objects must be usable in the
- POSIX select(2), poll(2), and epoll(7) routines (if
- available). However, a provider may signal an FD wait object by
- marking it as readable or with an error.
-
-- *FI_WAIT_MUTEX_COND*
-: Specifies that the wait set should use a pthread mutex and cond
- variable as a wait object.
-
-*flags*
-: Flags that set the default operation of the wait set. The use of
- this field is reserved and must be set to 0 by the caller.
-
-## fi_close
-
-The fi_close call releases all resources associated with a wait set.
-The wait set must not be bound to any other opened resources prior to
-being closed, otherwise the call will return -FI_EBUSY.
-
-## fi_wait
-
-Waits on a wait set until one or more of its underlying wait objects
-is signaled.
-
-## fi_trywait
-
-The fi_trywait call was introduced in libfabric version 1.3. The behavior
-of using native wait objects without the use of fi_trywait is provider
-specific and should be considered non-deterministic.
-
-The fi_trywait() call is used in conjunction with native operating
-system calls to block on wait objects, such as file descriptors. The
-application must call fi_trywait and obtain a return value of
-FI_SUCCESS prior to blocking on a native wait object. Failure to
-do so may result in the wait object not being signaled, and the
-application not observing the desired events. The following
-pseudo-code demonstrates the use of fi_trywait in conjunction with
-the OS select(2) call.
-
-```c
-fi_control(&cq->fid, FI_GETWAIT, (void *) &fd);
-FD_ZERO(&fds);
-FD_SET(fd, &fds);
-
-while (1) {
- if (fi_trywait(&cq, 1) == FI_SUCCESS)
- select(fd + 1, &fds, NULL, &fds, &timeout);
-
- do {
- ret = fi_cq_read(cq, &comp, 1);
- } while (ret > 0);
-}
-```
-
-fi_trywait() will return FI_SUCCESS if it is safe to block on the wait object(s)
-corresponding to the fabric descriptor(s), or -FI_EAGAIN if there are
-events queued on the fabric descriptor or if blocking could hang the
-application.
-
-The call takes an array of fabric descriptors. For each wait object
-that will be passed to the native wait routine, the corresponding
-fabric descriptor should first be passed to fi_trywait. All fabric
-descriptors passed into a single fi_trywait call must make use of the
-same underlying wait object type.
-
-The following types of fabric descriptors may be passed into fi_trywait:
-event queues, completion queues, counters, and wait sets. Applications
-that wish to use native wait calls should select specific wait objects
-when allocating such resources. For example, by setting the item's
-creation attribute wait_obj value to FI_WAIT_FD.
-
-In the case the wait object to check belongs to a wait set, only
-the wait set itself needs to be passed into fi_trywait. The fabric
-resources associated with the wait set do not.
-
-On receiving a return value of -FI_EAGAIN from fi_trywait, an application
-should read all queued completions and events, and call fi_trywait again
-before attempting to block. Applications can make use of a fabric
-poll set to identify completion queues and counters that may require
-processing.
-
-## fi_control
-
-The fi_control call is used to access provider or implementation specific
-details of the wait set. Access to the wait set should be serialized across
-all calls when fi_control is invoked, as it may redirect the implementation
-of wait set operations. The following control commands are usable with a
-wait set.
-
-*FI_GETWAIT (void \*\*)*
-: This command allows the user to retrieve the low-level wait object
- associated with the wait set. The format of the wait set is specified
- during wait set creation, through the wait set attributes. The fi_control
- arg parameter should be an address where a pointer to the returned wait
- object will be written. This should be an 'int *' for FI_WAIT_FD,
- or 'struct fi_mutex_cond' for FI_WAIT_MUTEX_COND. Support for FI_GETWAIT
- is provider specific and may fail if not supported or if the wait set is
- implemented using more than one wait object.
-
-# RETURN VALUES
-
-Returns FI_SUCCESS on success. On error, a negative value corresponding to
-fabric errno is returned.
-
-Fabric errno values are defined in
-`rdma/fi_errno.h`.
-
-fi_poll
-: On success, if events are available, returns the number of entries
- written to the context array.
-
-# NOTES
-
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_cntr`(3)](fi_cntr.3.html),
-[`fi_eq`(3)](fi_eq.3.html)
diff --git a/man/fi_provider.7.md b/man/fi_provider.7.md
deleted file mode 100644
index 945c539..0000000
--- a/man/fi_provider.7.md
+++ /dev/null
@@ -1,166 +0,0 @@
----
-layout: page
-title: fi_provider(7)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-Fabric Interface Providers
-
-# OVERVIEW
-
-Conceptually, a fabric provider may be viewed as a local hardware NIC
-driver, though a provider is not limited by this definition. The
-first component of libfabric is a general purpose framework that is
-capable of handling different types of fabric hardware. All fabric
-hardware devices and their software drivers are required to support
-this framework. Devices and the drivers that plug into the libfabric
-framework are referred to as fabric providers, or simply providers.
-
-This distribution of libfabric contains the following providers
-(although more may be available via run-time plug-ins):
-
-*GNI*
-: A provider for the Aries interconnect in Cray XC(TM) systems
- utilizing the user-space *Generic Networking Interface*. See
- [`fi_gni`(7)](fi_gni.7.html) for more information.
-
-*PSM*
-: High-speed InfiniBand networking from Intel. See
- [`fi_psm`(7)](fi_psm.7.html) for more information.
-
-*Sockets*
-: A general purpose provider that can be used on any network that
- supports TCP/UDP sockets. This provider is not intended to provide
- performance improvements over regular TCP/UDP sockets, but rather to
- allow developers to write, test, and debug application code even on
- platforms that do not have high-speed networking.
- See [`fi_sockets`(7)](fi_sockets.7.html) for more information.
-
-*usNIC*
-: Ultra low latency Ethernet networking over Cisco userspace VIC
- adapters.
- See [`fi_usnic`(7)](fi_usnic.7.html) for more information.
-
-*Verbs*
-: This provider uses the Linux Verbs API for network transport.
- Application performance is, obviously expected to be similar to that
- of the native Linux Verbs API. Analogous to the Sockets provider,
- the Verbs provider is intended to enable developers to write, test,
- and debug application code on platforms that only have Linux
- Verbs-based networking.
- See [`fi_verbs`(7)](fi_verbs.7.html) for more information.
-
-# PROVIDER REQUIREMENTS
-
-Libfabric provides a general framework for supporting multiple types
-of fabric objects and their related interfaces. Fabric providers have
-a large amount of flexibility in selecting which components they are
-able and willing to support, based on specific hardware constraints.
-Provider developers should refer to docs/provider for information on
-functionaliy supplied by the framework to assist in provider
-implementation. To assist in the development of applications,
-libfabric specifies the
-following requirements that must be met by any fabric provider, if
-requested by an application.
-
-Note that the instantiation of a specific fabric object is subject
-to application configuration parameters and need not meet these requirements.
-
-* A fabric provider must support at least one endpoint type.
-* All endpoints must support the message queue data transfer
- interface (fi_ops_msg).
-* An endpoint that advertises support for a specific endpoint
- capability must support the corresponding data transfer interface.
- * FI_ATOMIC - fi_ops_atomic
- * FI_RMA - fi_ops_rma
- * FI_TAGGED - fi_ops_tagged
-* Endpoints must support all transmit and receive operations for any
- data transfer interface that they support.
- * Exception: If an operation is only usable for an operation that
- the provider does not support, and support for that operation is
- conveyed using some other mechanism, the operation may return
- - FI_ENOSYS. For example, if the provider does not support
- injected data, it can set the attribute inject_size = 0, and fail
- all fi_inject operations.
- * The framework supplies wrappers around the 'msg' operations that
- can be used. For example, the framework implements the sendv()
- msg operation by calling sendmsg(). Providers may reference the
- general operation, and supply on the sendmsg() implementation.
-* Providers must set all operations to an implementation. Function
- pointers may not be left NULL or uninitialized. The framework supplies
- empty functions that return -FI_ENOSYS which can be used for this
- purpose.
-* Endpoints must support the CM interface as follows:
- * FI_EP_MSG endpoints must support all CM operations.
- * FI_EP_DGRAM endpoints must support CM getname and setname.
- * FI_EP_RDM endpoints must support CM getname and setname.
-* Providers that support connectionless endpoints must support all AV
- operations (fi_ops_av).
-* Providers that support memory registration, must support all MR operations
- (fi_ops_mr).
-* Providers should support both completion queues and counters.
- * If FI_RMA_EVENT is not supported, counter support is limited to local
- events only.
- * Completion queues must support the FI_CQ_FORMAT_CONTEXT and
- FI_CQ_FORMAT_MSG.
- * Providers that support FI_REMOTE_CQ_DATA shall support FI_CQ_FORMAT_DATA.
- * Providers that support FI_TAGGED shall support FI_CQ_FORMAT_TAGGED.
-* A provider is expected to be forward compatible, and must be able to
- be compiled against expanded `fi_xxx_ops` structures that define new
- functions added after the provider was written. Any unknown
- functions must be set to NULL.
-* Providers shall document in their man page which features they support,
- and any missing requirements.
-
-Future versions of libfabric will automatically enable a more complete
-set of features for providers that focus their implementation on a
-narrow subset of libfabric capabilities.
-
-# LOGGING INTERFACE
-
-Logging is performed using the FI_ERR, FI_LOG, and FI_DEBUG macros.
-
-## DEFINITIONS
-
-```c
-#define FI_ERR(prov_name, subsystem, ...)
-
-#define FI_LOG(prov_name, prov, level, subsystem, ...)
-
-#define FI_DEBUG(prov_name, subsystem, ...)
-```
-
-## ARGUMENTS
-*prov_name*
-: String representing the provider name.
-
-*prov*
-: Provider context structure.
-
-*level*
-: Log level associated with log statement.
-
-*subsystem*
-: Subsystem being logged from.
-
-## DESCRIPTION
-*FI_ERR*
-: Always logged.
-
-*FI_LOG*
-: Logged if the intended provider, log level, and subsystem parameters match
- the user supplied values.
-
-*FI_DEBUG*
-: Logged if configured with the --enable-debug flag.
-
-# SEE ALSO
-
-[`fi_gni`(7)](fi_gni.7.html),
-[`fi_psm`(7)](fi_psm.7.html),
-[`fi_sockets`(7)](fi_sockets.7.html),
-[`fi_usnic`(7)](fi_usnic.7.html),
-[`fi_verbs`(7)](fi_verbs.7.html),
diff --git a/man/fi_psm.7.md b/man/fi_psm.7.md
deleted file mode 100644
index 5ed10ae..0000000
--- a/man/fi_psm.7.md
+++ /dev/null
@@ -1,169 +0,0 @@
----
-layout: page
-title: fi_psm(7)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-The PSM Fabric Provider
-
-# OVERVIEW
-
-The *psm* provider runs over the PSM 1.x interface that is currently
-supported by the Intel TrueScale Fabric. PSM provides tag-matching
-message queue functions that are optimized for MPI implementations.
-PSM also has limited Active Message support, which is not officially
-published but is quite stable and well documented in the source code
-(part of the OFED release). The *psm* provider makes use of both the
-tag-matching message queue functions and the Active Message functions
-to support a variety of libfabric data transfer APIs, including tagged
-message queue, message queue, RMA, and atomic operations.
-
-The *psm* provider can work with the psm2-compat library, which exposes
-a PSM 1.x interface over the Intel Omni-Path Fabric.
-
-# LIMITATIONS
-
-The *psm* provider doesn't support all the features defined in the
-libfabric API. Here are some of the limitations:
-
-Endpoint types
-: Only support non-connection based types *FI_DGRAM* and *FI_RDM*
-
-Endpoint capabilities
-: Endpoints can support any combination of data transfer capabilities
- *FI_TAGGED*, *FI_MSG*, *FI_ATOMICS*, and *FI_RMA*. These capabilities
- can be further refined by *FI_SEND*, *FI_RECV*, *FI_READ*, *FI_WRITE*,
- *FI_REMOTE_READ*, and *FI_REMOTE_WRITE* to limit the direction of
- operations. The limitation is that no two endpoints can have overlapping
- receive or RMA target capabilities in any of the above categories. For
- example it is fine to have two endpoints with *FI_TAGGED* | *FI_SEND*,
- one endpoint with *FI_TAGGED* | *FI_RECV*, one endpoint with *FI_MSG*,
- one endpoint with *FI_RMA* | *FI_ATOMICS*. But it is not allowed to
- have two endpoints with *FI_TAGGED*, or two endpoints with *FI_RMA*.
-
- *FI_MULTI_RECV* is supported for non-tagged message queue only.
-
- Other supported capabilities include *FI_TRIGGER*.
-
-Modes
-: *FI_CONTEXT* is required for the *FI_TAGGED* and *FI_MSG*
- capabilities. That means, any request belonging to these two
- categories that generates a completion must pass as the operation
- context a valid pointer to type *struct fi_context*, and the space
- referenced by the pointer must remain untouched until the request
- has completed. If none of *FI_TAGGED* and *FI_MSG* is asked for,
- the *FI_CONTEXT* mode is not required.
-
-Progress
-: The *psm* provider requires manual progress. The application is
- expected to call *fi_cq_read* or *fi_cntr_read* function from time
- to time when no other libfabric function is called to ensure
- progress is made in a timely manner. The provider does support
- auto progress mode. However, the performance can be significantly
- impacted if the application purely depends on the provider to
- make auto progress.
-
-Unsupported features
-: These features are unsupported: connection management,
- scalable endpoint, passive endpoint, shared receive context,
- send/inject with immediate data.
-
-# RUNTIME PARAMETERS
-
-The *psm* provider checks for the following environment variables:
-
-*FI_PSM_UUID*
-: PSM requires that each job has a unique ID (UUID). All the processes
- in the same job need to use the same UUID in order to be able to
- talk to each other. The PSM reference manual advises to keep UUID
- unique to each job. In practice, it generally works fine to reuse
- UUID as long as (1) no two jobs with the same UUID are running at
- the same time; and (2) previous jobs with the same UUID have exited
- normally. If running into "resource busy" or "connection failure"
- issues with unknown reason, it is advisable to manually set the UUID
- to a value different from the default.
-
- The default UUID is 0FFF0FFF-0000-0000-0000-0FFF0FFF0FFF.
-
-*FI_PSM_NAME_SERVER*
-: The *psm* provider has a simple built-in name server that can be used
- to resolve an IP address or host name into a transport address needed
- by the *fi_av_insert* call. The main purpose of this name server is to
- allow simple client-server type applications (such as those in *fabtest*)
- to be written purely with libfabric, without using any out-of-band
- communication mechanism. For such applications, the server would run first,
- and the client would call *fi_getinfo* with the *node* parameter set to
- the IP address or host name of the server. The resulting *fi_info* structure
- would have the transport address of the server in the *dest_addr* field.
-
- The name server won't work properly if there are more than one processes
- from the same job (i.e. with the same UUID) running on the same node and
- acting as servers. For such scenario it is recommended to have each
- process getting local transport address with *fi_getname* and exchanging
- the addresses with out-of-band mechanism.
-
- The name server is on by default. It can be turned off by setting the
- variable to 0. This may save a small amount of resource since a separate
- thread is created when the name server is on.
-
- The provider detects OpenMPI and MPICH runs and changes the default setting
- to off.
-
-*FI_PSM_TAGGED_RMA*
-: The RMA functions are implemented on top of the PSM Active Message functions.
- The Active Message functions have limit on the size of data can be transferred
- in a single message. Large transfers can be divided into small chunks and
- be pipe-lined. However, the bandwidth is sub-optimal by doing this way.
-
- The *psm* provider use PSM tag-matching message queue functions to achieve
- higher bandwidth for large size RMA. For this purpose, a bit is reserved from
- the tag space to separate the RMA traffic from the regular tagged message queue.
-
- The option is on by default. To turn it off set the variable to 0.
-
-*FI_PSM_AM_MSG*
-: The *psm* provider implements the non-tagged message queue over the PSM
- tag-matching message queue. One tag bit is reserved for this purpose.
- Alternatively, the non-tagged message queue can be implemented over
- Active Message. This experimental feature has slightly larger latency.
-
- This option is off by default. To turn it on set the variable to 1.
-
-*FI_PSM_DELAY*
-: Time (seconds) to sleep before closing PSM endpoints. This is a workaround
- for a bug in some versions of PSM library.
-
- The default setting is 1.
-
-*FI_PSM_TIMEOUT*
-: Timeout (seconds) for gracefully closing PSM endpoints. A forced closing
- will be issued if timeout expires.
-
- The default setting is 5.
-
-*FI_PSM_PROG_INTERVAL*
-: When auto progress is enabled (asked via the hints to *fi_getinfo*),
- a progress thread is created to make progress calls from time to time.
- This option set the interval (microseconds) between progress calls.
-
- The default setting is 1 if affininty is set, or 1000 if not. See
- *FI_PSM_PROG_AFFINITY*.
-
-*FI_PSM_PROG_AFFINITY*
-: When set, specify the set of CPU cores to set the progress thread
- affinity to. The format is
- `<start>[:<end>[:<stride>]][,<start>[:<end>[:<stride>]]]*`,
- where each triplet `<start>:<end>:<stride>` defines a block of
- core_ids. Both `<start>` and `<end>` can be either the `core_id`
- (when >=0) or `core_id - num_cores` (when <0).
-
- By default affinity is not set.
-
-# SEE ALSO
-
-[`fabric`(7)](fabric.7.html),
-[`fi_provider`(7)](fi_provider.7.html),
-[`fi_psm2`(7)](fi_psm2.7.html),
diff --git a/man/fi_psm2.7.md b/man/fi_psm2.7.md
deleted file mode 100644
index 38cc629..0000000
--- a/man/fi_psm2.7.md
+++ /dev/null
@@ -1,151 +0,0 @@
----
-layout: page
-title: fi_psm2(7)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-The PSM2 Fabric Provider
-
-# OVERVIEW
-
-The *psm2* provider runs over the PSM 2.x interface that is supported
-by the Intel Omni-Path Fabric. PSM 2.x has all the PSM 1.x features
-plus a set of new functions with enhanced capabilities. Since PSM 1.x
-and PSM 2.x are not ABI compatible the *psm2* provider only works with
-PSM 2.x and doesn't support Intel TrueScale Fabric.
-
-# LIMITATIONS
-
-The *psm2* provider doesn't support all the features defined in the
-libfabric API. Here are some of the limitations:
-
-Endpoint types
-: Only support non-connection based types *FI_DGRAM* and *FI_RDM*
-
-Endpoint capabilities
-: Endpoints can support any combination of data transfer capabilities
- *FI_TAGGED*, *FI_MSG*, *FI_ATOMICS*, and *FI_RMA*. These capabilities
- can be further refined by *FI_SEND*, *FI_RECV*, *FI_READ*, *FI_WRITE*,
- *FI_REMOTE_READ*, and *FI_REMOTE_WRITE* to limit the direction of
- operations.
-
- *FI_MULTI_RECV* is supported for non-tagged message queue only.
-
- Other supported capabilities include *FI_TRIGGER*, *FI_REMOTE_CQ_DATA*,
- and *FI_SOURCE*.
-
-Modes
-: *FI_CONTEXT* is required for the *FI_TAGGED* and *FI_MSG*
- capabilities. That means, any request belonging to these two
- categories that generates a completion must pass as the operation
- context a valid pointer to type *struct fi_context*, and the space
- referenced by the pointer must remain untouched until the request
- has completed. If none of *FI_TAGGED* and *FI_MSG* is asked for,
- the *FI_CONTEXT* mode is not required.
-
-Progress
-: The *psm2* provider requires manual progress. The application is
- expected to call *fi_cq_read* or *fi_cntr_read* function from time
- to time when no other libfabric function is called to ensure
- progress is made in a timely manner. The provider does support
- auto progress mode. However, the performance can be significantly
- impacted if the application purely depends on the provider to
- make auto progress.
-
-Unsupported features
-: These features are unsupported: connection management,
- scalable endpoint, passive endpoint, shared receive context,
- and send/inject with immediate data over tagged message queue.
-
-# RUNTIME PARAMETERS
-
-The *psm2* provider checks for the following environment variables:
-
-*FI_PSM2_UUID*
-: PSM requires that each job has a unique ID (UUID). All the processes
- in the same job need to use the same UUID in order to be able to
- talk to each other. The PSM reference manual advises to keep UUID
- unique to each job. In practice, it generally works fine to reuse
- UUID as long as (1) no two jobs with the same UUID are running at
- the same time; and (2) previous jobs with the same UUID have exited
- normally. If running into "resource busy" or "connection failure"
- issues with unknown reason, it is advisable to manually set the UUID
- to a value different from the default.
-
- The default UUID is 00FF00FF-0000-0000-0000-00FF0F0F00FF.
-
-*FI_PSM2_NAME_SERVER*
-: The *psm2* provider has a simple built-in name server that can be used
- to resolve an IP address or host name into a transport address needed
- by the *fi_av_insert* call. The main purpose of this name server is to
- allow simple client-server type applications (such as those in *fabtest*)
- to be written purely with libfabric, without using any out-of-band
- communication mechanism. For such applications, the server would run first,
- and the client would call *fi_getinfo* with the *node* parameter set to
- the IP address or host name of the server. The resulting *fi_info* structure
- would have the transport address of the server in the *dest_addr* field.
-
- The name server won't work properly if there are more than one processes
- from the same job (i.e. with the same UUID) running on the same node and
- acting as servers. For such scenario it is recommended to have each
- process getting local transport address with *fi_getname* and exchanging
- the addresses with out-of-band mechanism.
-
- The name server is on by default. It can be turned off by setting the
- variable to 0. This may save a small amount of resource since a separate
- thread is created when the name server is on.
-
- The provider detects OpenMPI and MPICH runs and changes the default setting
- to off.
-
-*FI_PSM2_TAGGED_RMA*
-: The RMA functions are implemented on top of the PSM Active Message functions.
- The Active Message functions have limit on the size of data can be transferred
- in a single message. Large transfers can be divided into small chunks and
- be pipe-lined. However, the bandwidth is sub-optimal by doing this way.
-
- The *psm2* provider use PSM tag-matching message queue functions to achieve
- higher bandwidth for large size RMA. It takes avdantage of the extra tag bits
- available in PSM2 to separate the RMA traffic from the regular tagged message
- queue.
-
- The option is on by default. To turn it off set the variable to 0.
-
-*FI_PSM2_DELAY*
-: Time (seconds) to sleep before closing PSM endpoints. This is a workaround
- for a bug in some versions of PSM library.
-
- The default setting is 1.
-
-*FI_PSM2_TIMEOUT*
-: Timeout (seconds) for gracefully closing PSM endpoints. A forced closing
- will be issued if timeout expires.
-
- The default setting is 5.
-
-*FI_PSM2_PROG_INTERVAL*
-: When auto progress is enabled (asked via the hints to *fi_getinfo*),
- a progress thread is created to make progress calls from time to time.
- This option set the interval (microseconds) between progress calls.
-
- The default setting is 1 if affininty is set, or 1000 if not. See
- *FI_PSM2_PROG_AFFINITY*.
-
-*FI_PSM2_PROG_AFFINITY*
-: When set, specify the set of CPU cores to set the progress thread
- affinity to. The format is
- `<start>[:<end>[:<stride>]][,<start>[:<end>[:<stride>]]]*`,
- where each triplet `<start>:<end>:<stride>` defines a block of
- core_ids. Both `<start>` and `<end>` can be either the `core_id`
- (when >=0) or `core_id - num_cores` (when <0).
-
- By default affinity is not set.
-
-# SEE ALSO
-
-[`fabric`(7)](fabric.7.html),
-[`fi_provider`(7)](fi_provider.7.html),
-[`fi_psm`(7)](fi_psm.7.html),
diff --git a/man/fi_rma.3.md b/man/fi_rma.3.md
deleted file mode 100644
index 09bfd8c..0000000
--- a/man/fi_rma.3.md
+++ /dev/null
@@ -1,293 +0,0 @@
----
-layout: page
-title: fi_rma(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_rma - Remote memory access operations
-
-fi_read / fi_readv / fi_readmsg
-: Initiates a read from remote memory
-
-fi_write / fi_writev / fi_writemsg
-fi_inject_write / fi_writedata
-: Initiate a write to remote memory
-
-# SYNOPSIS
-
-```c
-#include <rdma/fi_rma.h>
-
-ssize_t fi_read(struct fid_ep *ep, void *buf, size_t len, void *desc,
- fi_addr_t src_addr, uint64_t addr, uint64_t key, void *context);
-
-ssize_t fi_readv(struct fid_ep *ep, const struct iovec *iov, void **desc,
- size_t count, fi_addr_t src_addr, uint64_t addr, uint64_t key,
- void *context);
-
-ssize_t fi_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
- uint64_t flags);
-
-ssize_t fi_write(struct fid_ep *ep, const void *buf, size_t len,
- void *desc, fi_addr_t dest_addr, uint64_t addr, uint64_t key,
- void *context);
-
-ssize_t fi_writev(struct fid_ep *ep, const struct iovec *iov, void **desc,
- size_t count, fi_addr_t dest_addr, uint64_t addr, uint64_t key,
- void *context);
-
-ssize_t fi_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
- uint64_t flags);
-
-ssize_t fi_inject_write(struct fid_ep *ep, const void *buf, size_t len,
- fi_addr_t dest_addr, uint64_t addr, uint64_t key);
-
-ssize_t fi_writedata(struct fid_ep *ep, const void *buf, size_t len,
- void *desc, uint64_t data, fi_addr_t dest_addr, uint64_t addr,
- uint64_t key, void *context);
-
-ssize_t fi_inject_writedata(struct fid_ep *ep, const void *buf, size_t len,
- uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key);
-```
-
-# ARGUMENTS
-
-*ep*
-: Fabric endpoint on which to initiate read or write operation.
-
-*buf*
-: Local data buffer to read into (read target) or write from (write
- source)
-
-*len*
-: Length of data to read or write, specified in bytes. Valid
- transfers are from 0 bytes up to the endpoint's max_msg_size.
-
-*iov*
-: Vectored data buffer.
-
-*count*
-: Count of vectored data entries.
-
-*addr*
-: Address of remote memory to access.
-
-*key*
-: Protection key associated with the remote memory.
-
-*desc*
-: Descriptor associated with the local data buffer
-
-*data*
-: Remote CQ data to transfer with the operation.
-
-*dest_addr*
-: Destination address for connectionless write transfers. Ignored
- for connected endpoints.
-
-*src_addr*
-: Source address to read from for connectionless transfers. Ignored
- for connected endpoints.
-
-*msg*
-: Message descriptor for read and write operations.
-
-*flags*
-: Additional flags to apply for the read or write operation.
-
-*context*
-: User specified pointer to associate with the operation.
-
-# DESCRIPTION
-
-RMA (remote memory access) operations are used to transfer data
-directly between a local data buffer and a remote data buffer. RMA
-transfers occur on a byte level granularity, and no message boundaries
-are maintained.
-
-The write functions -- fi_write, fi_writev, fi_writemsg,
-fi_inject_write, and fi_writedata -- are used to transmit
-data into a remote memory buffer. The main
-difference between write functions are the number and type of
-parameters that they accept as input. Otherwise, they perform the
-same general function.
-
-The read functions -- fi_read, fi_readv, and fi_readmsg --
-are used to transfer data from a remote memory region into local data
-buffer(s). Similar to the write operations, read operations operate
-asynchronously. Users should not touch the posted data buffer(s)
-until the read operation has completed.
-
-Completed RMA operations are reported to the user through one or more
-completion queues associated with the endpoint. Users provide context
-which are associated with each operation, and is returned to the user
-as part of the completion. See fi_cq for completion event details.
-
-By default, the remote endpoint does not generate an event or notify
-the user when a memory region has been accessed by an RMA read or
-write operation. However, immediate data may be associated with an
-RMA write operation. RMA writes with immediate data will generate a
-completion entry at the remote endpoint, so that the immediate data
-may be delivered.
-
-## fi_write
-
-The call fi_write transfers the data contained in the user-specified
-data buffer to a remote memory region. The local endpoint must be
-connected to a remote endpoint or destination before fi_write is
-called. Unless the endpoint has been configured differently, the data
-buffer passed into fi_write must not be touched by the application
-until the fi_write call completes asynchronously.
-
-## fi_writev
-
-The fi_writev call adds support for a scatter-gather list to fi_write.
-The fi_writev transfers the set of data buffers
-referenced by the iov parameter to the remote memory region.
-
-## fi_writemsg
-
-The fi_writemsg call supports data transfers over both connected and
-unconnected endpoints, with the ability to control the write operation
-per call through the use of flags. The fi_writemsg function takes a
-struct fi_msg_rma as input.
-
-```c
-struct fi_msg_rma {
- const struct iovec *msg_iov; /* local scatter-gather array */
- void **desc; /* operation descriptor */
- size_t iov_count; /* # elements in msg_iov */
- fi_addr_t addr; /* optional endpoint address */
- const struct fi_rma_iov *rma_iov;/* remote SGL */
- size_t rma_iov_count;/* # elements in rma_iov */
- void *context; /* user-defined context */
- uint64_t data; /* optional immediate data */
-};
-
-struct fi_rma_iov {
- uint64_t addr; /* target RMA address */
- size_t len; /* size of target buffer */
- uint64_t key; /* access key */
-};
-```
-
-## fi_inject_write
-
-The write inject call is an optimized version of fi_write. The
-fi_inject_write function behaves as if the FI_INJECT transfer flag
-were set, and FI_COMPLETION were not. That is, the data buffer is
-available for reuse immediately on returning from from
-fi_inject_write, and no completion event will be generated for this
-write. The completion event will be suppressed even if the endpoint
-has not been configured with FI_SELECTIVE_COMPLETION. See the flags
-discussion below for more details.
-
-## fi_writedata
-
-The write data call is similar to fi_write, but allows for the sending
-of remote CQ data (see FI_REMOTE_CQ_DATA flag) as part of the
-transfer.
-
-## fi_inject_writedata
-
-The inject write data call is similar to fi_inject_write, but allows for the sending
-of remote CQ data (see FI_REMOTE_CQ_DATA flag) as part of the
-transfer.
-
-## fi_read
-
-The fi_read call requests that the remote endpoint transfer data from
-the remote memory region into the local data buffer. The local
-endpoint must be connected to a remote endpoint or destination before
-fi_read is called.
-
-## fi_readv
-
-The fi_readv call adds support for a scatter-gather list to fi_read.
-The fi_readv transfers data from the remote memory region into
-the set of data buffers referenced by the iov parameter.
-
-## fi_readmsg
-
-The fi_readmsg call supports data transfers over both connected and
-unconnected endpoints, with the ability to control the read operation
-per call through the use of flags. The fi_readmsg function takes a
-struct fi_msg_rma as input.
-
-# FLAGS
-
-The fi_readmsg and fi_writemsg calls allow the user to specify flags
-which can change the default data transfer operation. Flags specified
-with fi_readmsg / fi_writemsg override most flags previously
-configured with the endpoint, except where noted (see fi_endpoint.3).
-The following list of flags are usable with fi_readmsg and/or
-fi_writemsg.
-
-*FI_REMOTE_CQ_DATA*
-: Applies to fi_writemsg and fi_writedata. Indicates
- that remote CQ data is available and should be sent as part of the
- request. See fi_getinfo for additional details on
- FI_REMOTE_CQ_DATA.
-
-*FI_COMPLETION*
-: Indicates that a completion entry should be generated for the
- specified operation. The endpoint must be bound to a completion queue
- with FI_SELECTIVE_COMPLETION that corresponds to the specified operation,
- or this flag is ignored.
-
-*FI_MORE*
-: Indicates that the user has additional requests that will
- immediately be posted after the current call returns. Use of this
- flag may improve performance by enabling the provider to optimize
- its access to the fabric hardware.
-
-*FI_INJECT*
-: Applies to fi_writemsg. Indicates that the outbound data buffer
- should be returned to user immediately after the write call
- returns, even if the operation is handled asynchronously. This may
- require that the underlying provider implementation copy the data
- into a local buffer and transfer out of that buffer.
-
-*FI_INJECT_COMPLETE*
-: Applies to fi_writemsg. Indicates that a completion should be
- generated when the source buffer(s) may be reused.
-
-*FI_TRANSMIT_COMPLETE*
-: Applies to fi_writemsg. Indicates that a completion should not be
- generated until the operation has been successfully transmitted and
- is no longer being tracked by the provider.
-
-*FI_DELIVERY_COMPLETE*
-: Applies to fi_writemsg. Indicates that a completion should be
- generated when the operation has been processed by the destination.
-
-*FI_FENCE*
-: Indicates that the requested operation, also
- known as the fenced operation, be deferred until all previous operations
- targeting the same target endpoint have completed.
-
-# RETURN VALUE
-
-Returns 0 on success. On error, a negative value corresponding to fabric
-errno is returned. Fabric errno values are defined in
-`rdma/fi_errno.h`.
-
-# ERRORS
-
-*-FI_EAGAIN*
-: Indicates that the underlying provider currently lacks the resources
- needed to initiate the requested operation. This may be the result
- of insufficient internal buffering, in the case of FI_INJECT,
- or processing queues are full. The operation may be retried after
- additional provider resources become available, usually through the
- completion of currently outstanding operations.
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_cq`(3)](fi_cq.3.html)
diff --git a/man/fi_sockets.7.md b/man/fi_sockets.7.md
deleted file mode 100644
index 7f8e0b7..0000000
--- a/man/fi_sockets.7.md
+++ /dev/null
@@ -1,90 +0,0 @@
----
-layout: page
-title: fi_sockets(7)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-The Sockets Fabric Provider
-
-# OVERVIEW
-
-The sockets provider is a general purpose provider that can be used on any
-system that supports TCP sockets. The provider is not intended to provide
-performance improvements over regular TCP sockets, but rather to allow
-developers to write, test, and debug application code even on platforms
-that do not have high-performance fabric hardware. The sockets provider
-supports all libfabric provider requirements and interfaces.
-
-# SUPPORTED FEATURES
-
-The sockets provider supports all the features defined for the libfabric API.
-Key features include:
-
-*Endpoint types*
-: The provider supports all endpoint types: *FI_EP_MSG*, *FI_EP_RDM*,
- and *FI_EP_DGRAM*.
-
-*Endpoint capabilities*
-: The following data transfer interface is supported for a all endpoint
- types: *fi_msg*. Additionally, these interfaces are supported
- for reliable endpoints (*FI_EP_MSG* and *FI_EP_RDM*): *fi_tagged*,
- *fi_atomic*, and *fi_rma*.
-
-*Modes*
-: The sockets provider supports all operational modes including
- *FI_CONTEXT* and *FI_MSG_PREFIX*.
-
-*Progress*
-: Sockets provider supports both *FI_PROGRESS_AUTO* and *FI_PROGRESS_MANUAL*,
- with a default set to auto. When progress is set to auto, a background
- thread runs to ensure that progress is made for asynchronous requests.
-
-# LIMITATIONS
-
-Sockets provider attempts to emulate the entire API set, including all
-defined options. In order to support development on a wide range of
-systems, it is implemented over TCP sockets. As a result, the
-performance numbers are lower compared to other providers implemented
-over high-speed fabric, and lower than what an application might see
-implementing to sockets directly.
-
-# RUNTIME PARAMETERS
-
-The sockets provider checks for the following environment variables -
-
-*FI_SOCKETS_PE_WAITTIME*
-: An integer value that specifies how many milliseconds to spin while waiting for progress in *FI_PROGRESS_AUTO* mode.
-
-*FI_SOCKETS_MAX_CONN_RETRY*
-: An integer value that specifies the number of socket connection retries before reporting as failure.
-
-*FI_SOCKETS_DEF_CONN_MAP_SZ*
-: An integer to specify the default connection map size.
-
-*FI_SOCKETS_DEF_AV_SZ*
-: An integer to specify the default address vector size.
-
-*FI_SOCKETS_DEF_CQ_SZ*
-: An integer to specify the default completion queue size.
-
-*FI_SOCKETS_DEF_EQ_SZ*
-: An integer to specify the default event queue size.
-
-*FI_SOCKETS_DGRAM_DROP_RATE*
-: An integer value to specify the drop rate of dgram frame when endpoint is *FI_EP_DGRAM*. This is for debugging purpose only.
-
-*FI_SOCKETS_PE_AFFINITY*
-: If specified, progress thread is bound to the indicated range(s) of Linux virtual processor ID(s). This option is currently not supported on OS X. The usage is - id_start[-id_end[:stride]][,].
-
-# LARGE SCALE JOBS
-
-For large scale runs one can use these environment variables to set the default parameters e.g. size of the address vector(AV), completion queue (CQ), connection map etc. that satisfies the requriment of the particular benchmark. The recommended parameters for large scale runs are *FI_SOCKETS_MAX_CONN_RETRY*, *FI_SOCKETS_DEF_CONN_MAP_SZ*, *FI_SOCKETS_DEF_AV_SZ*, *FI_SOCKETS_DEF_CQ_SZ*, *FI_SOCKETS_DEF_EQ_SZ*.
-
-# SEE ALSO
-
-[`fabric`(7)](fabric.7.html),
-[`fi_provider`(7)](fi_provider.7.html),
-[`fi_getinfo`(3)](fi_getinfo.3.html)
diff --git a/man/fi_tagged.3.md b/man/fi_tagged.3.md
deleted file mode 100644
index 8893151..0000000
--- a/man/fi_tagged.3.md
+++ /dev/null
@@ -1,353 +0,0 @@
----
-layout: page
-title: fi_tagged(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_tagged \- Tagged data transfer operations
-
-fi_trecv / fi_trecvv / fi_trecvmsg
-: Post a buffer to receive an incoming message
-
-fi_tsend / fi_tsendv / fi_tsendmsg / fi_tinject / fi_tsenddata
-: Initiate an operation to send a message
-
-# SYNOPSIS
-
-```c
-#include <rdma/fi_tagged.h>
-
-ssize_t fi_trecv(struct fid_ep *ep, void *buf, size_t len, void *desc,
- fi_addr_t src_addr, uint64_t tag, uint64_t ignore, void *context);
-
-ssize_t fi_trecvv(struct fid_ep *ep, const struct iovec *iov, void *desc,
- size_t count, fi_addr_t src_addr, uint64_t tag, uint64_t ignore,
- void *context);
-
-ssize_t fi_trecvmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg,
- uint64_t flags);
-
-ssize_t fi_tsend(struct fid_ep *ep, const void *buf, size_t len,
- void *desc, fi_addr_t dest_addr, uint64_t tag, void *context);
-
-ssize_t fi_tsendv(struct fid_ep *ep, const struct iovec *iov,
- void *desc, size_t count, fi_addr_t dest_addr, uint64_t tag,
- void *context);
-
-ssize_t fi_tsendmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg,
- uint64_t flags);
-
-ssize_t fi_tinject(struct fid_ep *ep, const void *buf, size_t len,
- fi_addr_t dest_addr, uint64_t tag);
-
-ssize_t fi_tsenddata(struct fid_ep *ep, const void *buf, size_t len,
- void *desc, uint64_t data, fi_addr_t dest_addr, uint64_t tag,
- void *context);
-
-ssize_t fi_tinjectdata(struct fid_ep *ep, const void *buf, size_t len,
- uint64_t data, fi_addr_t dest_addr, uint64_t tag);
-```
-
-# ARGUMENTS
-
-*fid*
-: Fabric endpoint on which to initiate tagged communication operation.
-
-*buf*
-: Data buffer to send or receive.
-
-*len*
-: Length of data buffer to send or receive.
-
-*iov*
-: Vectored data buffer.
-
-*count*
-: Count of vectored data entries.
-
-*tag*
-: Tag associated with the message.
-
-*ignore*
-: Mask of bits to ignore applied to the tag for receive operations.
-
-*desc*
-: Memory descriptor associated with the data buffer
-
-*data*
-: Remote CQ data to transfer with the sent data.
-
-*dest_addr*
-: Destination address for connectionless transfers. Ignored for
- connected endpoints.
-
-*src_addr*
-: Source address to receive from for connectionless transfers. Applies
- only to connectionless endpoints with the FI_DIRECTED_RECV capability
- enabled, otherwise this field is ignored. If set to FI_ADDR_UNSPEC,
- any source address may match.
-
-*msg*
-: Message descriptor for send and receive operations.
-
-*flags*
-: Additional flags to apply for the send or receive operation.
-
-*context*
-: User specified pointer to associate with the operation.
-
-# DESCRIPTION
-
-Tagged messages are data transfers which carry a key or tag with the
-message buffer. The tag is used at the receiving endpoint to match
-the incoming message with a corresponding receive buffer. Message
-tags match when the receive buffer tag is the same as the send buffer
-tag with the ignored bits masked out. This can be stated as:
-
-```c
-send_tag & ~ignore == recv_tag & ~ignore
-```
-
-In general, message tags are checked against receive buffers in the
-order in which messages have been posted to the endpoint. See the
-ordering discussion below for more details.
-
-The send functions -- fi_tsend, fi_tsendv, fi_tsendmsg,
-fi_tinject, and fi_tsenddata -- are used
-to transmit a tagged message from one endpoint to another endpoint.
-The main difference between send functions are the number and type of
-parameters that they accept as input. Otherwise, they perform the
-same general function.
-
-The receive functions -- fi_trecv, fi_trecvv, fi_recvmsg
--- post a data buffer to an endpoint to receive inbound tagged
-messages. Similar to the send operations, receive operations operate
-asynchronously. Users should not touch the posted data buffer(s)
-until the receive operation has completed. Posted receive buffers are
-matched with inbound send messages based on the tags associated with
-the send and receive buffers.
-
-Completed message operations are reported to the user through one or
-more event collectors associated with the endpoint. Users provide
-context which are associated with each operation, and is returned to
-the user as part of the event completion. See fi_cq for completion
-event details.
-
-## fi_tsend
-
-The call fi_tsend transfers the data contained in the user-specified
-data buffer to a remote endpoint, with message boundaries being
-maintained. The local endpoint must be connected to a remote endpoint
-or destination before fi_tsend is called. Unless the endpoint has
-been configured differently, the data buffer passed into fi_tsend must
-not be touched by the application until the fi_tsend call completes
-asynchronously.
-
-## fi_tsendv
-
-The fi_tsendv call adds support for a scatter-gather list to fi_tsend.
-The fi_sendv transfers the set of data buffers
-referenced by the iov parameter to a remote endpoint as a single
-message.
-
-## fi_tsendmsg
-
-The fi_tsendmsg call supports data transfers over both connected and
-unconnected endpoints, with the ability to control the send operation
-per call through the use of flags. The fi_tsendmsg function takes a
-struct fi_msg_tagged as input.
-
-```c
-struct fi_msg_tagged {
- const struct iovec *msg_iov; /* scatter-gather array */
- void *desc; /* data descriptor */
- size_t iov_count;/* # elements in msg_iov *
- const void *addr; /* optional endpoint address */
- uint64_t tag; /* tag associated with message */
- uint64_t ignore; /* mask applied to tag for receives */
- void *context; /* user-defined context */
- uint64_t data; /* optional immediate data */
-};
-```
-
-## fi_tinject
-
-The tagged inject call is an optimized version of fi_tsend. The
-fi_tinject function behaves as if the FI_INJECT transfer flag were
-set, and FI_COMPLETION were not. That is, the data buffer is
-available for reuse immediately on returning from from fi_tinject, and
-no completion event will be generated for this send. The completion
-event will be suppressed even if the endpoint has not been configured
-with FI_SELECTIVE_COMPLETION. See the flags discussion below for more
-details.
-
-## fi_tsenddata
-
-The tagged send data call is similar to fi_tsend, but allows for the
-sending of remote CQ data (see FI_REMOTE_CQ_DATA flag) as part of the
-transfer.
-
-## fi_tinjectdata
-
-The tagged inject data call is similar to fi_tinject, but allows for the
-sending of remote CQ data (see FI_REMOTE_CQ_DATA flag) as part of the
-transfer.
-
-## fi_trecv
-
-The fi_trecv call posts a data buffer to the receive queue of the
-corresponding endpoint. Posted receives are searched in the order in
-which they were posted in order to match sends. Message boundaries are
-maintained. The order in which the receives complete is dependent on
-the endpoint type and protocol.
-
-## fi_trecvv
-
-The fi_trecvv call adds support for a scatter-gather list to fi_trecv.
-The fi_trecvv posts the set of data buffers referenced by the iov
-parameter to a receive incoming data.
-
-## fi_trecvmsg
-
-The fi_trecvmsg call supports posting buffers over both connected and
-unconnected endpoints, with the ability to control the receive
-operation per call through the use of flags. The fi_trecvmsg function
-takes a struct fi_msg_tagged as input.
-
-# FLAGS
-
-The fi_trecvmsg and fi_tsendmsg calls allow the user to specify flags
-which can change the default message handling of the endpoint. Flags
-specified with fi_trecvmsg / fi_tsendmsg override most flags
-previously configured with the endpoint, except where noted (see
-fi_endpoint). The following list of flags are usable with fi_trecvmsg
-and/or fi_tsendmsg.
-
-*FI_REMOTE_CQ_DATA*
-: Applies to fi_tsendmsg and fi_tsenddata. Indicates
- that remote CQ data is available and should be sent as part of the
- request. See fi_getinfo for additional details on
- FI_REMOTE_CQ_DATA.
-
-*FI_COMPLETION*
-: Indicates that a completion entry should be generated for the
- specified operation. The endpoint must be bound to a completion queue
- with FI_SELECTIVE_COMPLETION that corresponds to the specified operation,
- or this flag is ignored.
-
-*FI_MORE*
-: Indicates that the user has additional requests that will
- immediately be posted after the current call returns. Use of this
- flag may improve performance by enabling the provider to optimize
- its access to the fabric hardware.
-
-*FI_INJECT*
-: Applies to fi_tsendmsg. Indicates that the outbound data buffer
- should be returned to user immediately after the send call returns,
- even if the operation is handled asynchronously. This may require
- that the underlying provider implementation copy the data into a
- local buffer and transfer out of that buffer.
-
-*FI_INJECT_COMPLETE*
-: Applies to fi_tsendmsg. Indicates that a completion should be
- generated when the source buffer(s) may be reused.
-
-*FI_TRANSMIT_COMPLETE*
-: Applies to fi_tsendmsg. Indicates that a completion should not be
- generated until the operation has been successfully transmitted and
- is no longer being tracked by the provider.
-
-*FI_FENCE*
-: Applies to transmits. Indicates that the requested operation, also
- known as the fenced operation, be deferred until all previous operations
- targeting the same target endpoint have completed.
-
-The following flags may be used with fi_trecvmsg.
-
-*FI_PEEK*
-: The peek flag may be used to see if a specified message has arrived.
- A peek request is often useful on endpoints that have provider
- allocated buffering enabled (see fi_rx_attr total_buffered_recv).
- Unlike standard receive operations, a receive operation with the FI_PEEK
- flag set does not remain queued with the provider after the peek completes
- successfully. The peek operation operates asynchronously, and the results
- of the peek operation are available in the completion queue associated with
- the endpoint. If no message is found matching the tags specified in the peek
- request, then a completion queue error entry with err field set to FI_ENOMSG
- will be available.
-
- If a peek request locates a matching message, the operation will complete
- successfully. The returned completion data will indicate the meta-data
- associated with the message, such as the message length, completion flags,
- available CQ data, tag, and source address. The data available is subject to
- the completion entry format (e.g. struct fi_cq_tagged_entry).
-
- An application may supply a buffer if it desires to receive data as
- a part of the peek operation. In order to receive data as a part of
- the peek operation, the buf and len fields must be available in the
- CQ format. In particular, FI_CQ_FORMAT_CONTEXT and FI_CQ_FORMAT_MSG
- cannot be used if peek operations desire to obtain a copy of the
- data. The returned data is limited to the size of the input
- buffer(s) or the message size, if smaller. A provider indicates if
- data is available by setting the buf field of the CQ entry to the
- user's first input buffer. If buf is NULL, no data was available to
- return. A provider may return NULL even if the peek operation
- completes successfully. Note that the CQ entry len field will
- reference the size of the message, not necessarily the size of the
- returned data.
-
-*FI_CLAIM*
-: If this flag is used in conjunction with FI_PEEK, it indicates if the
- peek request completes successfully -- indicating that a matching message
- was located -- the message is claimed by caller. Claimed messages can only
- be retrieved using a subsequent, paired receive operation with the FI_CLAIM
- flag set. A receive operation with the FI_CLAIM flag set, but FI_PEEK not
- set is used to retrieve a previously claimed message.
-
- In order to use the FI_CLAIM flag, an application must supply a struct
- fi_context structure as the context for the receive operation. The same
- fi_context structure used for an FI_PEEK + FI_CLAIM operation must be used
- by the paired FI_CLAIM request.
-
-*FI_DISCARD*
-: This flag must be used in conjunction with either FI_PEEK or FI_CLAIM.
- If this flag is used in conjunction with FI_PEEK, it indicates if the
- peek request completes successfully -- indicating that a matching message
- was located -- the message is discarded by the provider, as the data is not
- needed by the application. This flag may also be used in conjunction with
- FI_CLAIM in order to retrieve and discard a message previously claimed
- using an FI_PEEK + FI_CLAIM request.
-
- If this flag is set, the input buffer(s) and length parameters.
-
-# RETURN VALUE
-
-The tagged send and receive calls return 0 on success. On error, a
-negative value corresponding to fabric _errno _ is returned. Fabric
-errno values are defined in `fi_errno.h`.
-
-# ERRORS
-
-*-FI_EAGAIN*
-: Indicates that the underlying provider currently lacks the resources
- needed to initiate the requested operation. This may be the result
- of insufficient internal buffering, in the case of FI_INJECT,
- or processing queues are full. The operation may be retried after
- additional provider resources become available, usually through the
- completion of currently outstanding operations.
-
-*-FI_EINVAL*
-: Indicates that an invalid argument was supplied by the user.
-
-*-FI_EOTHER*
-: Indicates that an unspecified error occurred.
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_domain`(3)](fi_domain.3.html),
-[`fi_cq`(3)](fi_cq.3.html)
diff --git a/man/fi_trigger.3.md b/man/fi_trigger.3.md
deleted file mode 100644
index 15bc06a..0000000
--- a/man/fi_trigger.3.md
+++ /dev/null
@@ -1,93 +0,0 @@
----
-layout: page
-title: fi_trigger(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_trigger - Triggered operations
-
-# SYNOPSIS
-
-```c
-#include <rdma/fi_trigger.h>
-```
-
-# DESCRIPTION
-
-Triggered operations allow an application to queue a data transfer
-request that is deferred until a specified condition is met. A typical
-use is to send a message only after receiving all input data.
-
-A triggered operation may be requested by specifying the FI_TRIGGER
-flag as part of the operation. Alternatively, an endpoint alias may
-be created and configured with the FI_TRIGGER flag. Such an endpoint
-is referred to as a trigger-able endpoint. All data transfer
-operations on a trigger-able endpoint are deferred.
-
-Any data transfer operation is potentially trigger-able, subject to
-provider constraints. Trigger-able endpoints are initialized such that
-only those interfaces supported by the provider which are trigger-able
-are available.
-
-Triggered operations require that applications use struct
-fi_triggered_context as their per operation context parameter. The
-use of struct fi_triggered_context replaces struct fi_context, if
-required by the provider. Although struct fi_triggered_context is not
-opaque to the application, the contents of the structure may be
-modified by the provider. This structure has similar requirements as
-struct fi_context. It must be allocated by the application and remain
-valid until the corresponding operation completes or is successfully
-canceled.
-
-Struct fi_triggered_context is used to specify the condition that must
-be met before the triggered data transfer is initiated. If the
-condition is met when the request is made, then the data transfer may
-be initiated immediately. The format of struct fi_triggered_context
-is described below.
-
-```c
-struct fi_triggered_context {
- enum fi_trigger_event event_type; /* trigger type */
- union {
- struct fi_trigger_threshold threshold;
- void *internal[3]; /* reserved */
- } trigger;
-};
-```
-
-The triggered context indicates the type of event assigned to the
-trigger, along with a union of trigger details that is based on the
-event type.
-
-## TRIGGER EVENTS
-
-The following trigger events are defined.
-
-*FI_TRIGGER_THRESHOLD*
-: This indicates that the data transfer operation will be deferred
- until an event counter crosses an application specified threshold
- value. The threshold is specified using struct
- fi_trigger_threshold:
-
-```c
-struct fi_trigger_threshold {
- struct fid_cntr *cntr; /* event counter to check */
- size_t threshold; /* threshold value */
-};
-```
-
-Threshold operations are triggered in the order of the threshold
-values. This is true even if the counter increments by a value
-greater than 1. If two triggered operations have the same threshold,
-they will be triggered in the order in which they were submitted to
-the endpoint.
-
-# SEE ALSO
-
-[`fi_getinfo`(3)](fi_getinfo.3.html),
-[`fi_endpoint`(3)](fi_endpoint.3.html),
-[`fi_alias`(3)](fi_alias.3.html),
-[`fi_cntr`(3)](fi_cntr.3.html)
diff --git a/man/fi_udp.7.md b/man/fi_udp.7.md
deleted file mode 100644
index ea1a76b..0000000
--- a/man/fi_udp.7.md
+++ /dev/null
@@ -1,60 +0,0 @@
----
-layout: page
-title: fi_udp(7)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-The UDP Fabric Provider
-
-# OVERVIEW
-
-The UDP provider is a basic provider that can be used on any
-system that supports UDP sockets. The provider is not intended to provide
-performance improvements over regular TCP sockets, but rather to allow
-application and provider developers to write, test, and debug their code.
-The UDP provider forms the foundation of a utility provider that enables
-the implementation of libfabric features over any hardware.
-
-# SUPPORTED FEATURES
-
-The UDP provider supports a minimal set of features useful for sending and
-receiving datagram messages over an unreliable endpoint.
-
-*Endpoint types*
-: The provider supports only endpoint type *FI_EP_DGRAM*.
-
-*Endpoint capabilities*
-: The following data transfer interface is supported: *fi_msg*.
-
-*Modes*
-: The provider does not require the use of any mode bits.
-
-*Progress*
-: The UDP provider supports both *FI_PROGRESS_AUTO* and *FI_PROGRESS_MANUAL*,
- with a default set to auto. However, receive side data buffers are not
- modified outside of completion processing routines.
-
-# LIMITATIONS
-
-The UDP provider has hard-coded maximums for supported queue sizes and data
-transfers. These values are reflected in the related fabric attribute
-structures
-
-EPs must be bound to both RX and TX CQs.
-
-No support for selective completions or multi-recv.
-
-No support for counters.
-
-# RUNTIME PARAMETERS
-
-No runtime parameters are currently defined.
-
-# SEE ALSO
-
-[`fabric`(7)](fabric.7.html),
-[`fi_provider`(7)](fi_provider.7.html),
-[`fi_getinfo`(3)](fi_getinfo.3.html)
diff --git a/man/fi_usnic.7.md b/man/fi_usnic.7.md
deleted file mode 100644
index 1e6e37f..0000000
--- a/man/fi_usnic.7.md
+++ /dev/null
@@ -1,288 +0,0 @@
----
-layout: page
-title: fi_usnic(7)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-The usNIC Fabric Provider
-
-# OVERVIEW
-
-The *usnic* provider is designed to run over the Cisco VIC
-(virtualized NIC) hardware on Cisco UCS servers. It utilizes the
-Cisco usNIC (userspace NIC) capabilities of the VIC to enable ultra
-low latency and other offload capabilities on Ethernet networks.
-
-# RELEASE NOTES
-
-* The *usnic* libfabric provider requires the use of the "libnl"
- library.
- - There are two versions of libnl generally available: v1 and v3;
- the usnic provider can use either version.
- - If you are building libfabric/the usnic provider from source, you
- will need to have the libnl header files available (e.g., if you
- are installing libnl from RPM or other packaging system, install
- the "-devel" versions of the package).
- - If you have libnl (either v1 or v3) installed in a non-standard
- location (e.g., not in /usr/lib or /usr/lib64), you may need to
- tell libfabric's configure where to find libnl via the
- `--with-libnl=DIR` command line option (where DIR is the
- installation prefix of the libnl package).
-* The most common way to use the libfabric usnic provider is via an
- MPI implementation that uses libfabric (and the usnic provider) as a
- lower layer transport. MPI applications do not need to know
- anything about libfabric or usnic in this use case -- the MPI
- implementation hides all these details from the application.
-* If you are writing applications directly to the libfabric API:
- - *FI_EP_DGRAM* endpoints are the best supported method of utilizing
- the usNIC interface. Specifically, the *FI_EP_DGRAM* endpoint
- type has been extensively tested as the underlying layer for Open
- MPI's *usnic* BTL.
- - *FI_EP_MSG* and *FI_EP_RDM* endpoints are implemented, but are
- only lightly tested. It is likely that there are still some bugs
- in these endpoint types. In particular, there are known bugs in RDM
- support in the presence of congestion or packet loss (issue 1621).
- RMA is not yet supported.
- - [`fi_provider`(7)](fi_provider.7.html) lists requirements for all
- providers. The following limitations exist in the *usnic*
- provider:
- * multicast operations are not supported on *FI_EP_DGRAM* and
- *FI_EP_RDM* endpoints.
- * *FI_EP_MSG* endpoints only support connect, accept, and getname
- CM operations.
- * Passive endpoints only support listen, setname, and getname CM
- operations.
- * *FI_EP_DGRAM* endpoints support `fi_sendmsg()` and
- `fi_recvmsg()`, but some flags are ignored. `fi_sendmsg()`
- supports `FI_INJECT` and `FI_COMPLETION`. `fi_recvmsg()`
- supports `FI_MORE`.
- * Address vectors only support `FI_AV_MAP`.
- * No counters are supported.
- * The tag matching interface is not supported.
- * *FI_MSG_PREFIX* is only supported on *FI_EP_DGRAM* and usage
- is limited to releases 1.1 and beyond.
- * fi_control with FI_GETWAIT may only be used on CQs that have been
- bound to an endpoint. If fi_control is used on an unbound CQ, it will
- return -FI_EOPBADSTATE.
- - The usnic libfabric provider supports extensions that provide
- information and functionality beyond the standard libfabric
- interface. See the "USNIC EXTENSIONS" section, below.
-
-# USNIC EXTENSIONS
-
-The usnic libfabric provider exports extensions for additional VIC,
-usNIC, and Ethernet capabilities not provided by the standard
-libfabric interface.
-
-These extensions are available via the "fi_ext_usnic.h" header file.
-
-## Fabric Extension: getinfo
-
-Version 2 of the "fabric getinfo" extension was introduced in Libfabric release
-v1.3.0 and can be used to retrieve IP and SR-IOV information about a usNIC
-device obtained from the [`fi_getinfo`(3)](fi_getinfo.3.html) function.
-
-The "fabric getinfo" extension is obtained by calling `fi_open_ops` and
-requesting `FI_USNIC_FABRIC_OPS_1` to get the usNIC fabric extension
-operations. The `getinfo` function accepts a version parameter that can be
-used to select different versions of the extension. The information returned by
-the "fabric getinfo" extension is accessible through a `fi_usnic_info` struct
-that uses a version tagged union. The accessed union member must correspond
-with the requested version. It is recommended that applications explicitly
-request a version rather than using the header provided
-`FI_EXT_USNIC_INFO_VERSION`. Although there is a version 1 of the extension,
-its use is discouraged, and it may not be available in future releases.
-
-
-```c
-#include <rdma/fi_ext_usnic.h>
-
-struct fi_usnic_info {
- uint32_t ui_version;
- union {
- struct fi_usnic_info_v1 v1;
- struct fi_usnic_info_v2 v2;
- } ui;
-};
-
-int getinfo(uint32_t version, struct fid_fabric *fabric,
- struct fi_usnic_info *info);
-```
-
-*version*
-: Version of getinfo to be used
-
-*fabric*
-: Fabric descriptor
-
-*info*
-: Upon successful return, this parameter will contain information about the
-fabric.
-
-- Version 2
-
-```c
-struct fi_usnic_cap {
- const char *uc_capability;
- int uc_present;
-};
-
-struct fi_usnic_info_v2 {
- uint32_t ui_link_speed;
- uint32_t ui_netmask_be;
- char ui_ifname[IFNAMSIZ];
- unsigned ui_num_vf;
- unsigned ui_qp_per_vf;
- unsigned ui_cq_per_vf;
-
- char ui_devname[FI_EXT_USNIC_MAX_DEVNAME];
- uint8_t ui_mac_addr[6];
-
- uint32_t ui_ipaddr_be;
- uint32_t ui_prefixlen;
- uint32_t ui_mtu;
- uint8_t ui_link_up;
-
- uint32_t ui_vendor_id;
- uint32_t ui_vendor_part_id;
- uint32_t ui_device_id;
- char ui_firmware[64];
-
- unsigned ui_intr_per_vf;
- unsigned ui_max_cq;
- unsigned ui_max_qp;
-
- unsigned ui_max_cqe;
- unsigned ui_max_send_credits;
- unsigned ui_max_recv_credits;
-
- const char *ui_nicname;
- const char *ui_pid;
-
- struct fi_usnic_cap **ui_caps;
-};
-```
-
-- Version 1
-
-```c
-struct fi_usnic_info_v1 {
- uint32_t ui_link_speed;
- uint32_t ui_netmask_be;
- char ui_ifname[IFNAMSIZ];
-
- uint32_t ui_num_vf;
- uint32_t ui_qp_per_vf;
- uint32_t ui_cq_per_vf;
-};
-```
-
-Version 1 of the "fabric getinfo" extension can be used by explicitly
-requesting it in the call to `getinfo` and accessing the `v1` portion of the
-`fi_usnic_info.ui` union. Use of version 1 is not recommended and it may be
-removed from future releases.
-
-
-The following is an example of how to utilize version 2 of the usnic "fabric
-getinfo" extension.
-
-```c
-#include <stdio.h>
-#include <rdma/fabric.h>
-
-/* The usNIC extensions are all in the
- rdma/fi_ext_usnic.h header */
-#include <rdma/fi_ext_usnic.h>
-
-int main(int argc, char *argv[]) {
- struct fi_info *info;
- struct fi_info *info_list;
- struct fi_info hints = {0};
- struct fi_ep_attr ep_attr = {0};
- struct fi_fabric_attr fabric_attr = {0};
-
- fabric_attr.prov_name = "usnic";
- ep_attr.type = FI_EP_DGRAM;
-
- hints.caps = FI_MSG;
- hints.mode = FI_LOCAL_MR | FI_MSG_PREFIX;
- hints.addr_format = FI_SOCKADDR;
- hints.ep_attr = &ep_attr;
- hints.fabric_attr = &fabric_attr;
-
- /* Find all usnic providers */
- fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, &hints, &info_list);
-
- for (info = info_list; NULL != info; info = info->next) {
- /* Open the fabric on the interface */
- struct fid_fabric *fabric;
- fi_fabric(info->fabric_attr, &fabric, NULL);
-
- /* Pass FI_USNIC_FABRIC_OPS_1 to get usnic ops
- on the fabric */
- struct fi_usnic_ops_fabric *usnic_fabric_ops;
- fi_open_ops(&fabric->fid, FI_USNIC_FABRIC_OPS_1, 0,
- (void **) &usnic_fabric_ops, NULL);
-
- /* Now use the returned usnic ops structure to call
- usnic extensions. The following extension queries
- some IP and SR-IOV characteristics about the
- usNIC device. */
- struct fi_usnic_info usnic_info;
-
- /* Explicitly request version 2. */
- usnic_fabric_ops->getinfo(2, fabric, &usnic_info);
-
- printf("Fabric interface %s is %s:\n"
- "\tNetmask: 0x%08x\n\tLink speed: %d\n"
- "\tSR-IOV VFs: %d\n\tQPs per SR-IOV VF: %d\n"
- "\tCQs per SR-IOV VF: %d\n",
- info->fabric_attr->name,
- usnic_info.ui.v2.ui_ifname,
- usnic_info.ui.v2.ui_netmask_be,
- usnic_info.ui.v2.ui_link_speed,
- usnic_info.ui.v2.ui_num_vf,
- usnic_info.ui.v2.ui_qp_per_vf,
- usnic_info.ui.v2.ui_cq_per_vf);
-
- fi_close(&fabric->fid);
- }
-
- fi_freeinfo(info_list);
- return 0;
-}
-```
-
-## Adress Vector Extension: get_distance
-
-The "address vector get_distance" extension was introduced in Libfabric release
-v1.0.0 and can be used to retrieve the network distance of an address.
-
-The "get_distance" extension is obtained by calling `fi_open_ops` and
-requesting `FI_USNIC_AV_OPS_1` to get the usNIC address vector extension
-operations.
-
-```c
-int get_distance(struct fid_av *av, void *addr, int *metric);
-```
-
-*av*
-: Address vector
-
-*addr*
-: Destination address
-
-*metric*
-: On output this will contain `-1` if the destination host is unreachable, `0`
-is the destination host is locally connected, and `1` otherwise.
-
-See fi_ext_usnic.h for more details.
-
-# SEE ALSO
-
-[`fabric`(7)](fabric.7.html),
-[`fi_open_ops`(3)](fi_open_ops.3.html),
-[`fi_provider`(7)](fi_provider.7.html),
diff --git a/man/fi_verbs.7.md b/man/fi_verbs.7.md
deleted file mode 100644
index e204c61..0000000
--- a/man/fi_verbs.7.md
+++ /dev/null
@@ -1,86 +0,0 @@
----
-layout: page
-title: fi_verbs(7)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-The Verbs Fabric Provider
-
-# OVERVIEW
-
-The verbs provider enables applications using OFI to be run over any verbs
-hardware (Infiniband, iWarp, etc). It uses the Linux Verbs API for network
-transport and provides a translation of OFI calls to appropriate verbs API calls.
-It uses librdmacm for communication management and libibverbs for other control
-and data transfer operations.
-
-# SUPPORTED FEATURES
-
-The verbs provider supports a subset of OFI features.
-
-*Endpoint types*
-: FI_EP_MSG, FI_EP_RDM (Experimental support FI_TAGGED and FI_RMA interfaces).
-
-*Endpoint capabilities*
-: FI_MSG, FI_RMA, FI_ATOMIC.
-
-*Modes*
-: Verbs provider requires applications to support the following modes:
- * FI_LOCAL_MR for all applications.
- * FI_RX_CQ_DATA for applications that want to use RMA. Applications must
- take responsibility of posting receives for any incoming CQ data.
- * FI_CONTEXT for applications making uses of the experimental FI_EP_RDM capability.
-
-*Addressing Formats*
-: Supported addresing formats include FI_SOCKADDR, FI_SOCKADDR_IN, FI_SOCKADDR_IN6,
- FI_SOCKADDR_IB
-
-*Progress*
-: Verbs provider supports FI_PROGRESS_AUTO: Asynchronous operations make forward
- progress automatically.
-
-*Operation flags*
-: Verbs provider supports FI_INJECT, FI_COMPLETION, FI_REMOTE_CQ_DATA.
-
-*Msg Ordering*
-: Verbs provider support the following messaging ordering on the TX side:
- * Read after Read
- * Read after Write
- * Read after Send
- * Write after Write
- * Write after Send
- * Send after Write
- * Send after Send
-
-# LIMITATIONS
-
-*CQ*
-: cq_readfrom operations are not supported.
-
-*Memory Regions*
-: Only FI_MR_BASIC mode is supported. Adding regions via s/g list is not supported.
- Generic fi_mr_regattr is not supported. No support for binding memory regions to
- a counter.
-
-*Wait objects*
-: Only FI_WAIT_FD wait object is supported. Wait sets are not supported.
-
-*Resource Management*
-: Application has to make sure CQs are not overrun as this cannot be detected
- by the provider.
-
-*RDM*
-: The RDM support for verbs have the following limitations:
- * iWARP is not supported yet.
- * Supports iovs of only size 1.
- * Max data transfer size is 1 GB
- * Not thread safe.
- * Data and flags arguments are ignored
-
-# SEE ALSO
-
-[`fabric`(7)](fabric.7.html),
-[`fi_provider`(7)](fi_provider.7.html),
diff --git a/man/fi_version.3.md b/man/fi_version.3.md
deleted file mode 100644
index 014b2ff..0000000
--- a/man/fi_version.3.md
+++ /dev/null
@@ -1,43 +0,0 @@
----
-layout: page
-title: fi_version(3)
-tagline: Libfabric Programmer's Manual
----
-{% include JB/setup %}
-
-# NAME
-
-fi_version \- Version of the library interfaces
-
-# SYNOPSIS
-
-```c
-#include <rdma/fabric.h>
-
-uint32_t fi_version(void);
-
-FI_MAJOR(version)
-
-FI_MINOR(version)
-```
-
-# DESCRIPTION
-
-This call returns the current version of the library interfaces. The
-version includes major and minor numbers. These may be extracted
-from the returned value using the FI_MAJOR() and FI_MINOR() macros.
-
-# NOTES
-
-The library may support older versions of the interfaces.
-
-# RETURN VALUE
-
-Returns the current library version. The upper 16-bits of the version
-correspond to the major number, and the lower 16-bits correspond with
-the minor number.
-
-# SEE ALSO
-
-[`fabric`(7)](fabric.7.html),
-[`fi_getinfo`(3)](fi_getinfo.3.html)
diff --git a/man/man1/fi_info.1 b/man/man1/fi_info.1
new file mode 100644
index 0000000..1582585
--- /dev/null
+++ b/man/man1/fi_info.1
@@ -0,0 +1,109 @@
+.TH "fi_info" "1" "2016\-09\-12" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
+.SH NAME
+.PP
+fi_info \- Simple utility to query for fabric interfaces
+.SH SYNOPSYS
+.IP
+.nf
+\f[C]
+\ fi_info\ [OPTIONS]
+\f[]
+.fi
+.SH DESCRIPTION
+.PP
+The fi_info utility can be used to query for available fabric
+interfaces.
+The utility supports filtering based on a number of options such as
+endpoint type, provider name, or supported modes.
+Additionally, fi_info can also be used to discover the environment
+variables that can be used to tune provider specific parameters.
+If no filters are specified, then all available fabric interfaces for
+all providers and endpoint types will be returned.
+.SH OPTIONS
+.SS Filtering
+.PP
+\f[I]\-n, \-\-node=<NAME>\f[] : Node name or address used to filter
+interfaces.
+Only interfaces which can reach the given node or address will respond.
+.PP
+\f[I]\-P, \-\-port=<PORT>\f[] : Port number used to filter interfaces.
+.PP
+\f[I]\-c, \-\-caps=<CAP1|CAP2>..\f[] : Pipe separated list of
+capabilities used to filter interfaces.
+Only interfaces supporting all of the given capabilities will respond.
+For more information on capabilities, see fi_getinfo(3).
+.PP
+\f[I]\-m, \-\-mode=<MOD1|MOD2>..\f[] : Pipe separated list of modes used
+to filter interfaces.
+Only interfaces supporting all of the given modes will respond.
+For more information on, modes see fi_getinfo(3).
+.PP
+\f[I]\-t, \-\-ep_type=<EPTYPE>\f[] : Specifies the type of fabric
+interface communication desired.
+For example, specifying FI_EP_DGRAM would return only interfaces which
+support unreliable datagram.
+For more information on endpoint types, see fi_endpoint(3).
+.PP
+\f[I]\-a, \-\-addr_format=<FMT>\f[] : Filter fabric interfaces by their
+address format.
+For example, specifying FI_SOCKADDR_IN would return only interfaces
+which use sockaddr_in structures for addressing.
+For more information on address formats, see fi_getinfo(3).
+.PP
+\f[I]\-p, \-\-provider=<PROV>\f[] : Filter fabric interfaces by the
+provider implementation.
+For a list of providers, see the \f[C]\-\-list\f[] option.
+.PP
+\f[I]\-d, \-\-domain=<DOMAIN>\f[] : Filter interfaces to only those with
+the given domain name.
+.PP
+\f[I]\-f, \-\-fabric=<FABRIC>\f[] : Filter interfaces to only those with
+the given fabric name.
+.SS Discovery
+.PP
+\f[I]\-e, \-\-env\f[] : List libfabric related environment levels which
+can be used to enable extra configuration or tuning.
+.PP
+\f[I]\-l, \-\-list\f[] : List available libfabric providers.
+.PP
+\f[I]\-v, \-\-verbose\f[] : By default, fi_info will display a summary
+of each of the interfaces discovered.
+If the verbose option is enabled, then all of the contents of the
+fi_info structure are displayed.
+For more information on the data contained in the fi_info structure, see
+fi_getinfo(3).
+.PP
+\f[I]\-\-version\f[] : Display versioning information.
+.SH USAGE EXAMPLES
+.IP
+.nf
+\f[C]
+$\ fi_info\ \-n\ 30.0.11.1\ \-p\ usnic\ \-t\ FI_EP_DGRAM
+\f[]
+.fi
+.PP
+This will respond with all fabric interfaces that can reach address
+30.0.11.1 using endpoint type FI_EP_DGRAM with the usNIC provider.
+.SH OUTPUT
+.PP
+By default fi_info will output a summary of the fabric interfaces
+discovered:
+.IP
+.nf
+\f[C]
+$\ ./fi_info\ \-n\ 30.0.11.1\ \-p\ usnic\ \-t\ FI_EP_DGRAM
+provider:\ usnic
+\ \ \ \ fabric:\ 30.0.11.0/24
+\ \ \ \ domain:\ usnic_2
+\ \ \ \ version:\ 1.0
+\ \ \ \ type:\ FI_EP_DGRAM
+\ \ \ \ protocol:\ FI_PROTO_UDP
+\f[]
+.fi
+.PP
+To see the full fi_info structure, specify the \f[C]\-v\f[] option.
+.SH SEE ALSO
+.PP
+\f[C]fi_getinfo(3)\f[], \f[C]fi_endpoint(3)\f[]
+.SH AUTHORS
+OpenFabrics.
diff --git a/man/man1/fi_pingpong.1 b/man/man1/fi_pingpong.1
new file mode 100644
index 0000000..9899f98
--- /dev/null
+++ b/man/man1/fi_pingpong.1
@@ -0,0 +1,168 @@
+.TH "fi_pingpong" "1" "2016\-09\-02" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
+.SH NAME
+.PP
+fi_pingpong \- Quick and simple pingpong test for libfabric
+.SH SYNOPSYS
+.IP
+.nf
+\f[C]
+\ fi_pingpong\ [OPTIONS]\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ start\ server
+\ fi_pingpong\ [OPTIONS]\ <server\ address>\ \ \ \ \ connect\ to\ server
+\f[]
+.fi
+.SH DESCRIPTION
+.PP
+fi_pingpong is a pingpong test for the core feature of the libfabric
+library: transmitting data between two processes.
+fi_pingpong also displays aggregated statistics after each test run, and
+can additionally verify data integrity upon receipt.
+.PP
+By default, the datagram (FI_EP_DGRAM) endpoint is used for the test,
+unless otherwise specified via \f[C]\-e\f[].
+.SH HOW TO RUN TESTS
+.PP
+Two copies of the program must be launched: first, one copy must be
+launched as the server.
+Second, another copy is launched with the address of the server.
+.PP
+As a client\-server test, each have the following usage model:
+.SS Start the server
+.IP
+.nf
+\f[C]
+server$\ fi_pingpong
+\f[]
+.fi
+.SS Start the client
+.IP
+.nf
+\f[C]
+client$\ fi_pingpong\ <server\ address>
+\f[]
+.fi
+.SH OPTIONS
+.PP
+The server and client must be able to communicate properly for the
+fi_pingpong utility to function.
+If any of the \f[C]\-e\f[], \f[C]\-I\f[], \f[C]\-S\f[], or \f[C]\-p\f[]
+options are used, then they must be specified on the invocation for both
+the server and the client process.
+If the \f[C]\-d\f[] option is specified on the server, then the client
+will select the appropriate domain if no hint is provided on the client
+side.
+If the \f[C]\-d\f[] option is specified on the client, then it must also
+be specified on the server.
+If both the server and client specify the \f[C]\-d\f[] option and the
+given domains cannot communicate, then the application will fail.
+.SS Control Messaging
+.PP
+\f[I]\-B <src_port>\f[] : The non\-default source port number of the
+control socket.
+If this is not provided then the server will bind to port 47592 by
+default and the client will allow the port to be selected automatically.
+.PP
+\f[I]\-P <dest_port>\f[] : The non\-default destination port number of
+the control socket.
+If this is not provided then the client will connect to 47592 by
+default.
+The server ignores this option.
+.SS Fabric Filtering
+.PP
+\f[I]\-p <provider_name>\f[] : The name of the underlying fabric
+provider (e.g., sockets, psm, usnic, etc.).
+If a provider is not specified via the \-p switch, the test will pick
+one from the list of available providers (as returned by fi_getinfo(3)).
+.PP
+\f[I]\-e <endpoint>\f[] : The type of endpoint to be used for data
+messaging between the two processes.
+Supported values are dgram, rdm, and msg.
+For more information on endpoint types, see fi_endpoint(3).
+.PP
+\f[I]\-d <domain>\f[] : The name of the specific domain to be used.
+.SS Test Options
+.PP
+\f[I]\-I <iter>\f[] : The number of iterations of the test will run.
+.PP
+\f[I]\-S <msg_size>\f[] : The specific size of the message in bytes the
+test will use or \[aq]all\[aq] to run all the default sizes.
+.PP
+\f[I]\-c\f[] : Activate data integrity checks at the receiver (note:
+this will degrade performance).
+.SS Utility
+.PP
+\f[I]\-v\f[] : Activate output debugging (warning: highly verbose)
+.PP
+\f[I]\-h\f[] : Displays help output for the pingpong test.
+.SH USAGE EXAMPLES
+.SS A simple example
+.SS Server: \f[C]fi_pingpong\ \-p\ <provider_name>\f[]
+.PP
+\f[C]server$\ fi_pingpong\ \-p\ sockets\f[]
+.SS Client: \f[C]fi_pingpong\ \-p\ <provider_name>\ <server_addr>\f[]
+.PP
+\f[C]client$\ fi_pingpong\ \-p\ sockets\ 192.168.0.123\f[]
+.SS An example with various options
+.SS Server:
+.PP
+\f[C]server$\ fi_pingpong\ \-p\ usnic\ \-I\ 1000\ \-S\ 1024\f[]
+.SS Client:
+.PP
+\f[C]client$\ fi_pingpong\ \-p\ usnic\ \-I\ 1000\ \-S\ 1024\ 192.168.0.123\f[]
+.PP
+Specifically, this will run a pingpong test with:
+.IP \[bu] 2
+usNIC provider
+.IP \[bu] 2
+1000 iterations
+.IP \[bu] 2
+1024 bytes message size
+.IP \[bu] 2
+server node as 192.168.0.123
+.SS A longer test
+.SS Server:
+.PP
+\f[C]server$\ fi_pingpong\ \-p\ usnic\ \-I\ 10000\ \-S\ all\f[]
+.SS Client:
+.PP
+\f[C]client$\ fi_pingpong\ \-p\ usnic\ \-I\ 10000\ \-S\ all\ 192.168.0.123\f[]
+.SH DEFAULTS
+.PP
+There is no default provider; if a provider is not specified via the
+\f[C]\-p\f[] switch, the test will pick one from the list of available
+providers (as returned by fi_getinfo(3)).
+.PP
+If no endpoint type is specified, \[aq]dgram\[aq] is used.
+.PP
+The default tested sizes are: 64, 256, 1024, 4096, 65536, and 1048576.
+The test will only test sizes that are within the selected endpoints
+maximum message size boundary.
+.SH OUTPUT
+.PP
+Each test generates data messages which are accounted for.
+Specifically, the displayed statistics at the end are :
+.IP \[bu] 2
+\f[I]bytes\f[] : number of bytes per message sent
+.IP \[bu] 2
+\f[I]#sent\f[] : number of messages (ping) sent from the client to the
+server
+.IP \[bu] 2
+\f[I]#ack\f[] : number of replies (pong) of the server received by the
+client
+.IP \[bu] 2
+\f[I]total\f[] : amount of memory exchanged between the processes
+.IP \[bu] 2
+\f[I]time\f[] : duration of this single test
+.IP \[bu] 2
+\f[I]MB/sec\f[] : throughput computed from \f[I]total\f[] and
+\f[I]time\f[]
+.IP \[bu] 2
+\f[I]usec/xfer\f[] : average time for transferring a message outbound
+(ping or pong) in microseconds
+.IP \[bu] 2
+\f[I]Mxfers/sec\f[] : average amount of transfers of message outbound
+per second
+.SH SEE ALSO
+.PP
+\f[C]fi_getinfo\f[](3), \f[C]fi_endpoint\f[](3) \f[C]fabric\f[](7),
+.SH AUTHORS
+OpenFabrics.
diff --git a/man/man1/fi_strerror.1 b/man/man1/fi_strerror.1
new file mode 100644
index 0000000..f903057
--- /dev/null
+++ b/man/man1/fi_strerror.1
@@ -0,0 +1,25 @@
+.TH "fi_strerror" "1" "2016\-06\-30" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
+.SH NAME
+.PP
+fi_strerror \- display libfabric error strings
+.SH SYNOPSIS
+.IP
+.nf
+\f[C]
+fi_strerror\ FI_ERROR_CODE
+\f[]
+.fi
+.SH DESCRIPTION
+.PP
+Display the error string for the given numeric \f[C]FI_ERROR_CODE\f[].
+\f[C]FI_ERROR_CODE\f[] may be a hexadecimal, octal, or decimal constant.
+Although the \f[C]fi_strerror\f[](3) library function only accepts
+positive error values, for convenience this utility accepts both
+positive and negative error values.
+.PP
+This is primarily a convenience tool for developers.
+.SH SEE ALSO
+.PP
+\f[C]fabric\f[](7) \f[C]fi_errno\f[](3)
+.SH AUTHORS
+OpenFabrics.
diff --git a/man/man3/fi_atomic.3 b/man/man3/fi_atomic.3
index d065413..d4d7df8 100644
--- a/man/man3/fi_atomic.3
+++ b/man/man3/fi_atomic.3
@@ -1,4 +1,4 @@
-.TH "fi_atomic" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_atomic" "3" "2016\-09\-19" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_atomic \- Remote atomic functions
@@ -178,6 +178,13 @@ imaginary portion.
.PP
\f[I]FI_LONG_DOUBLE\f[] : A double\-extended precision floating point
value (IEEE 754).
+Note that the size of a long double and number of bits used for
+precision is compiler, platform, and/or provider specific.
+Developers that use long double should ensure that libfabric is built
+using a long double format that is compatible with their application,
+and that format is supported by the provider.
+The mechanism used for this validation is currently beyond the scope of
+the libfabric API.
.PP
\f[I]FI_LONG_DOUBLE_COMPLEX\f[] : An ordered pair of double\-extended
precision floating point values (IEEE 754), with the first value
@@ -278,7 +285,7 @@ addr[i]\ =\ addr[i]\ ^\ buf[i]
.IP
.nf
\f[C]
-buf[i]\ =\ addr[i]
+result[i]\ =\ addr[i]
\f[]
.fi
.PP
@@ -388,6 +395,8 @@ for this atomic.
The completion event will be suppressed even if the endpoint has not
been configured with FI_SELECTIVE_COMPLETION.
See the flags discussion below for more details.
+The requested message size that can be used with fi_inject_atomic is
+limited by inject_size.
.PP
The fi_atomicmsg call supports atomic functions over both connected and
unconnected endpoints, with the ability to control the atomic operation
@@ -410,9 +419,9 @@ struct\ fi_msg_atomic\ {
};
struct\ fi_rma_ioc\ {
-\ \ \ \ uint64_t\ \ \ \ \ \ \ \ \ \ \ addr;\ \ \ \ \ \ \ \ \ /*\ target\ address\ */
-\ \ \ \ size_t\ \ \ \ \ \ \ \ \ \ \ \ \ count;\ \ \ \ \ \ \ \ /*\ #\ target\ operands\ */
-\ \ \ \ uint64_t\ \ \ \ \ \ \ \ \ \ \ key;\ \ \ \ \ \ \ \ \ \ /*\ access\ key\ */
+\ \ \ \ uint64_t\ \ \ \ \ \ \ \ \ \ \ addr;\ \ \ \ \ \ /*\ target\ address\ */
+\ \ \ \ size_t\ \ \ \ \ \ \ \ \ \ \ \ \ count;\ \ \ \ \ /*\ #\ target\ operands\ */
+\ \ \ \ uint64_t\ \ \ \ \ \ \ \ \ \ \ key;\ \ \ \ \ \ \ /*\ access\ key\ */
};
\f[]
.fi
@@ -435,6 +444,10 @@ read access.
The following list of atomic operations are usable with fetch atomic
operations: FI_MIN, FI_MAX, FI_SUM, FI_PROD, FI_LOR, FI_LAND, FI_BOR,
FI_BAND, FI_LXOR, FI_BXOR, FI_ATOMIC_READ, and FI_ATOMIC_WRITE.
+.PP
+For FI_ATOMIC_READ operations, the source buffer operand (e.g.
+fi_fetch_atomic buf parameter) is ignored and may be NULL.
+The results are written into the result buffer.
.SS Compare\-Atomic Functions
.PP
The compare atomic functions \-\- fi_compare_atomic, fi_compare_atomicv,
@@ -503,6 +516,7 @@ after the call returns, even if the operation is handled asynchronously.
This may require that the underlying provider implementation copy the
data into a local buffer and transfer out of that buffer.
The use of output result buffers are not affected by this flag.
+This flag can only be used with messages smaller than inject_size.
.PP
\f[I]FI_FENCE\f[] : Indicates that the requested operation, also known
as the fenced operation, be deferred until all previous operations
@@ -514,6 +528,9 @@ On error, a negative value corresponding to fabric errno is returned.
Fabric errno values are defined in \f[C]rdma/fi_errno.h\f[].
.SH ERRORS
.PP
+\f[I]\-FI_EAGAIN\f[] : See \f[C]fi_msg\f[](3) for a detailed description
+of handling FI_EAGAIN.
+.PP
\f[I]\-FI_EOPNOTSUPP\f[] : The requested atomic operation is not
supported on this endpoint.
.PP
@@ -532,11 +549,11 @@ bytes to an aligned memory location.
.nf
\f[C]
fi_atomic(ep,\ buf,\ count,\ NULL,\ dest_addr,\ addr,\ key,
-\ \ \ \ FI_UINT64,\ FI_ATOMIC_WRITE,\ context);
+\ \ \ \ \ \ FI_UINT64,\ FI_ATOMIC_WRITE,\ context)
{
\ \ \ \ for\ (i\ =\ 1;\ i\ <\ count;\ i\ ++)
\ \ \ \ \ \ \ \ ATOMIC_WRITE_U64(((uint64_t\ *)\ addr)[i],
-\ \ \ \ \ \ \ \ \ \ \ \ ((uint64_t\ *)\ buf)[i]);
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ((uint64_t\ *)\ buf)[i]);
}
\f[]
.fi
diff --git a/man/man3/fi_av.3 b/man/man3/fi_av.3
index a942d41..be81049 100644
--- a/man/man3/fi_av.3
+++ b/man/man3/fi_av.3
@@ -1,4 +1,4 @@
-.TH "fi_av" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_av" "3" "2016\-09\-19" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_av \- Address vector operations
@@ -47,7 +47,7 @@ fi_addr_t\ fi_rx_addr(fi_addr_t\ fi_addr,\ int\ rx_index,
\ \ \ \ \ \ int\ rx_ctx_bits);
const\ char\ *\ fi_av_straddr(struct\ fid_av\ *av,\ const\ void\ *addr,
-\ \ \ \ \ \ void\ *buf,\ size_t\ *len);
+\ \ \ \ \ \ char\ *buf,\ size_t\ *len);
\f[]
.fi
.SH ARGUMENTS
@@ -260,7 +260,8 @@ The number of addresses is specified through the count parameter.
The addr parameter references an array of addresses to insert into the
AV.
Addresses inserted into an address vector must be in the same format as
-specified in struct fi_info:addr_format for the corresponding domain.
+specified in the addr_format field of the fi_info struct provided when
+opening the corresponding domain.
.PP
For AV\[aq]s of type FI_AV_MAP, once inserted addresses have been
mapped, the mapped values are written into the buffer referenced by
@@ -294,6 +295,11 @@ Note that if fi_addr is NULL and synchronous operation is requested,
individual insertion failures cannot be reported and the application
must use other calls, such as \f[C]fi_av_lookup\f[] to learn which
specific addresses failed to insert.
+Since fi_av_remove is provider\-specific, it is recommended that calls
+to fi_av_insert following a call to fi_av_remove always reference a
+valid buffer in the fi_addr parameter.
+Otherwise it may be difficult to determine what the next assigned index
+will be.
.PP
\f[I]flags\f[] : The following flag may be passed to fi_av_insert
.IP \[bu] 2
@@ -342,9 +348,13 @@ The total number of inserted addresses will be nodecnt x svccnt.
.SS fi_av_remove
.PP
fi_av_remove removes a set of addresses from an address vector.
-All resources associated with the indicated addresses are released, and
-no future references to either the mapped address (in the case of
-FI_AV_MAP) or index (FI_AV_TABLE) are allowed.
+All resources associated with the indicated addresses are released.
+The removed address \- either the mapped address (in the case of
+FI_AV_MAP) or index (FI_AV_TABLE) \- is invalid until it is returned
+again by a new fi_av_insert.
+.PP
+The behavior of operations in progress that reference the removed
+addresses is undefined.
.PP
The use of fi_av_remove is an optimization that applications may use to
free memory allocated with addresses that will no longer be accessed.
@@ -407,6 +417,10 @@ FI_EVENT flag specified) will return 0 if the operation was successfully
initiated.
In the case of failure, a negative fabric errno will be returned.
.PP
+In both the synchronous and asynchronous modes of operation, the fi_addr
+buffer associated with a failed insertion will be set to
+FI_ADDR_NOTAVAIL.
+.PP
All other calls return 0 on success, or a negative value corresponding
to fabric errno on error.
Fabric errno values are defined in \f[C]rdma/fi_errno.h\f[].
diff --git a/man/man3/fi_cm.3 b/man/man3/fi_cm.3
index 8c089ff..557ab60 100644
--- a/man/man3/fi_cm.3
+++ b/man/man3/fi_cm.3
@@ -1,4 +1,4 @@
-.TH "fi_cm" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_cm" "3" "2016\-09\-19" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_cm \- Connection management operations
@@ -80,6 +80,7 @@ but may be adjusted through the fi_control call.
.PP
The fi_connect call initiates a connection request on a
connection\-oriented endpoint to the destination address.
+fi_connect may only be called on an endpoint once in its lifetime.
.SS fi_accept / fi_reject
.PP
The fi_accept and fi_reject calls are used on the passive (listening)
diff --git a/man/man3/fi_cntr.3 b/man/man3/fi_cntr.3
index cf637f9..d3ea560 100644
--- a/man/man3/fi_cntr.3
+++ b/man/man3/fi_cntr.3
@@ -1,4 +1,4 @@
-.TH "fi_cntr" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_cntr" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_cntr \- Completion and event counter operations
diff --git a/man/man3/fi_control.3 b/man/man3/fi_control.3
index 2669e17..503ea73 100644
--- a/man/man3/fi_control.3
+++ b/man/man3/fi_control.3
@@ -1,4 +1,4 @@
-.TH "fi_control" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_control" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_control \- Perform an operation on a fabric resource.
diff --git a/man/man3/fi_cq.3 b/man/man3/fi_cq.3
index 4d761a1..8601f83 100644
--- a/man/man3/fi_cq.3
+++ b/man/man3/fi_cq.3
@@ -1,4 +1,4 @@
-.TH "fi_cq" "3" "2016\-03\-09" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_cq" "3" "2016\-06\-23" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_cq \- Completion queue operations
@@ -126,7 +126,9 @@ struct\ fi_cq_attr\ {
A value of 0 indicates that the provider may choose a default value.
.PP
\f[I]flags\f[] : Flags that control the configuration of the CQ.
-This field must be 0.
+.IP \[bu] 2
+\f[I]FI_AFFINITY\f[] : Indicates that the signaling_vector field (see
+below) is valid.
.PP
\f[I]format\f[] : Completion queues allow the application to select the
amount of detail that it must store and report.
@@ -167,7 +169,7 @@ struct\ fi_cq_msg_entry\ {
.fi
.IP \[bu] 2
\f[I]FI_CQ_FORMAT_DATA\f[] : Provides data associated with a completion.
-Includes support for received message length, remote EQ data, and
+Includes support for received message length, remote CQ data, and
multi\-receive buffers.
.IP
.nf
@@ -235,9 +237,16 @@ readable, writable, or with an error.
.IP \[bu] 2
\f[I]FI_WAIT_MUTEX_COND\f[] : Specifies that the CQ should use a pthread
mutex and cond variable as a wait object.
-.PP
-\f[I]signaling_vector\f[] : Indicates which processor core interrupts
-associated with the EQ should target.
+.IP \[bu] 2
+\f[I]FI_WAIT_CRITSEC_COND\f[] : Windows specific.
+Specifies that the CQ should use a critical section and condition
+variable as a wait object.
+.PP
+\f[I]signaling_vector\f[] : If the FI_AFFINITY flag is set, this
+indicates the logical cpu number (0..max cpu \- 1) that interrupts
+associated with the CQ should target.
+This field should be treated as a hint to the provider and may be
+ignored if the provider does not support interrupt affinity.
.PP
\f[I]wait_cond\f[] : By default, when a completion is inserted into an
CQ that supports blocking reads (fi_cq_sread/fi_cq_sreadfrom), the
diff --git a/man/man3/fi_domain.3 b/man/man3/fi_domain.3
index 9c3bd5a..b1d9400 100644
--- a/man/man3/fi_domain.3
+++ b/man/man3/fi_domain.3
@@ -1,5 +1,5 @@
.\"t
-.TH "fi_domain" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_domain" "3" "2016\-09\-26" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_domain \- Open a fabric access domain
@@ -532,7 +532,7 @@ CQ data must be at least 4\-bytes.
The optimal number of completion queues supported by the domain,
relative to any specified or default CQ attributes.
The cq_cnt value may be a fixed value of the maximum number of CQs
-supported by the underlying provider, or may be a dynamic value, based
+supported by the underlying hardware, or may be a dynamic value, based
on the default attributes of an allocated CQ, such as the CQ size and
data format.
.SS Endpoint Count (ep_cnt)
@@ -540,7 +540,7 @@ data format.
The total number of endpoints supported by the domain, relative to any
specified or default endpoint attributes.
The ep_cnt value may be a fixed value of the maximum number of endpoints
-supported by the underlying provider, or may be a dynamic value, based
+supported by the underlying hardware, or may be a dynamic value, based
on the default attributes of an allocated endpoint, such as the endpoint
capabilities and size.
The endpoint count is the number of addressable endpoints supported by
@@ -594,9 +594,17 @@ Fabric errno values are defined in \f[C]rdma/fi_errno.h\f[].
Users should call fi_close to release all resources allocated to the
fabric domain.
.PP
-The following fabric resources are associated with access domains:
-active endpoints, memory regions, completion event queues, and address
-vectors.
+The following fabric resources are associated with domains: active
+endpoints, memory regions, completion event queues, and address vectors.
+.PP
+Domain attributes reflect the limitations and capabilities of the
+underlying hardware and/or software provider.
+They do not reflect system limitations, such as the number of physical
+pages that an application may pin or number of file descriptors that the
+application may open.
+As a result, the reported maximums may not be achievable, even on a
+lightly loaded systems, without an administrator configuring system
+resources appropriately for the installed provider(s).
.SH SEE ALSO
.PP
\f[C]fi_getinfo\f[](3), \f[C]fi_endpoint\f[](3), \f[C]fi_av\f[](3),
diff --git a/man/man3/fi_endpoint.3 b/man/man3/fi_endpoint.3
index 1796b87..8784562 100644
--- a/man/man3/fi_endpoint.3
+++ b/man/man3/fi_endpoint.3
@@ -1,4 +1,4 @@
-.TH "fi_endpoint" "3" "2016\-03\-05" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_endpoint" "3" "2016\-09\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_endpoint \- Fabric endpoint operations
@@ -9,8 +9,8 @@ Allocate or close an endpoint.
.RE
.TP
.B fi_ep_bind
-Associate an endpoint with an event queue, completion queue, counter,
-address vector, or memory region
+Associate an endpoint with an event queue, completion queue, counter, or
+address vector
.RS
.RE
.TP
@@ -25,7 +25,7 @@ Associate a passive endpoint with an event queue
.RE
.TP
.B fi_enable
-Transitions an endpoint into an active state.
+Transitions an active endpoint into an enabled state.
.RS
.RE
.TP
@@ -34,7 +34,7 @@ Cancel a pending asynchronous data transfer
.RS
.RE
.TP
-.B fi_alias
+.B fi_ep_alias
Create an alias to the endpoint
.RS
.RE
@@ -104,7 +104,7 @@ int\ fi_enable(struct\ fid_ep\ *ep);
int\ fi_cancel(struct\ fid_ep\ *ep,\ void\ *context);
-int\ fi_alias(struct\ fid_ep\ *ep,\ fid_t\ *alias_ep,\ uint64_t\ flags);
+int\ fi_ep_alias(struct\ fid_ep\ *ep,\ struct\ fid_ep\ **alias_ep,\ uint64_t\ flags);
int\ fi_control(struct\ fid\ *ep,\ int\ command,\ void\ *arg);
@@ -183,7 +183,7 @@ to the receive queue.
Active endpoints are created in the disabled state.
They must transition into an enabled state before accepting data
transfer operations, including posting of receive buffers.
-The fi_enable call is used to transition an endpoint into an active
+The fi_enable call is used to transition an active endpoint into an
enabled state.
The fi_connect and fi_accept calls will also transition an endpoint into
the enabled state, if it is not already active.
@@ -192,12 +192,13 @@ In order to transition an endpoint into an enabled state, it must be
bound to one or more fabric resources.
An endpoint that will generate asynchronous completions, either through
data transfer operations or communication establishment events, must be
-bound to the appropriate completion queues or event queues before being
-enabled.
+bound to the appropriate completion queues or event queues,
+respectively, before being enabled.
+Unconnected endpoints must be bound to an address vector.
.PP
-Once an endpoint has been activated, it may be associated with memory
-regions and address vectors.
-Receive buffers may be posted to it, and calls may be made to connection
+Once an endpoint has been activated, it may be associated with an
+address vector.
+Receive buffers may be posted to it and calls may be made to connection
establishment routines.
Connectionless endpoints may also perform data transfers.
.PP
@@ -282,9 +283,6 @@ binding an endpoint to a completion domain CQ.
\f[I]FI_TRANSMIT\f[] : Directs the completion of outbound data transfer
requests to the specified completion queue.
This includes send message, RMA, and atomic operations.
-The FI_SEND flag may be used interchangeably.
-This binding automatically includes FI_READ and FI_WRITE, if applicable
-to the endpoint.
.PP
\f[I]FI_RECV\f[] : Directs the notification of inbound data transfers to
the specified completion queue.
@@ -300,8 +298,8 @@ completions are generated.
If FI_SELECTIVE_COMPLETION is specified, data transfer operations will
not generate entries for successful completions unless FI_COMPLETION is
set as an operational flag for the given operation.
-FI_SELECTIVE_COMPLETION must be OR\[aq]ed with FI_SEND and/or FI_RECV
-flags.
+FI_SELECTIVE_COMPLETION must be OR\[aq]ed with FI_TRANSMIT and/or
+FI_RECV flags.
.PP
When FI_SELECTIVE_COMPLETION is set, the user must determine when a
request that does NOT have FI_COMPLETION set has completed indirectly,
@@ -315,7 +313,7 @@ using the following general approach:
.nf
\f[C]
\ \ fi_tx_attr::op_flags\ =\ 0;\ //\ default\ \-\ no\ completion
-\ \ fi_ep_bind(ep,\ cq,\ FI_SEND\ |\ FI_SELECTIVE_COMPLETION);
+\ \ fi_ep_bind(ep,\ cq,\ FI_TRANSMIT\ |\ FI_SELECTIVE_COMPLETION);
\ \ fi_send(ep,\ ...);\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ //\ no\ completion
\ \ fi_sendv(ep,\ ...);\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ //\ no\ completion
\ \ fi_sendmsg(ep,\ ...,\ FI_COMPLETION);\ //\ completion!
@@ -329,7 +327,7 @@ modifying the operational flags:
.nf
\f[C]
\ \ fi_tx_attr::op_flags\ =\ FI_COMPLETION;\ //\ default\ \-\ completion
-\ \ fi_ep_bind(ep,\ cq,\ FI_SEND\ |\ FI_SELECTIVE_COMPLETION);
+\ \ fi_ep_bind(ep,\ cq,\ FI_TRANSMIT\ |\ FI_SELECTIVE_COMPLETION);
\ \ fi_send(ep,\ ...);\ \ \ \ \ \ \ //\ completion
\ \ fi_sendv(ep,\ ...);\ \ \ \ \ \ //\ completion
\ \ fi_sendmsg(ep,\ ...,\ 0);\ //\ no\ completion!
@@ -343,7 +341,7 @@ completions for all non\-fi_inject calls:
.nf
\f[C]
\ \ fi_tx_attr::op_flags\ =\ 0;
-\ \ fi_ep_bind(ep,\ cq,\ FI_SEND);\ \ //\ default\ \-\ completion
+\ \ fi_ep_bind(ep,\ cq,\ FI_TRANSMIT);\ \ \ \ //\ default\ \-\ completion
\ \ fi_send(ep,\ ...);\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ //\ completion
\ \ fi_sendv(ep,\ ...);\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ //\ completion
\ \ fi_sendmsg(ep,\ ...,\ 0);\ \ \ \ \ \ \ \ \ \ \ \ \ //\ completion!
@@ -357,30 +355,32 @@ An endpoint may also, or instead, be bound to a fabric counter.
When binding an endpoint to a counter, the following flags may be
specified.
.PP
-\f[I]FI_SEND\f[] : Increments the specified counter whenever a
-successful message is transferred over the endpoint.
+\f[I]FI_SEND\f[] : Increments the specified counter whenever a message
+transfer initiated over the endpoint has completed successfully or in
+error.
Sent messages include both tagged and normal message operations.
.PP
-\f[I]FI_RECV\f[] : Increments the specified counter whenever a
-successful message is received over the endpoint.
+\f[I]FI_RECV\f[] : Increments the specified counter whenever a message
+is received over the endpoint.
Received messages include both tagged and normal message operations.
.PP
-\f[I]FI_READ\f[] : Increments the specified counter whenever a
-successful RMA read or atomic fetch operation is initiated from the
-endpoint.
+\f[I]FI_READ\f[] : Increments the specified counter whenever an RMA read
+or atomic fetch operation initiated from the endpoint has completed
+successfully or in error.
.PP
-\f[I]FI_WRITE\f[] : Increments the specified counter whenever a
-successful RMA write or atomic operation is initiated from the endpoint.
+\f[I]FI_WRITE\f[] : Increments the specified counter whenever an RMA
+write or atomic operation initiated from the endpoint has completed
+successfully or in error.
.PP
-\f[I]FI_REMOTE_READ\f[] : Increments the specified counter whenever a
-successful RMA read or atomic fetch operation is initiated from a remote
-endpoint that targets the given endpoint.
+\f[I]FI_REMOTE_READ\f[] : Increments the specified counter whenever an
+RMA read or atomic fetch operation is initiated from a remote endpoint
+that targets the given endpoint.
Use of this flag requires that the endpoint be created using
FI_RMA_EVENT.
.PP
-\f[I]FI_REMOTE_WRITE\f[] : Increments the specified counter whenever a
-successful RMA write or atomic operation is initiated from a remote
-endpoint that targets the given endpoint.
+\f[I]FI_REMOTE_WRITE\f[] : Increments the specified counter whenever an
+RMA write or atomic operation is initiated from a remote endpoint that
+targets the given endpoint.
Use of this flag requires that the endpoint be created using
FI_RMA_EVENT.
.PP
@@ -407,6 +407,8 @@ An endpoint must be enabled before it may be used to perform data
transfers.
Enabling an endpoint typically results in hardware resources being
assigned to it.
+Endpoints making use of completion queues, counters, event queues,
+and/or address vectors must be bound to them before being enabled.
.PP
Calling connect or accept on an endpoint will implicitly enable an
endpoint if it has not already been enabled.
@@ -426,15 +428,20 @@ will be canceled.
In this case, the operation which is canceled is provider specific.
The cancel operation is asynchronous, but will complete within a bounded
period of time.
-.SS fi_alias
+.SS fi_ep_alias
.PP
This call creates an alias to the specified endpoint.
Conceptually, an endpoint alias provides an alternate software path from
the application to the underlying provider hardware.
Applications configure an alias endpoint with data transfer flags,
-specified through the fi_alias call.
-Typically, the data transfer flags will be different than those assigned
-to the actual endpoint.
+specified through the fi_ep_alias call.
+The flags must include FI_TRANSMIT or FI_RECV (not both) with other
+flags OR\[aq]ed to indicate the type of data transfer the flags should
+apply to.
+This will override the transmit and recieve attributes of the alias
+endpoint.
+Typically the attributes of the alias endpoint are different than those
+assigned to the actual endpoint.
The alias mechanism allows a single endpoint to have multiple optimized
software interfaces.
All allocated aliases must be closed for the underlying endpoint to be
@@ -454,14 +461,19 @@ The following control commands and arguments may be assigned to an
endpoint.
.PP
**FI_GETOPSFLAG \-\- uint64_t *flags** : Used to retrieve the current
-value of flags associated with data transfer operations initiated on the
-endpoint.
+value of flags associated with the data transfer operations initiated on
+the endpoint.
+The control argument must include FI_TRANSMIT or FI_RECV (not both)
+flags to indicate the type of data transfer flags to be returned.
See below for a list of control flags.
.PP
**FI_SETOPSFLAG \-\- uint64_t *flags** : Used to change the data
transfer operation flags associated with an endpoint.
-The FI_READ, FI_WRITE, FI_SEND, FI_RECV flags indicate the type of data
-transfer that the flags should apply to, with other flags OR\[aq]ed in.
+The control argument must include FI_TRANSMIT or FI_RECV (not both) to
+indicate the type of data transfer that the flags should apply to, with
+other flags OR\[aq]ed in.
+The given flags will override the previous transmit and receive
+attributes that were set when the endpoint was created.
Valid control flags are defined below.
.PP
**FI_BACKLOG \- int *value** : This option only applies to passive
@@ -495,6 +507,10 @@ space in CM messages for user\-defined data.
This value limits the amount of data that applications can exchange
between peer endpoints using the fi_connect, fi_accept, and fi_reject
operations.
+The size returned is dependent upon the properties of the endpoint,
+except in the case of passive endpoints, in which the size reflects the
+maximum size of the data that may be present as part of a connection
+request event.
This option is read only.
.SS fi_rx_size_left
.PP
@@ -702,7 +718,7 @@ of fields, with each field being at least the size requested, or fail
the request.
A provider may increase the size of the fields.
When reporting completions (see FI_CQ_FORMAT_TAGGED), the provider must
-provide the exact value of the recieved tag, clearing out any
+provide the exact value of the received tag, clearing out any
unsupported tag bits.
.PP
It is recommended that field sizes be ordered from smallest to largest.
@@ -818,6 +834,7 @@ met.
\f[I]FI_ORDER_NONE\f[] : No ordering is specified.
This value may be used as input in order to obtain the default message
order supported by the provider.
+FI_ORDER_NONE is an alias for the value 0.
.PP
\f[I]FI_ORDER_RAR\f[] : Read after read.
If set, RMA and atomic read operations are transmitted in the order
@@ -1009,10 +1026,14 @@ and between operations if message ordering is guaranteed.
Defines the total available space allocated by the provider to buffer
messages that are received for which there is no matching receive
operation.
-If set to 0, and the domain does not support FI_RM_ENABLED, any messages
-that arrive before a receive buffer has been posted are lost.
-When the domain supports FI_RM_ENABLED, the actual amount of buffering
-provided may exceed the value specified in total_buffered_recv.
+That is, this defines the minimal amount of receive side buffering
+available.
+If receive side buffering is disabled (total_buffered_recv = 0) and a
+message is received by an endpoint, then the behavior is dependent on
+whether resource management has been enabled (FI_RM_ENABLED has be set
+or not).
+See the Resource Management section of fi_domain.3 for further
+clarification.
.SS size
.PP
The size of the context.
@@ -1162,15 +1183,16 @@ Operation flags are obtained by OR\-ing the following flags together.
Operation flags define the default flags applied to an endpoint\[aq]s
data transfer operations, where a flags parameter is not available.
Data transfer operations that take flags as input override the op_flags
-value of an endpoint.
+value of transmit or receive context attributes of an endpoint.
.PP
\f[I]FI_INJECT\f[] : Indicates that all outbound data buffers should be
returned to the user\[aq]s control immediately after a data transfer
call returns, even if the operation is handled asynchronously.
This may require that the provider copy the data into a local buffer and
transfer out of that buffer.
-A provider may limit the total amount of send data that may be buffered
-and/or the size of a single send.
+A provider can limit the total amount of send data that may be buffered
+and/or the size of a single send that can use this flag.
+This limit is indicated using inject_size (see inject_size above).
.PP
\f[I]FI_MULTI_RECV\f[] : Applies to posted receive operations.
This flag allows the user to post a single buffer that will receive
@@ -1197,7 +1219,8 @@ No other guarantees are made with respect to the state of the operation.
Note: This flag is used to control when a completion entry is inserted
into a completion queue.
It does not apply to operations that do not generate a completion queue
-entry, such as the fi_inject operation.
+entry, such as the fi_inject operation, and is not subject to the
+inject_size message limit restriction.
.PP
\f[I]FI_TRANSMIT_COMPLETE\f[] : Indicates that a completion should be
generated when the transmit operation has completed relative to the
@@ -1257,6 +1280,21 @@ Operations that complete with an error increment the error counter and
generate a completion event.
The generation of entries going to the CQ can then be controlled using
FI_SELECTIVE_COMPLETION.
+.PP
+As mentioned in fi_getinfo(3), the ep_attr structure can be used to
+query providers that support various endpoint attributes.
+fi_getinfo can return provider info structures that can support the
+minimal set of requirements (such that the application maintains
+correctness).
+However, it can also return provider info structures that exceed
+application requirements.
+As an example, consider an application requesting msg_order as
+FI_ORDER_NONE.
+The resulting output from getinfo may have all the ordering bits set.
+The application can reset the ordering bits it does not require before
+creating the endpoint.
+The provider is free to implement a stricter ordering than is required
+by the application.
.SH RETURN VALUES
.PP
Returns 0 on success.
diff --git a/man/man3/fi_eq.3 b/man/man3/fi_eq.3
index 36e16a2..2b0a9c3 100644
--- a/man/man3/fi_eq.3
+++ b/man/man3/fi_eq.3
@@ -1,4 +1,4 @@
-.TH "fi_eq" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_eq" "3" "2016\-06\-23" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_eq \- Event queue operations
@@ -113,6 +113,9 @@ If this flag is set, then the fi_eq_write operation must be supported by
the provider.
If the FI_WRITE flag is not set, then the application may not invoke
fi_eq_write.
+.IP \[bu] 2
+\f[I]FI_AFFINITY\f[] : Indicates that the signaling_vector field (see
+below) is valid.
.PP
\f[I]wait_obj\f[] : EQ\[aq]s may be associated with a specific wait
object.
@@ -150,9 +153,16 @@ readable or with an error.
.IP \[bu] 2
\f[I]FI_WAIT_MUTEX_COND\f[] : Specifies that the EQ should use a pthread
mutex and cond variable as a wait object.
+.IP \[bu] 2
+\f[I]FI_WAIT_CRITSEC_COND\f[] : Windows specific.
+Specifies that the EQ should use a critical section and condition
+variable as a wait object.
.PP
-\f[I]signaling_vector\f[] : Indicates which processor core interrupts
+\f[I]signaling_vector\f[] : If the FI_AFFINITY flag is set, this
+indicates the logical cpu number (0..max cpu \- 1) that interrupts
associated with the EQ should target.
+This field should be treated as a hint to the provider and may be
+ignored if the provider does not support interrupt affinity.
.PP
\f[I]wait_set\f[] : If wait_obj is FI_WAIT_SET, this field references a
wait object to which the event queue should attach.
@@ -394,6 +404,8 @@ information into a printable string for debugging purposes.
.PP
If err_data_size is > 0, then the buffer referenced by err_data is
directly user\-accessible.
+The contents of the buffer will remain valid until a subsequent read
+call against the EQ.
Applications which read the err_data buffer must ensure that they do not
read past the end of the referenced buffer.
.SH RETURN VALUES
diff --git a/man/man3/fi_errno.3 b/man/man3/fi_errno.3
index b02d361..2968af9 100644
--- a/man/man3/fi_errno.3
+++ b/man/man3/fi_errno.3
@@ -1,4 +1,4 @@
-.TH "fi_errno" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_errno" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_errno \- fabric errors
diff --git a/man/man3/fi_fabric.3 b/man/man3/fi_fabric.3
index 11e9962..095fbb5 100644
--- a/man/man3/fi_fabric.3
+++ b/man/man3/fi_fabric.3
@@ -1,4 +1,4 @@
-.TH "fi_fabric" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_fabric" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_fabric \- Fabric domain operations
diff --git a/man/man3/fi_getinfo.3 b/man/man3/fi_getinfo.3
index 93a7a99..1d3edc5 100644
--- a/man/man3/fi_getinfo.3
+++ b/man/man3/fi_getinfo.3
@@ -1,4 +1,4 @@
-.TH "fi_getinfo" "3" "2016\-02\-29" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_getinfo" "3" "2016\-06\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_getinfo / fi_freeinfo \- Obtain / free fabric interface information
@@ -463,9 +463,9 @@ However, when a large number of IOV entries are supported, IOV buffering
may have a negative impact on performance and memory consumption.
The FI_ASYNC_IOV mode indicates that the application must provide the
buffering needed for the IO vectors.
-When set, an application must not modify an IO vector, including any
-related memory descriptor array, until the associated operation has
-completed.
+When set, an application must not modify an IO vector of length > 1,
+including any related memory descriptor array, until the associated
+operation has completed.
.PP
\f[I]FI_RX_CQ_DATA\f[] : This mode bit only applies to data transfers
that set FI_REMOTE_CQ_DATA.
diff --git a/man/man3/fi_mr.3 b/man/man3/fi_mr.3
index 08b130d..cfea4c0 100644
--- a/man/man3/fi_mr.3
+++ b/man/man3/fi_mr.3
@@ -1,4 +1,4 @@
-.TH "fi_mr" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_mr" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_mr \- Memory region operations
diff --git a/man/man3/fi_msg.3 b/man/man3/fi_msg.3
index 2b89fda..08e4b74 100644
--- a/man/man3/fi_msg.3
+++ b/man/man3/fi_msg.3
@@ -1,4 +1,4 @@
-.TH "fi_msg" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_msg" "3" "2016\-09\-19" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_msg \- Message data transfer operations
@@ -151,6 +151,8 @@ The completion event will be suppressed even if the CQ was bound without
FI_SELECTIVE_COMPLETION or the endpoint\[aq]s op_flags contain
FI_COMPLETION.
See the flags discussion below for more details.
+The requested message size that can be used with fi_inject is limited by
+inject_size.
.SS fi_senddata
.PP
The send data call is similar to fi_send, but allows for the sending of
@@ -214,6 +216,7 @@ immediately after the send call returns, even if the operation is
handled asynchronously.
This may require that the underlying provider implementation copy the
data into a local buffer and transfer out of that buffer.
+This flag can only be used with messages smaller than inject_size.
.PP
\f[I]FI_MULTI_RECV\f[] : Applies to posted receive operations.
This flag allows the user to post a single buffer that will receive
@@ -260,15 +263,35 @@ buffer length.
Returns 0 on success.
On error, a negative value corresponding to fabric errno is returned.
Fabric errno values are defined in \f[C]rdma/fi_errno.h\f[].
+.PP
+See the discussion below for details handling FI_EAGAIN.
.SH ERRORS
.PP
\f[I]\-FI_EAGAIN\f[] : Indicates that the underlying provider currently
lacks the resources needed to initiate the requested operation.
-This may be the result of insufficient internal buffering, in the case
-of FI_INJECT, or processing queues are full.
-The operation may be retried after additional provider resources become
-available, usually through the completion of currently outstanding
-operations.
+The reasons for a provider returning FI_EAGAIN are varied.
+However, common reasons include insufficient internal buffering or full
+processing queues.
+.PP
+Insufficient internal buffering is often associated with operations that
+use FI_INJECT.
+In such cases, additional buffering may become available as posted
+operations complete.
+.PP
+Full processing queues may be a temporary state related to local
+processing (for example, a large message is being transferred), or may
+be the result of flow control.
+In the latter case, the queues may remain blocked until additional
+resources are made available at the remote side of the transfer.
+.PP
+In all cases, the operation may be retried after additional resources
+become available.
+It is strongly recommended that applications check for transmit and
+receive completions after receiving FI_EAGAIN as a return value,
+independent of the operation which failed.
+This is particularly important in cases where manual progress is
+employed, as acknowledgements or flow control messages may need to be
+processed in order to resume execution.
.SH SEE ALSO
.PP
\f[C]fi_getinfo\f[](3), \f[C]fi_endpoint\f[](3), \f[C]fi_domain\f[](3),
diff --git a/man/man3/fi_poll.3 b/man/man3/fi_poll.3
index 21a6eea..220c562 100644
--- a/man/man3/fi_poll.3
+++ b/man/man3/fi_poll.3
@@ -1,4 +1,4 @@
-.TH "fi_poll" "3" "2016\-03\-16" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_poll" "3" "2016\-09\-30" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_poll \- Polling and wait set operations
@@ -111,14 +111,23 @@ Removes an completion queue or counter from a poll set.
.PP
Progresses all completion queues and counters associated with a poll set
and checks for events.
-If events have occurred, contexts associated with the completion queues
-and/or counters are returned.
+If events might have occurred, contexts associated with the completion
+queues and/or counters are returned.
Completion queues will return their context if they are not empty.
The context associated with a counter will be returned if the
counter\[aq]s success value or error value have changed since the last
-time fi_poll was called.
+time fi_poll, fi_cntr_set, or fi_cntr_add were called.
The number of contexts is limited to the size of the context array,
indicated by the count parameter.
+.PP
+Note that fi_poll only indicates that events might be available.
+In some cases, providers may consume such events internally, to drive
+progress, for example.
+This can result in fi_poll returning false positives.
+Applications should drive their progress based on the results of reading
+events from a completion queue or reading counter values.
+The fi_poll function will always return all completion queues and
+counters that do have new events.
.SS fi_wait_open
.PP
fi_wait_open allocates a new wait set.
@@ -168,6 +177,10 @@ readable or with an error.
.IP \[bu] 2
\f[I]FI_WAIT_MUTEX_COND\f[] : Specifies that the wait set should use a
pthread mutex and cond variable as a wait object.
+.IP \[bu] 2
+\f[I]FI_WAIT_CRITSEC_COND\f[] : Windows specific.
+Specifies that the EQ should use a critical section and condition
+variable as a wait object.
.PP
\f[I]flags\f[] : Flags that set the default operation of the wait set.
The use of this field is reserved and must be set to 0 by the caller.
diff --git a/man/man3/fi_rma.3 b/man/man3/fi_rma.3
index 695ce59..6edd4a5 100644
--- a/man/man3/fi_rma.3
+++ b/man/man3/fi_rma.3
@@ -1,4 +1,4 @@
-.TH "fi_rma" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_rma" "3" "2016\-09\-19" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_rma \- Remote memory access operations
@@ -173,6 +173,8 @@ this write.
The completion event will be suppressed even if the endpoint has not
been configured with FI_SELECTIVE_COMPLETION.
See the flags discussion below for more details.
+The requested message size that can be used with fi_inject_write is
+limited by inject_size.
.SS fi_writedata
.PP
The write data call is similar to fi_write, but allows for the sending
@@ -231,6 +233,7 @@ immediately after the write call returns, even if the operation is
handled asynchronously.
This may require that the underlying provider implementation copy the
data into a local buffer and transfer out of that buffer.
+This flag can only be used with messages smaller than inject_size.
.PP
\f[I]FI_INJECT_COMPLETE\f[] : Applies to fi_writemsg.
Indicates that a completion should be generated when the source
@@ -255,13 +258,8 @@ On error, a negative value corresponding to fabric errno is returned.
Fabric errno values are defined in \f[C]rdma/fi_errno.h\f[].
.SH ERRORS
.PP
-\f[I]\-FI_EAGAIN\f[] : Indicates that the underlying provider currently
-lacks the resources needed to initiate the requested operation.
-This may be the result of insufficient internal buffering, in the case
-of FI_INJECT, or processing queues are full.
-The operation may be retried after additional provider resources become
-available, usually through the completion of currently outstanding
-operations.
+\f[I]\-FI_EAGAIN\f[] : See \f[C]fi_msg\f[](3) for a detailed description
+of handling FI_EAGAIN.
.SH SEE ALSO
.PP
\f[C]fi_getinfo\f[](3), \f[C]fi_endpoint\f[](3), \f[C]fi_domain\f[](3),
diff --git a/man/man3/fi_tagged.3 b/man/man3/fi_tagged.3
index 1b7de81..a71d24d 100644
--- a/man/man3/fi_tagged.3
+++ b/man/man3/fi_tagged.3
@@ -1,4 +1,4 @@
-.TH "fi_tagged" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_tagged" "3" "2016\-09\-19" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_tagged \- Tagged data transfer operations
@@ -175,6 +175,8 @@ send.
The completion event will be suppressed even if the endpoint has not
been configured with FI_SELECTIVE_COMPLETION.
See the flags discussion below for more details.
+The requested message size that can be used with fi_tinject is limited
+by inject_size.
.SS fi_tsenddata
.PP
The tagged send data call is similar to fi_tsend, but allows for the
@@ -237,6 +239,7 @@ immediately after the send call returns, even if the operation is
handled asynchronously.
This may require that the underlying provider implementation copy the
data into a local buffer and transfer out of that buffer.
+This flag can only be used with messages smaller than inject_size.
.PP
\f[I]FI_INJECT_COMPLETE\f[] : Applies to fi_tsendmsg.
Indicates that a completion should be generated when the source
@@ -325,13 +328,8 @@ returned.
Fabric errno values are defined in \f[C]fi_errno.h\f[].
.SH ERRORS
.PP
-\f[I]\-FI_EAGAIN\f[] : Indicates that the underlying provider currently
-lacks the resources needed to initiate the requested operation.
-This may be the result of insufficient internal buffering, in the case
-of FI_INJECT, or processing queues are full.
-The operation may be retried after additional provider resources become
-available, usually through the completion of currently outstanding
-operations.
+\f[I]\-FI_EAGAIN\f[] : See \f[C]fi_msg\f[](3) for a detailed description
+of handling FI_EAGAIN.
.PP
\f[I]\-FI_EINVAL\f[] : Indicates that an invalid argument was supplied
by the user.
diff --git a/man/man3/fi_trigger.3 b/man/man3/fi_trigger.3
index 9a367a8..a8ccc82 100644
--- a/man/man3/fi_trigger.3
+++ b/man/man3/fi_trigger.3
@@ -1,4 +1,4 @@
-.TH "fi_trigger" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_trigger" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_trigger \- Triggered operations
diff --git a/man/man3/fi_version.3 b/man/man3/fi_version.3
index e2e8fc1..2d48630 100644
--- a/man/man3/fi_version.3
+++ b/man/man3/fi_version.3
@@ -1,4 +1,4 @@
-.TH "fi_version" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_version" "3" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
fi_version \- Version of the library interfaces
diff --git a/man/man7/fabric.7 b/man/man7/fabric.7
index 051ed58..1746e21 100644
--- a/man/man7/fabric.7
+++ b/man/man7/fabric.7
@@ -1,4 +1,4 @@
-.TH "fabric" "7" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fabric" "7" "2016\-06\-29" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
Fabric Interface Library
@@ -231,6 +231,25 @@ operations.
\f[I]eq\f[] : Provides output specific to event queue operations.
.IP \[bu] 2
\f[I]mr\f[] : Provides output specific to memory registration.
+.SH NOTES
+.PP
+Because libfabric is designed to provide applications direct access to
+fabric hardware, there are limits on how libfabric resources may be used
+in conjunction with system calls.
+These limitations are notable for developers who may be familiar
+programming to the sockets interface.
+Although limits are provider specific, the following restrictions apply
+to many providers and should be adhered to by applications desiring
+portability across providers.
+.PP
+\f[I]fork\f[] : Fabric resources are not guaranteed to be available by
+child processes.
+This includes objects, such as endpoints and completion queues, as well
+as application controlled data buffers which have been assigned to the
+network.
+For example, data buffers that have been registered with a fabric domain
+may not be available in a child process because of copy on write
+restrictions.
.SH SEE ALSO
.PP
\f[C]fi_provider\f[](7), \f[C]fi_getinfo\f[](3),
diff --git a/man/man7/fi_direct.7 b/man/man7/fi_direct.7
index d6beaa6..540f5ec 100644
--- a/man/man7/fi_direct.7
+++ b/man/man7/fi_direct.7
@@ -1,4 +1,4 @@
-.TH "fi_direct" "7" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_direct" "7" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
Direct fabric provider access
diff --git a/man/man7/fi_gni.7 b/man/man7/fi_gni.7
index 510185d..507d8dd 100644
--- a/man/man7/fi_gni.7
+++ b/man/man7/fi_gni.7
@@ -1,4 +1,4 @@
-.TH "fi_gni" "7" "2016\-03\-30" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_gni" "7" "2016\-10\-07" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
The GNI Fabric Provider
@@ -96,20 +96,27 @@ The GNI provider exposes low\-level tuning parameters via a domain and
endpoint \f[C]fi_open_ops\f[] interface named
\f[I]FI_GNI_DOMAIN_OPS_1\f[] and \f[I]FI_GNI_EP_OPS_1\f[].
The flags parameter is currently ignored.
-The fi_open_ops function takes either a
-\f[C]struct\ fi_gni_ops_domain\f[] or a \f[C]struct\ fi_gni_ops_ep\f[]
-parameter and populates it with the following:
+The fi_open_ops function takes a \f[C]struct\ fi_gni_ops_domain\f[] or a
+\f[C]struct\ fi_gni_ops_ep\f[] parameter respectively and populates it
+with the following:
.IP
.nf
\f[C]
struct\ fi_gni_ops_domain\ {
\ \ \ \ int\ (*set_val)(struct\ fid\ *fid,\ dom_ops_val_t\ t,\ void\ *val);
\ \ \ \ int\ (*get_val)(struct\ fid\ *fid,\ dom_ops_val_t\ t,\ void\ *val);
+\ \ \ \ int\ (*flush_cache)(struct\ fid\ *fid);
};
struct\ fi_gni_ops_ep\ {
\ \ \ \ int\ (*set_val)(struct\ fid\ *fid,\ dom_ops_val_t\ t,\ void\ *val);
\ \ \ \ int\ (*get_val)(struct\ fid\ *fid,\ dom_ops_val_t\ t,\ void\ *val);
+\ \ \ \ \ \ \ \ size_t\ (*native_amo)(struct\ fid_ep\ *ep,\ const\ void\ *buf,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ size_t\ count,void\ *desc,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ fi_addr_t\ dest_addr,\ uint64_t\ addr,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ uint64_t\ key,\ enum\ fi_datatype\ datatype,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ enum\ gnix_fab_req_type\ req_type,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ void\ *context);
};
\f[]
.fi
@@ -172,14 +179,73 @@ The value is of type uint32_t.
of memory.
The value is of type int32_t.
.PP
+\f[I]GNI_MR_CACHE\f[] : Select the type of cache that the domain will
+use.
+Valid choices are the following: \[aq]internal\[aq], \[aq]udreg\[aq], or
+\[aq]none\[aq].
+\[aq]internal\[aq] refers to the GNI provider internal registration
+cache.
+\[aq]udreg\[aq] refers to a user level dreg library based cache.
+Lastly, \[aq]none\[aq] refers to device direct registration without a
+provider cache.
+.PP
+\f[I]GNI_MR_HARD_REG_LIMIT\f[] : Maximum number of registrations.
+Applies only to the GNI provider cache.
+The value is of type int32_t (\-1 for no limit).
+.PP
+\f[I]GNI_MR_SOFT_REG_LIMIT\f[] : Soft cap on the registration limit.
+Applies only to the GNI provider cache.
+The value is of type int32_t (\-1 for no limit).
+.PP
+\f[I]GNI_MR_HARD_STALE_REG_LIMIT\f[] : Maximum number of stale
+registrations to be held in cache.
+This applies to the GNI provider cache and the udreg cache.
+The value is of type int32_t (\-1 for no limit for the GNI provider
+cache and udreg cache values must be greater than 0).
+.PP
+\f[I]GNI_MR_UDREG_LIMIT\f[] : Maximum number of registrations.
+Applies only to the udreg cache.
+The value is of type int32_t.
+The value must be greater than 0.
+.PP
+\f[I]GNI_XPMEM_ENABLE\f[] : Enable or disable use of XPMEM for on node
+messages using the GNI provider internal rendezvous protocol.
+The value is of type bool.
+.PP
+The \f[C]flush_cache\f[] function allows the user to flush any stale
+registration cache entries from the cache.
+This has the effect of removing registrations from the cache that have
+been deregistered with the provider, but still exist in case that they
+may be reused in the near future.
+Flushing the stale registrations forces hardware\-level deregistration
+of the stale memory registrations and frees any memory related to those
+stale registrations.
+Only the provider\-level registration struct is freed, not the user
+buffer associated with the registration.
+The parameter for \f[C]flush_cache\f[] is a struct fid pointer to a
+fi_domain.
+The memory registration cache is tied to the domain, so issuing a
+\f[C]flush_cache\f[] to the domain will flush the registration cache of
+the domain.
+.PP
For \f[I]FI_GNI_EP_OPS_1\f[], the currently supported values are:
\f[I]GNI_HASH_TAG_IMPL\f[] : Use a hashlist for the tag list
implementation.
The value is of type uint32_t.
+.PP
+The \f[C]native_amo\f[] function allows the user to call GNI native
+atomics that are not implemented in the libfabric API.
+The parameters for native_amo are the same as the fi_atomic function but
+adds the following parameter:
+.PP
+\f[I]enum gnix_fab_req_type req_type\f[] : The req_type\[aq]s supported
+with this call are GNIX_FAB_RQ_NAMO_AX (AND and XOR), and
+GNIX_FAB_RQ_NAMO_AX_S (AND and XOR 32 bit), GNIX_FAB_RQ_NAMO_FAX (Fetch
+AND and XOR) and GNIX_FAB_RQ_NAMO_FAX_S (Fetch AND and XOR 32 bit).
.SH SEE ALSO
.PP
\f[C]fabric\f[](7), \f[C]fi_open_ops\f[](3), \f[C]fi_provider\f[](7),
-\f[C]fi_getinfo\f[](3)
+\f[C]fi_getinfo\f[](3) \f[C]fi_atomic\f[](3)
.PP
For more information on uGNI, see \f[I]Using the GNI and DMAPP APIs\f[]
(S\-2446\-3103, Cray Inc.).
diff --git a/man/man7/fi_mxm.7 b/man/man7/fi_mxm.7
index 250c543..3f0e8c5 100644
--- a/man/man7/fi_mxm.7
+++ b/man/man7/fi_mxm.7
@@ -1,9 +1,12 @@
-.TH "fi_mxm" "7" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_mxm" "7" "2016\-09\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
The MXM Fabric Provider
.SH OVERVIEW
.PP
+The mxm provider is deprecated and will be replaced in a future release
+of libfabric.
+.PP
The \f[I]mxm\f[] provider runs over the MXM (Mellanox messaging)
interface that is currently supported by the Mellanox infiniband
fabrics.
diff --git a/man/man7/fi_provider.7 b/man/man7/fi_provider.7
index a7eb620..bfe6a79 100644
--- a/man/man7/fi_provider.7
+++ b/man/man7/fi_provider.7
@@ -1,4 +1,4 @@
-.TH "fi_provider" "7" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_provider" "7" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
Fabric Interface Providers
diff --git a/man/man7/fi_psm.7 b/man/man7/fi_psm.7
index 21f14d6..289121f 100644
--- a/man/man7/fi_psm.7
+++ b/man/man7/fi_psm.7
@@ -1,4 +1,4 @@
-.TH "fi_psm" "7" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_psm" "7" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
The PSM Fabric Provider
diff --git a/man/man7/fi_psm2.7 b/man/man7/fi_psm2.7
index 45350b7..4f2c9d6 100644
--- a/man/man7/fi_psm2.7
+++ b/man/man7/fi_psm2.7
@@ -1,4 +1,4 @@
-.TH "fi_psm2" "7" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_psm2" "7" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
The PSM2 Fabric Provider
diff --git a/man/man7/fi_sockets.7 b/man/man7/fi_sockets.7
index bdba518..71f9382 100644
--- a/man/man7/fi_sockets.7
+++ b/man/man7/fi_sockets.7
@@ -1,4 +1,4 @@
-.TH "fi_sockets" "7" "2016\-03\-29" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_sockets" "7" "2016\-03\-29" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
The Sockets Fabric Provider
diff --git a/man/man7/fi_udp.7 b/man/man7/fi_udp.7
index eb7cf60..0f2e9d5 100644
--- a/man/man7/fi_udp.7
+++ b/man/man7/fi_udp.7
@@ -1,4 +1,4 @@
-.TH "fi_udp" "7" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_udp" "7" "2016\-02\-28" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
The UDP Fabric Provider
diff --git a/man/man7/fi_usnic.7 b/man/man7/fi_usnic.7
index 72382b2..c47364b 100644
--- a/man/man7/fi_usnic.7
+++ b/man/man7/fi_usnic.7
@@ -1,4 +1,4 @@
-.TH "fi_usnic" "7" "2016\-03\-30" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_usnic" "7" "2016\-10\-11" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
The usNIC Fabric Provider
@@ -86,6 +86,9 @@ to an endpoint.
If fi_control is used on an unbound CQ, it will return \-FI_EOPBADSTATE.
.RE
.IP \[bu] 2
+Resource management is not supported.
+The application is responsible for resource protection.
+.IP \[bu] 2
The usnic libfabric provider supports extensions that provide
information and functionality beyond the standard libfabric interface.
See the "USNIC EXTENSIONS" section, below.
@@ -116,6 +119,20 @@ It is recommended that applications explicitly request a version rather
than using the header provided \f[C]FI_EXT_USNIC_INFO_VERSION\f[].
Although there is a version 1 of the extension, its use is discouraged,
and it may not be available in future releases.
+.SS Compatibility issues
+.PP
+The addition of version 2 of the extension caused an alignmment issue
+that could lead to invalid data in the v1 portion of the structure.
+This means that the alignment difference manifests when an application
+using v1 of the extension is compiled with Libfabric v1.1.x or v1.2.x,
+but then runs with Libfabric.so that is v1.3.x or higher (and vice
+versa).
+.PP
+The v1.4.0 release of Libfabric introduced a padding field to explicitly
+maintain compatibility with the v1.3.0 release.
+If the issue is encountered, then it is recommended that you upgrade to
+a release containing version 2 of the extension, or recompile with a
+patched version of an older release.
.IP
.nf
\f[C]
@@ -123,11 +140,12 @@ and it may not be available in future releases.
struct\ fi_usnic_info\ {
\ \ \ \ uint32_t\ ui_version;
+\ \ \ \ uint8_t\ ui_pad0[4];
\ \ \ \ union\ {
\ \ \ \ \ \ \ \ struct\ fi_usnic_info_v1\ v1;
\ \ \ \ \ \ \ \ struct\ fi_usnic_info_v2\ v2;
\ \ \ \ }\ ui;
-};
+}\ __attribute__((packed));
int\ getinfo(uint32_t\ version,\ struct\ fid_fabric\ *fabric,
\ \ \ \ \ \ \ \ struct\ fi_usnic_info\ *info);
@@ -148,7 +166,7 @@ Version 2
struct\ fi_usnic_cap\ {
\ \ \ \ const\ char\ *uc_capability;
\ \ \ \ int\ uc_present;
-};
+}\ __attribute__((packed));
struct\ fi_usnic_info_v2\ {
\ \ \ \ uint32_t\ \ \ \ \ \ \ \ ui_link_speed;
@@ -161,11 +179,15 @@ struct\ fi_usnic_info_v2\ {
\ \ \ \ char\ \ \ \ \ \ \ \ \ \ \ \ ui_devname[FI_EXT_USNIC_MAX_DEVNAME];
\ \ \ \ uint8_t\ \ \ \ \ \ \ \ \ ui_mac_addr[6];
+\ \ \ \ uint8_t\ \ \ \ \ \ \ \ \ ui_pad0[2];
+
\ \ \ \ uint32_t\ \ \ \ \ \ \ \ ui_ipaddr_be;
\ \ \ \ uint32_t\ \ \ \ \ \ \ \ ui_prefixlen;
\ \ \ \ uint32_t\ \ \ \ \ \ \ \ ui_mtu;
\ \ \ \ uint8_t\ \ \ \ \ \ \ \ \ ui_link_up;
+\ \ \ \ uint8_t\ \ \ \ \ \ \ \ \ ui_pad1[3];
+
\ \ \ \ uint32_t\ \ \ \ \ \ \ \ ui_vendor_id;
\ \ \ \ uint32_t\ \ \ \ \ \ \ \ ui_vendor_part_id;
\ \ \ \ uint32_t\ \ \ \ \ \ \ \ ui_device_id;
@@ -183,7 +205,7 @@ struct\ fi_usnic_info_v2\ {
\ \ \ \ const\ char\ \ \ \ \ \ *ui_pid;
\ \ \ \ struct\ fi_usnic_cap\ **ui_caps;
-};
+}\ __attribute__((packed));
\f[]
.fi
.IP \[bu] 2
@@ -199,7 +221,7 @@ struct\ fi_usnic_info_v1\ {
\ \ \ \ uint32_t\ ui_num_vf;
\ \ \ \ uint32_t\ ui_qp_per_vf;
\ \ \ \ uint32_t\ ui_cq_per_vf;
-};
+}\ __attribute__((packed));
\f[]
.fi
.PP
@@ -305,6 +327,33 @@ destination host is unreachable, \f[C]0\f[] is the destination host is
locally connected, and \f[C]1\f[] otherwise.
.PP
See fi_ext_usnic.h for more details.
+.SH VERSION DIFFERENCES
+.SS New naming convention for fabric/domain starting with libfabric v1.4
+.PP
+The release of libfabric v1.4 introduced a new naming convention for
+fabric and domain.
+However the usNIC provider remains backward compatible with applications
+supporting the old scheme and decides which one to use based on the
+version passed to \f[C]fi_getinfo\f[]:
+.IP \[bu] 2
+When \f[C]FI_VERSION(1,4)\f[] or higher is used:
+.RS 2
+.IP \[bu] 2
+fabric name is the network address with the CIDR notation (i.e.,
+\f[C]a.b.c.d/e\f[])
+.IP \[bu] 2
+domain name is the usNIC Linux interface name (i.e., \f[C]usnic_X\f[])
+.RE
+.IP \[bu] 2
+When a lower version number is used, like \f[C]FI_VERSION(1,\ 3)\f[], it
+follows the same behavior the usNIC provider exhibited in libfabric <=
+v1.3:
+.RS 2
+.IP \[bu] 2
+fabric name is the usNIC Linux interface name (i.e., \f[C]usnic_X\f[])
+.IP \[bu] 2
+domain name is \f[C]NULL\f[]
+.RE
.SH SEE ALSO
.PP
\f[C]fabric\f[](7), \f[C]fi_open_ops\f[](3), \f[C]fi_provider\f[](7),
diff --git a/man/man7/fi_verbs.7 b/man/man7/fi_verbs.7
index 5ea67b8..64302e7 100644
--- a/man/man7/fi_verbs.7
+++ b/man/man7/fi_verbs.7
@@ -1,4 +1,4 @@
-.TH "fi_verbs" "7" "2016\-04\-06" "Libfabric Programmer\[aq]s Manual" "\@VERSION\@"
+.TH "fi_verbs" "7" "2016\-10\-06" "Libfabric Programmer\[aq]s Manual" "Libfabric v1.4.0rc2"
.SH NAME
.PP
The Verbs Fabric Provider
@@ -13,53 +13,149 @@ control and data transfer operations.
.SH SUPPORTED FEATURES
.PP
The verbs provider supports a subset of OFI features.
-.PP
-\f[I]Endpoint types\f[] : FI_EP_MSG, FI_EP_RDM (Experimental support
-FI_TAGGED and FI_RMA interfaces).
-.PP
-\f[I]Endpoint capabilities\f[] : FI_MSG, FI_RMA, FI_ATOMIC.
-.PP
-\f[I]Modes\f[] : Verbs provider requires applications to support the
-following modes: * FI_LOCAL_MR for all applications.
-* FI_RX_CQ_DATA for applications that want to use RMA.
+.SS Endpoint types
+.PP
+: FI_EP_MSG, FI_EP_RDM (Experimental support FI_MSG, FI_TAGGED, FI_RMA
+interfaces).
+.SS Endpoint capabilities and features
+.SS MSG endpoints
+.PP
+: FI_MSG, FI_RMA, FI_ATOMIC and shared receive contexts.
+##### RDM endpoints : FI_MSG, FI_TAGGED, FI_RMA
+.SS Modes
+.PP
+: Verbs provider requires applications to support the following modes:
+.IP \[bu] 2
+FI_LOCAL_MR for all applications.
+.IP \[bu] 2
+FI_RX_CQ_DATA for applications that want to use RMA.
Applications must take responsibility of posting receives for any
incoming CQ data.
-* FI_CONTEXT for applications making uses of the experimental FI_EP_RDM
+.IP \[bu] 2
+FI_CONTEXT for applications making uses of the experimental FI_EP_RDM
capability.
-.PP
-\f[I]Addressing Formats\f[] : Supported addresing formats include
-FI_SOCKADDR, FI_SOCKADDR_IN, FI_SOCKADDR_IN6, FI_SOCKADDR_IB
-.PP
-\f[I]Progress\f[] : Verbs provider supports FI_PROGRESS_AUTO:
-Asynchronous operations make forward progress automatically.
-.PP
-\f[I]Operation flags\f[] : Verbs provider supports FI_INJECT,
-FI_COMPLETION, FI_REMOTE_CQ_DATA.
-.PP
-\f[I]Msg Ordering\f[] : Verbs provider support the following messaging
-ordering on the TX side: * Read after Read * Read after Write * Read
-after Send * Write after Write * Write after Send * Send after Write *
+.SS Addressing Formats
+.PP
+: Supported addressing formats include FI_SOCKADDR, FI_SOCKADDR_IN,
+FI_SOCKADDR_IN6, FI_SOCKADDR_IB
+.SS Progress
+.PP
+: Verbs provider supports FI_PROGRESS_AUTO: Asynchronous operations make
+forward progress automatically.
+.SS Operation flags
+.PP
+: Verbs provider supports FI_INJECT, FI_COMPLETION, FI_REMOTE_CQ_DATA.
+.SS Msg Ordering
+.PP
+: Verbs provider support the following messaging ordering on the TX
+side:
+.IP \[bu] 2
+Read after Read
+.IP \[bu] 2
+Read after Write
+.IP \[bu] 2
+Read after Send
+.IP \[bu] 2
+Write after Write
+.IP \[bu] 2
+Write after Send
+.IP \[bu] 2
+Send after Write
+.IP \[bu] 2
Send after Send
+.SS Fork
+.PP
+: Verbs provider supports the fork system call by default.
+See the limitations section for restrictions.
+It can be turned off by setting the FI_FORK_UNSAFE environment variable
+to "yes".
+This can improve the performance of memory registrations but it also
+makes the use of fork unsafe.
.SH LIMITATIONS
+.SS Memory Regions
.PP
-\f[I]CQ\f[] : cq_readfrom operations are not supported.
-.PP
-\f[I]Memory Regions\f[] : Only FI_MR_BASIC mode is supported.
+: Only FI_MR_BASIC mode is supported.
Adding regions via s/g list is not supported.
Generic fi_mr_regattr is not supported.
No support for binding memory regions to a counter.
+.SS Wait objects
.PP
-\f[I]Wait objects\f[] : Only FI_WAIT_FD wait object is supported.
+: Only FI_WAIT_FD wait object is supported.
Wait sets are not supported.
-.PP
-\f[I]Resource Management\f[] : Application has to make sure CQs are not
-overrun as this cannot be detected by the provider.
-.PP
-\f[I]RDM\f[] : The RDM support for verbs have the following limitations:
-* iWARP is not supported yet.
-* Supports iovs of only size 1.
-* Max data transfer size is 1 GB * Not thread safe.
-* Data and flags arguments are ignored
+.SS Resource Management
+.PP
+: Application has to make sure CQs are not overrun as this cannot be
+detected by the provider.
+.SS Unsupported Features
+.PP
+: The following features are not supported in verbs provider.
+.SS Unsupported Endpoint types
+.PP
+: FI_EP_DGRAM
+.SS Unsupported Capabilities
+.PP
+: FI_NAMED_RX_CTX, FI_DIRECTED_RECV, FI_TRIGGER, FI_MULTI_RECV,
+FI_RMA_EVENT, FI_FENCE
+.SS Other unsupported features
+.PP
+: Scalable endpoints, FABRIC_DIRECT
+.SS Unsupported features specific to MSG endpoints
+.PP
+: Counters, FI_SOURCE, FI_TAGGED, FI_PEEK, FI_CLAIM, fi_cancel,
+fi_ep_alias, Shared TX context, cq_readfrom operations.
+.SS Unsupported features specific to RDM endpoints
+.PP
+: The RDM support for verbs have the following limitations:
+.IP \[bu] 2
+Supports iovs of only size 1.
+.IP \[bu] 2
+Wait objects are not supported.
+.IP \[bu] 2
+Not thread safe.
+.SS Fork
+.PP
+: The support for fork in the provider has the following limitations: *
+Fabric resources like endpoint, CQ, EQ, etc.
+should not be used in the forked process.
+* The memory registered using fi_mr_reg has to be page aligned since
+ibv_reg_mr marks the entire page that a memory region belongs to as not
+to be re\-mapped when the process is forked (MADV_DONTFORK).
+.SH RUNTIME PARAMETERS
+.PP
+The verbs provider checks for the following environment variables.
+.SS Variables specific to RDM endpoints
+.SS FI_VERBS_IFACE
+.PP
+: The prefix or the full name of the network interface associated with
+the IB device (default: ib)
+.SS FI_VERBS_RDM_BUFFER_NUM
+.PP
+: The number of pre\-registered buffers for buffered operations between
+the endpoints, must be a power of 2 (default: 8).
+.SS FI_VERBS_RDM_BUFFER_SIZE
+.PP
+: The maximum size of a buffered operation (bytes) (default: platform
+specific).
+.SS FI_VERBS_RDM_RNDV_SEG_SIZE
+.PP
+: The segment size for zero copy protocols (bytes)(default: 1073741824).
+.SS FI_VERBS_RDM_CQREAD_BUNCH_SIZE
+.PP
+: The number of entries to be read from the verbs completion queue at a
+time (default: 8).
+.SS FI_VERBS_RDM_THREAD_TIMEOUT
+.PP
+: The wake up timeout of the helper thread (usec) (default: 100).
+.SS FI_VERBS_RDM_EAGER_SEND_OPCODE
+.PP
+: The operation code that will be used for eager messaging.
+Only IBV_WR_SEND and IBV_WR_RDMA_WRITE_WITH_IMM are supported.
+The last one is not applicable for iWarp.
+(default: IBV_WR_SEND)
+.SS Environment variables notes
+.PP
+: The fi_info utility would give the up\-to\-date information on
+environment variables: fi_info \-p verbs \-e
.SH SEE ALSO
.PP
\f[C]fabric\f[](7), \f[C]fi_provider\f[](7),
diff --git a/prov/gni/Makefile.include b/prov/gni/Makefile.include
index 837b98b..d6afb28 100644
--- a/prov/gni/Makefile.include
+++ b/prov/gni/Makefile.include
@@ -7,7 +7,7 @@ if HAVE_GNI
# source code, so just add prov/gni to the include path
# rather than prov/gni/ccan
#
-AM_CFLAGS += -I$(top_srcdir)/prov/gni/include -I$(top_srcdir)/prov/gni
+AM_CPPFLAGS += -I$(top_srcdir)/prov/gni/include -I$(top_srcdir)/prov/gni
_gni_files = \
prov/gni/src/gnix_atomic.c \
@@ -29,6 +29,7 @@ _gni_files = \
prov/gni/src/gnix_mbox_allocator.c \
prov/gni/src/gnix_mr.c \
prov/gni/src/gnix_mr_cache.c \
+ prov/gni/src/gnix_mr_notifier.c \
prov/gni/src/gnix_msg.c \
prov/gni/src/gnix_nameserver.c \
prov/gni/src/gnix_nic.c \
@@ -36,9 +37,11 @@ _gni_files = \
prov/gni/src/gnix_queue.c \
prov/gni/src/gnix_rma.c \
prov/gni/src/gnix_tags.c \
+ prov/gni/src/gnix_trigger.c \
prov/gni/src/gnix_util.c \
prov/gni/src/gnix_vc.c \
prov/gni/src/gnix_vector.c \
+ prov/gni/src/gnix_xpmem.c \
prov/gni/src/gnix_wait.c
_gni_headers = \
@@ -59,6 +62,7 @@ _gni_headers = \
prov/gni/include/gnix_mbox_allocator.h \
prov/gni/include/gnix_mr.h \
prov/gni/include/gnix_mr_cache.h \
+ prov/gni/include/gnix_mr_notifier.h \
prov/gni/include/gnix_msg.h \
prov/gni/include/gnix_nameserver.h \
prov/gni/include/gnix_nic.h \
@@ -67,9 +71,11 @@ _gni_headers = \
prov/gni/include/gnix_queue.h \
prov/gni/include/gnix_rma.h \
prov/gni/include/gnix_tags.h \
+ prov/gni/include/gnix_trigger.h \
prov/gni/include/gnix_util.h \
prov/gni/include/gnix_vc.h \
prov/gni/include/gnix_vector.h \
+ prov/gni/include/gnix_xpmem.h \
prov/gni/include/gnix_wait.h
if HAVE_CRITERION
@@ -91,6 +97,7 @@ nodist_prov_gni_test_gnitest_SOURCES = \
prov/gni/test/freelist.c \
prov/gni/test/hashtable.c \
prov/gni/test/mr.c \
+ prov/gni/test/mr_notifier.c \
prov/gni/test/nic.c \
prov/gni/test/pmi_utils.c \
prov/gni/test/queue.c \
@@ -103,16 +110,17 @@ nodist_prov_gni_test_gnitest_SOURCES = \
prov/gni/test/tags.c \
prov/gni/test/api.c \
prov/gni/test/api_cq.c \
+ prov/gni/test/api_cntr.c \
prov/gni/test/utils.c \
prov/gni/test/vc.c \
+ prov/gni/test/vc_lookup.c \
prov/gni/test/vector.c \
prov/gni/test/wait.c \
prov/gni/test/common.c
-prov_gni_test_gnitest_LDFLAGS = $(CRAY_PMI_LIBS) $(gni_LDFLAGS) -static
-prov_gni_test_gnitest_CPPFLAGS = $(AM_CPPFLAGS) $(CRAY_PMI_CFLAGS) $(gni_CPPFLAGS)
-prov_gni_test_gnitest_LDADD = -lcriterion \
- $(linkback) $(gni_LIBS)
+prov_gni_test_gnitest_LDFLAGS = $(CRAY_PMI_LIBS) $(gnitest_LDFLAGS) -static
+prov_gni_test_gnitest_CPPFLAGS = $(AM_CPPFLAGS) $(CRAY_PMI_CFLAGS) $(CRAY_XPMEM_CFLAGS) $(gnitest_CPPFLAGS)
+prov_gni_test_gnitest_LDADD = $(gnitest_LIBS) $(linkback)
endif HAVE_CRITERION
if HAVE_GNI_DL
@@ -122,7 +130,7 @@ libgnix_fi_la_SOURCES = $(_gni_files) $(_gni_headers) $(common_srcs)
libgnix_fi_la_LDFLAGS = \
$(gni_LDFLAGS) \
-module -avoid-version -shared -export-dynamic
-libgnix_fi_la_LIBADD = $(linkback) $(gni_LIBS)
+libgnix_fi_la_LIBADD = $(linkback)
libgnix_fi_la_DEPENDENCIES = $(linkback)
else !HAVE_GNI_DL
src_libfabric_la_SOURCES += $(_gni_files) $(_gni_headers)
diff --git a/prov/gni/README.md b/prov/gni/README.md
deleted file mode 100644
index f774272..0000000
--- a/prov/gni/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# OFI libfabric GNI provider
-
-The GNI provider is a research prototype provider layer for OFI
-libfabric running on Cray XC (TM) systems. It is being developed as a
-collaboration between Los Alamos National Laboratory and Cray Inc.
-The goals of the collaboration are to enable libfabric clients such as
-OpenMPI to experiment at scale on today's hardware. As such, we have
-initially tried to address requirements of MPI, SHMEM and PGAS
-language and library implementation. As with any provider, there
-features that are difficult to support efficiently (e.g.,
-FI_MR_SCALABLE) and have been omitted for the time being.
-
-Bugs should be filed as GitHub issues on the ofi-cray/libfabric-cray
-repository (https://github.com/ofi-cray/libfabric-cray).
-
-## For Developers
-
-See the wiki pages at the ofi-cray/libfabric-cray repository
-(https://github.com/ofi-cray/libfabric-cray) for tips and other useful
-information.
-
diff --git a/prov/gni/configure.m4 b/prov/gni/configure.m4
index f725d26..7e7d6d6 100644
--- a/prov/gni/configure.m4
+++ b/prov/gni/configure.m4
@@ -8,6 +8,7 @@ AC_DEFUN([FI_GNI_CONFIGURE],[
# Determine if we can support the gni provider
# have to pull in pkg.m4 manually
ugni_lib_happy=0
+ udreg_lib_happy=0
gni_header_happy=0
alps_lli_happy=0
alps_util_happy=0
@@ -15,7 +16,15 @@ AC_DEFUN([FI_GNI_CONFIGURE],[
criterion_tests_present=true
gni_CPPFLAGS=
gni_LDFLAGS=
- gni_LIBS=
+ gnitest_CPPFLAGS=
+ gnitest_LDFLAGS=
+ gnitest_LIBS=
+
+
+ AC_ARG_ENABLE([xpmem],
+ [AS_HELP_STRING([--enable-xpmem],
+ [Enable xpmem (gni provider) @<:@default=yes@:>@])],
+ )
AS_IF([test x"$enable_gni" != x"no"],
[FI_PKG_CHECK_MODULES([CRAY_GNI_HEADERS], [cray-gni-headers],
@@ -45,6 +54,26 @@ AC_DEFUN([FI_GNI_CONFIGURE],[
gni_LDFLAGS="$CRAY_ALPS_UTIL_LIBS $gni_LDFLAGS"
],
[alps_util_happy=0])
+ FI_PKG_CHECK_MODULES([CRAY_UDREG], [cray-udreg],
+ [udreg_lib_happy=1
+ gni_CPPFLAGS="-DHAVE_UDREG $CRAY_UDREG_CFLAGS $gni_CPPFLAGS"
+ gni_LDFLAGS="$CRAY_UDREG_LIBS $gni_LDFLAGS"
+ ],
+ [udreg_lib_happy=0])
+ AS_IF([test x"$enable_xpmem" != x"no"],
+ [FI_PKG_CHECK_MODULES([CRAY_XPMEM], [cray-xpmem],
+ [AC_DEFINE_UNQUOTED([HAVE_XPMEM], [1], [Define to 1 if xpmem available])
+ gni_CPPFLAGS="$CRAY_XPMEM_CFLAGS $gni_CPPFLAGS"
+ gni_LDFLAGS="$CRAY_XPMEM_LIBS $gni_LDFLAGS"
+ ],
+ [])
+ ],
+ [AC_DEFINE_UNQUOTED([HAVE_XPMEM], [0], [Define to 1 if xpmem available])
+ ])
+
+ gni_path_to_gni_pub=${CRAY_GNI_HEADERS_CFLAGS:2}
+dnl looks like we need to get rid of some white space
+ gni_path_to_gni_pub=${gni_path_to_gni_pub%?}/gni_pub.h
gni_path_to_gni_pub=${CRAY_GNI_HEADERS_CFLAGS:2}
dnl looks like we need to get rid of some white space
gni_path_to_gni_pub=${gni_path_to_gni_pub%?}/gni_pub.h
@@ -61,20 +90,28 @@ dnl looks like we need to get rid of some white space
[criterion_tests_present=false])
if test "$with_criterion" != "" && test "$with_criterion" != "no"; then
+ if test "$enable_direct" != "" && test "$enable_direct" != "no"; then
+ gnitest_CPPFLAGS="-I$srcdir/prov/gni/include"
+ fi
+
AS_IF([test "$criterion_tests_present" = "true"],
[AC_MSG_CHECKING([criterion path])
if test -d "$with_criterion"; then
AC_MSG_RESULT([yes])
- gni_CPPFLAGS="-I$with_criterion/include $gni_CPPFLAGS"
+ gnitest_CPPFLAGS="-I$with_criterion/include -DHAVE_UDREG $CRAY_UDREG_CFLAGS $gnitest_CPPFLAGS"
+ gnitest_LIBS="-lcriterion -ludreg $gnitest_LIBS"
+
if test -d "$with_criterion/lib"; then
- gni_LDFLAGS="$CRAY_ALPS_LLI_STATIC_LIBS -L$with_criterion/lib -Wl,-rpath=$with_criterion/lib $gni_LDFLAGS"
+ gnitest_LDFLAGS="$CRAY_ALPS_LLI_STATIC_LIBS -L$with_criterion/lib -Wl,-rpath=$with_criterion/lib $gnitest_LDFLAGS"
have_criterion=true
elif test -d "$with_criterion/lib64"; then
- gni_LDFLAGS="$CRAY_ALPS_LLI_STATIC_LIBS -L$with_criterion/lib64 -Wl,-rpath=$with_criterion/lib64 $gni_LDFLAGS"
+ gnitest_LDFLAGS="$CRAY_ALPS_LLI_STATIC_LIBS -L$with_criterion/lib64 -Wl,-rpath=$with_criterion/lib64 $gnitest_LDFLAGS"
have_criterion=true
else
have_criterion=false
fi
+
+ gnitest_LDFLAGS="$CRAY_UDREG_LIBS $gnitest_LDFLAGS"
FI_PKG_CHECK_MODULES([CRAY_PMI], [cray-pmi],
[],
[have_criterion=false])
@@ -93,12 +130,23 @@ dnl looks like we need to get rid of some white space
])
+ AC_ARG_WITH([kdreg], [AS_HELP_STRING([--with-kdreg],
+ [Install directory for kdreg headers])])
+
+ if test "$with_kdreg" != "" && test "$with_kdreg" != "no"; then
+ gni_CPPFLAGS="-I$with_kdreg/include -DHAVE_KDREG $gni_CPPFLAGS"
+ fi
+
+
AM_CONDITIONAL([HAVE_CRITERION], [test "x$have_criterion" = "xtrue"])
AC_SUBST(gni_CPPFLAGS)
AC_SUBST(gni_LDFLAGS)
- AC_SUBST(gni_LIBS)
+ AC_SUBST(gnitest_CPPFLAGS)
+ AC_SUBST(gnitest_LDFLAGS)
+ AC_SUBST(gnitest_LIBS)
AS_IF([test $gni_header_happy -eq 1 -a $ugni_lib_happy -eq 1 \
- -a $alps_lli_happy -eq 1 -a $alps_util_happy -eq 1], [$1], [$2])
+ -a $alps_lli_happy -eq 1 -a $alps_util_happy -eq 1 \
+ -a $udreg_lib_happy -eq 1], [$1], [$2])
])
diff --git a/prov/gni/contrib/gnitest.supp b/prov/gni/contrib/gnitest.supp
deleted file mode 100644
index 92697e3..0000000
--- a/prov/gni/contrib/gnitest.supp
+++ /dev/null
@@ -1,463 +0,0 @@
-#
-# These are from Criterion and should be fixed in a subsequent version
-#
-
-{
- calloc_criterion_run_all_tests
- Memcheck:Leak
- match-leak-kinds: definite
- fun:calloc
- fun:_dl_allocate_tls
- fun:pthread_create@@GLIBC_2.2.5
- fun:init_proc_compat
- fun:criterion_run_all_tests_impl
- fun:criterion_run_all_tests
- fun:main
-}
-
-{
- malloc_criterion_runn_all_tests
- Memcheck:Leak
- match-leak-kinds: definite
- fun:malloc
- fun:alloc_entry
- fun:smalloc_impl
- fun:smalloc
- fun:test_stats_init
- fun:run_next_test
- fun:run_tests_async
- fun:criterion_run_all_tests_impl
- fun:criterion_run_all_tests
- fun:main
-}
-
-#
-# This is an actual memory leak in uGNI. A bug has been submitted
-#
-{
- cq_vector_wait_event_leak
- Memcheck:Leak
- match-leak-kinds: definite
- fun:malloc
- fun:cq_vector_wait_event
- ...
-}
-
-#
-# These are benign
-#
-{
- dgram_handle_alloc
- Memcheck:Leak
- match-leak-kinds: definite
- fun:calloc
- fun:_dl_allocate_tls
- fun:pthread_create@@GLIBC_2.2.5
- fun:_gnix_dgram_hndl_alloc
- fun:_gnix_cm_nic_alloc
- fun:gnix_ep_open
- fun:fi_endpoint
-}
-
-#
-# These are from specifying an additional .init function for a test
-# (there's no way to specify a replacement .init function with
-# Criterion).
-#
-{
- dg_allocation::dgram_wc_post_exchg_manual-1
- Memcheck:Leak
- match-leak-kinds: definite
- fun:calloc
- fun:fi_allocinfo_internal
- fun:fi_dupinfo@@FABRIC_1.0
- fun:fi_allocinfo
- fun:dg_setup
- ...
-}
-
-{
- dg_allocation::dgram_wc_post_exchg_manual-2
- Memcheck:Leak
- match-leak-kinds: definite
- fun:calloc
- fun:fi_allocinfo_internal
- fun:fi_dupinfo@@FABRIC_1.0
- fun:fi_allocinfo
- fun:gnix_getinfo
- fun:fi_getinfo@@FABRIC_1.0
- fun:dg_setup
- ...
-}
-
-{
- cq_msg::multi_sread_setup-1
- Memcheck:Leak
- match-leak-kinds: definite
- fun:calloc
- fun:gnix_fabric_open
- fun:setup
- fun:cq_wait_none_setup
- ...
-}
-
-{
- cq_msg::multi_sread_setup-2
- Memcheck:Leak
- match-leak-kinds: definite
- fun:calloc
- fun:fi_allocinfo_internal
- fun:fi_dupinfo@@FABRIC_1.0
- fun:fi_allocinfo
- fun:setup
- fun:cq_wait_none_setup
- ...
-}
-
-{
- cq_msg::multi_sread_setup-3
- Memcheck:Leak
- match-leak-kinds: definite
- fun:calloc
- fun:fi_allocinfo_internal
- fun:fi_dupinfo@@FABRIC_1.0
- fun:fi_allocinfo
- fun:gnix_getinfo
- fun:fi_getinfo@@FABRIC_1.0
- fun:setup
- fun:cq_wait_none_setup
- ...
-}
-
-{
- cq_msg::multi_sread_setup-4
- Memcheck:Leak
- match-leak-kinds: definite
- fun:calloc
- fun:gnix_cq_open
- fun:fi_cq_open
- fun:cq_create
- fun:criterion_internal_test_setup
- ...
-}
-
-#
-# These are due to writing fewer than 4 bytes of data, but the
-# compiler reading a whole word in the generated code.
-#
-{
- rdm_rma_check_data
- Memcheck:Cond
- fun:check_data
- fun:do_read_buf
- fun:do_read_alignment
- fun:xfer_for_each_size
- ...
-}
-
-{
- rdm_src_check_data_multirecv
- Memcheck:Cond
- fun:rdm_sr_check_data
- fun:do_multirecv
- fun:rdm_sr_xfer_for_each_size
- ...
-}
-
-{
- rdm_src_check_data_multirecv2
- Memcheck:Cond
- fun:rdm_sr_check_data
- fun:do_multirecv2
- fun:rdm_sr_xfer_for_each_size
- ...
-}
-
-#
-# These are from uGNI itself
-#
-{
- ioctl_cq_create
- Memcheck:Param
- ioctl(generic)
- fun:ioctl
- fun:cq_create
- ...
-}
-{
- ioctl_GNI_MemRegister
- Memcheck:Param
- ioctl(generic)
- fun:ioctl
- fun:GNI_MemRegister
- ...
-}
-{
- ioctl_GNI_EpPostDataWId
- Memcheck:Param
- ioctl(generic)
- fun:ioctl
- fun:GNI_EpPostDataWId
- ...
-}
-{
- ioctl_gni_fma_assign
- Memcheck:Param
- ioctl(generic)
- fun:ioctl
- fun:gni_fma_assign
- ...
-}
-{
- GNI_EpPostDataTestById
- Memcheck:Addr4
- fun:GNI_EpPostDataTestById
- ...
-}
-{
- GNI_PostDataProbeById
- Memcheck:Addr4
- fun:GNI_PostDataProbeById
- ...
-}
-
-{
- GNI_CqGetEvent
- Memcheck:Addr8
- fun:GNI_CqGetEvent
- ...
-}
-
-{
- GNII_DlaProgress
- Memcheck:Addr8
- fun:GNII_DlaProgress
- ...
-}
-
-{
- GNII_DLA_PROGRESS_NOLOCK
- Memcheck:Addr8
- fun:GNII_DLA_PROGRESS_NOLOCK
- ...
-}
-
-{
- gni_fma_assign
- Memcheck:Addr4
- fun:gni_fma_assign
- ...
-}
-
-{
- GNII_POST_FMA_GET
- Memcheck:Addr8
- fun:GNII_POST_FMA_GET
- ...
-}
-
-{
- GNII_FmaGetWithMode
- Memcheck:Addr4
- fun:GNII_FmaGetWithMode
- ...
-}
-
-{
- GNII_FmaGetWithMode
- Memcheck:Addr8
- fun:GNII_FmaGetWithMode
- ...
-}
-
-{
- GNII_GenAllocSeqid
- Memcheck:Addr8
- fun:GNII_GenAllocSeqid
- ...
-}
-
-{
- GNII_PostRdma
- Memcheck:Addr4
- fun:GNII_PostRdma
- ...
-}
-
-{
- GNII_PostRdma
- Memcheck:Addr8
- fun:GNII_PostRdma
- ...
-}
-
-{
- GNII_PostFlbte
- Memcheck:Addr4
- fun:GNII_PostFlbte
- ...
-}
-
-{
- GNII_PostFlbte
- Memcheck:Addr8
- fun:GNII_PostFlbte
- ...
-}
-
-{
- GNII_FmaPut
- Memcheck:Addr8
- fun:GNII_FmaPut
- ...
-}
-
-{
- GNII_FmaPut
- Memcheck:Addr4
- fun:GNII_FmaPut
- ...
-}
-
-{
- GNII_SmsgSend
- Memcheck:Addr8
- fun:GNII_SmsgSend
- ...
-}
-
-{
- GNII_SmsgSend
- Memcheck:Addr4
- fun:GNII_SmsgSend
- ...
-}
-
-{
- GNII_SmsgSend
- Memcheck:Addr2
- fun:GNII_SmsgSend
- ...
-}
-
-{
- GNII_SmsgSend
- Memcheck:Addr1
- fun:GNII_SmsgSend
- ...
-}
-
-{
- GNII_POST_FMA_PUT
- Memcheck:Addr8
- fun:GNII_POST_FMA_PUT
- ...
-}
-
-{
- GNII_POST_FMA_PUT
- Memcheck:Addr4
- fun:GNII_POST_FMA_PUT
- ...
-}
-
-{
- GNII_POST_FMA_PUT
- Memcheck:Addr2
- fun:GNII_POST_FMA_PUT
- ...
-}
-
-{
- GNII_POST_FMA_PUT
- Memcheck:Addr1
- fun:GNII_POST_FMA_PUT
- ...
-}
-
-{
- GNI_PostFma
- Memcheck:Addr8
- fun:GNI_PostFma
- ...
-}
-
-{
- GNII_POST_AMO
- Memcheck:Addr4
- fun:GNII_POST_AMO
- ...
-}
-
-{
- GNII_POST_AMO
- Memcheck:Addr8
- fun:GNII_POST_AMO
- ...
-}
-
-{
- GNI_PostFma
- Memcheck:Addr8
- fun:GNI_PostFma
- ...
-}
-
-{
- return_back_credits
- Memcheck:Addr8
- fun:return_back_credits
- ...
-}
-
-{
- GNI_CqTestEvent
- Memcheck:Addr8
- fun:GNI_CqTestEvent
- ...
-}
-
-{
- cq_vector_wait_event
- Memcheck:Addr4
- fun:cq_vector_wait_event
- ...
-}
-
-{
- cq_vector_wait_event
- Memcheck:Addr8
- fun:cq_vector_wait_event
- ...
-}
-
-{
- ioctl_cq_vector_wait_event
- Memcheck:Param
- ioctl(generic)
- fun:ioctl
- ...
-}
-
-{
- GNII_CqPeek
- Memcheck:Addr8
- fun:GNII_CqPeek
- ...
-}
-
-{
- GNII_PostCqWrite
- Memcheck:Addr8
- fun:GNII_PostCqWrite
- fun:GNI_PostCqWrite
- ...
-}
-
-{
- GNII_PostCqWrite
- Memcheck:Addr4
- fun:GNII_PostCqWrite
- fun:GNI_PostCqWrite
- ...
-}
diff --git a/prov/gni/gnix.map b/prov/gni/gnix.map
deleted file mode 100644
index 6731405..0000000
--- a/prov/gni/gnix.map
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * used for exporting GNI provider
- * symbols when building to support FI_DIRECT
- */
- gnix_av_straddr;
- gnix_cq_strerror;
- gnix_eq_strerror;
- gnix_accept;
- gnix_av_bind;
- gnix_av_insertsvc;
- gnix_av_insertsym;
- gnix_av_open;
- gnix_cntr_open;
- gnix_cntr_wait;
- gnix_connect;
- gnix_cq_open;
- gnix_domain_bind;
- gnix_domain_open;
- gnix_ep_atomic_compwrite;
- gnix_ep_atomic_compwritemsg;
- gnix_ep_atomic_compwritev;
- gnix_ep_atomic_inject;
- gnix_ep_atomic_readwrite;
- gnix_ep_atomic_readwritemsg;
- gnix_ep_atomic_readwritev;
- gnix_ep_atomic_write;
- gnix_ep_atomic_writemsg;
- gnix_ep_atomic_writev;
- gnix_ep_msg_injectdata;
- gnix_ep_open;
- gnix_ep_send;
- gnix_ep_senddata;
- gnix_ep_tinjectdata;
- gnix_ep_tsenddata;
- gnix_eq_open;
- gnix_eq_read;
- gnix_eq_sread;
- gnix_getpeer;
- gnix_listen;
- gnix_mr_bind;
- gnix_mr_reg;
- gnix_passive_ep_open;
- gnix_pep_bind;
- gnix_poll_add;
- gnix_poll_del;
- gnix_poll_open;
- gnix_poll_poll;
- gnix_reject;
- gnix_rma_inject_write;
- gnix_rma_inject_writedata;
- gnix_rma_read;
- gnix_rma_readmsg;
- gnix_rma_readv;
- gnix_rma_write;
- gnix_rma_writedata;
- gnix_rma_writemsg;
- gnix_rma_writev;
- gnix_scalable_ep_bind;
- gnix_scalable_ep_open;
- gnix_setname;
- gnix_shutdown;
- gnix_srx_context;
- gnix_stx_open;
- gnix_wait_open;
- gnix_wait_wait;
- gnix_av_insert;
- gnix_av_lookup;
- gnix_av_remove;
- gnix_cntr_add;
- gnix_cntr_set;
- gnix_cq_signal;
- gnix_ep_atomic_valid;
- gnix_ep_bind;
- gnix_ep_cmp_atomic_valid;
- gnix_ep_control;
- gnix_ep_fetch_atomic_valid;
- gnix_ep_getopt;
- gnix_ep_setopt;
- gnix_eq_close;
- gnix_eq_control;
- gnix_getname;
- gnix_rx_context;
- gnix_tx_context;
- gnix_cq_read;
- gnix_cq_readerr;
- gnix_cq_readfrom;
- gnix_cq_sread;
- gnix_cq_sreadfrom;
- gnix_ep_cancel;
- gnix_ep_msg_inject;
- gnix_ep_recv;
- gnix_ep_recvmsg;
- gnix_ep_recvv;
- gnix_ep_rx_size_left;
- gnix_ep_sendmsg;
- gnix_ep_sendv;
- gnix_ep_tinject;
- gnix_ep_trecv;
- gnix_ep_trecvmsg;
- gnix_ep_trecvv;
- gnix_ep_tsend;
- gnix_ep_tsendmsg;
- gnix_ep_tsendv;
- gnix_ep_tx_size_left;
- gnix_eq_readerr;
- gnix_eq_write;
- gnix_cntr_read;
- gnix_cntr_readerr;
diff --git a/prov/gni/include/fi_ext_gni.h b/prov/gni/include/fi_ext_gni.h
index 6fac1b5..c41037b 100644
--- a/prov/gni/include/fi_ext_gni.h
+++ b/prov/gni/include/fi_ext_gni.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -41,6 +41,8 @@ extern "C" {
#include <config.h>
#endif /* HAVE_CONFIG_H */
+#include <stdbool.h>
+
#define FI_GNI_DOMAIN_OPS_1 "domain ops 1"
typedef enum dom_ops_val { GNI_MSG_RENDEZVOUS_THRESHOLD,
GNI_RMA_RDMA_THRESHOLD,
@@ -57,6 +59,12 @@ typedef enum dom_ops_val { GNI_MSG_RENDEZVOUS_THRESHOLD,
GNI_MAX_RETRANSMITS,
GNI_ERR_INJECT_COUNT,
GNI_MR_CACHE_LAZY_DEREG,
+ GNI_MR_CACHE,
+ GNI_MR_UDREG_REG_LIMIT,
+ GNI_MR_SOFT_REG_LIMIT,
+ GNI_MR_HARD_REG_LIMIT,
+ GNI_MR_HARD_STALE_REG_LIMIT,
+ GNI_XPMEM_ENABLE,
GNI_NUM_DOM_OPS
} dom_ops_val_t;
@@ -70,11 +78,26 @@ typedef enum ep_ops_val {
struct fi_gni_ops_domain {
int (*set_val)(struct fid *fid, dom_ops_val_t t, void *val);
int (*get_val)(struct fid *fid, dom_ops_val_t t, void *val);
+ int (*flush_cache)(struct fid *fid);
+};
+
+#include <rdma/fi_atomic.h>
+enum gnix_native_amo_types {
+ GNIX_NAMO_AX = 0x20,
+ GNIX_NAMO_AX_S,
+ GNIX_NAMO_FAX,
+ GNIX_NAMO_FAX_S,
};
struct fi_gni_ops_ep {
int (*set_val)(struct fid *fid, ep_ops_val_t t, void *val);
int (*get_val)(struct fid *fid, ep_ops_val_t t, void *val);
+ size_t (*native_amo)(struct fid_ep *ep, const void *buf, size_t count,
+ void *desc, void *result, void *result_desc,
+ /*void *desc,*/ fi_addr_t dest_addr, uint64_t addr,
+ uint64_t key, enum fi_datatype datatype,
+ int req_type,
+ void *context);
};
/* per domain parameters */
@@ -93,6 +116,11 @@ struct gnix_ops_domain {
uint32_t tx_cq_size;
uint32_t max_retransmits;
int32_t err_inject_count;
+ bool xpmem_enabled;
};
+#ifdef __cplusplus
+}
+#endif
+
#endif /* _FI_EXT_GNI_H_ */
diff --git a/prov/gni/include/gnix.h b/prov/gni/include/gnix.h
index 33da4c3..02ff3d1 100644
--- a/prov/gni/include/gnix.h
+++ b/prov/gni/include/gnix.h
@@ -2,6 +2,7 @@
* Copyright (c) 2015-2016 Cray Inc. All rights reserved.
* Copyright (c) 2015-2016 Los Alamos National Security, LLC.
* All rights reserved.
+ * Copyright (c) 2015-2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -35,10 +36,6 @@
#ifndef _GNIX_H_
#define _GNIX_H_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
#if HAVE_CONFIG_H
#include <config.h>
#endif /* HAVE_CONFIG_H */
@@ -53,7 +50,7 @@ extern "C" {
#include <rdma/fi_endpoint.h>
#include <rdma/fi_eq.h>
#include <rdma/fi_errno.h>
-#include <rdma/fi_prov.h>
+#include <rdma/providers/fi_prov.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_tagged.h>
#include <rdma/fi_trigger.h>
@@ -63,6 +60,12 @@ extern "C" {
#include <fi_indexer.h>
#include <fi_rbuf.h>
#include <fi_list.h>
+#include <fi_file.h>
+
+#ifdef HAVE_UDREG
+#include <udreg_pub.h>
+#endif
+
#include "gni_pub.h"
#include "gnix_util.h"
#include "gnix_freelist.h"
@@ -71,6 +74,8 @@ extern "C" {
#include "fi_ext_gni.h"
#include "gnix_tags.h"
#include "gnix_mr_cache.h"
+#include "gnix_mr_notifier.h"
+#include "gnix_nic.h"
#define GNI_MAJOR_VERSION 1
#define GNI_MINOR_VERSION 0
@@ -109,6 +114,11 @@ extern "C" {
#define compiler_barrier() asm volatile ("" ::: "memory")
#endif
+#define GNIX_MAX_MSG_IOV_LIMIT 8
+#define GNIX_MAX_RMA_IOV_LIMIT 1
+#define GNIX_MAX_ATOMIC_IOV_LIMIT 1
+#define GNIX_ADDR_CACHE_SIZE 5
+
/*
* GNI GET alignment
*/
@@ -117,6 +127,17 @@ extern "C" {
#define GNI_READ_ALIGN_MASK (GNI_READ_ALIGN - 1)
/*
+ * GNI IOV GET alignment
+ *
+ * We always pull 4byte chucks for unaligned GETs. To prevent stomping on
+ * someone else's head or tail data, each segment must be four bytes
+ * (i.e. GNI_READ_ALIGN bytes).
+ *
+ * Note: "* 2" for head and tail
+ */
+#define GNIX_HTD_BUF_SZ (GNIX_MAX_MSG_IOV_LIMIT * GNI_READ_ALIGN * 2)
+
+/*
* Flags
* The 64-bit flag field is used as follows:
* 1-grow up common (usable with multiple operations)
@@ -132,6 +153,7 @@ extern "C" {
#define GNIX_MSG_RENDEZVOUS (1ULL << 61) /* MSG only flag */
#define GNIX_MSG_GET_TAIL (1ULL << 62) /* MSG only flag */
+#define GNIX_MSG_MULTI_RECV_SUP (1ULL << 63) /* MSG only flag */
/*
* Cray gni provider supported flags for fi_getinfo argument for now, needs
@@ -150,15 +172,25 @@ extern "C" {
/*
* See capabilities section in fi_getinfo.3.
*/
+
+/* Primary capabilities. Each must be explicitly requested (unless the full
+ * set is requested by setting input hints->caps to NULL). */
#define GNIX_EP_RDM_PRIMARY_CAPS \
(FI_MSG | FI_RMA | FI_TAGGED | FI_ATOMICS | \
FI_DIRECTED_RECV | FI_READ | \
FI_WRITE | FI_SEND | FI_RECV | FI_REMOTE_READ | FI_REMOTE_WRITE)
-#define GNIX_EP_RDM_SEC_CAPS \
- (FI_MULTI_RECV | FI_SOURCE | FI_TRIGGER | FI_FENCE)
+/* No overhead secondary capabilities. These can be silently enabled by the
+ * provider. */
+#define GNIX_EP_RDM_SEC_CAPS (FI_MULTI_RECV | FI_TRIGGER | FI_FENCE)
+
+/* Secondary capabilities that introduce overhead. Must be requested. */
+#define GNIX_EP_RDM_SEC_CAPS_OH (FI_SOURCE | FI_RMA_EVENT)
-#define GNIX_EP_RDM_CAPS (GNIX_EP_RDM_PRIMARY_CAPS | GNIX_EP_RDM_SEC_CAPS)
+/* FULL set of capabilities for the provider. */
+#define GNIX_EP_RDM_CAPS_FULL (GNIX_EP_RDM_PRIMARY_CAPS | \
+ GNIX_EP_RDM_SEC_CAPS | \
+ GNIX_EP_RDM_SEC_CAPS_OH)
/*
* see Operations flags in fi_endpoint.3
@@ -172,7 +204,7 @@ extern "C" {
*/
#define GNIX_SENDMSG_FLAGS (FI_REMOTE_CQ_DATA | FI_COMPLETION | \
FI_MORE | FI_INJECT | FI_INJECT_COMPLETE | \
- FI_TRANSMIT_COMPLETE | FI_FENCE)
+ FI_TRANSMIT_COMPLETE | FI_FENCE | FI_TRIGGER)
#define GNIX_RECVMSG_FLAGS (FI_COMPLETION | FI_MORE | FI_MULTI_RECV)
#define GNIX_TRECVMSG_FLAGS \
(GNIX_RECVMSG_FLAGS | FI_CLAIM | FI_PEEK | FI_DISCARD)
@@ -182,12 +214,15 @@ extern "C" {
*/
#define GNIX_WRITEMSG_FLAGS (FI_REMOTE_CQ_DATA | FI_COMPLETION | \
FI_MORE | FI_INJECT | FI_INJECT_COMPLETE | \
- FI_TRANSMIT_COMPLETE | FI_FENCE)
+ FI_TRANSMIT_COMPLETE | FI_FENCE | FI_TRIGGER)
#define GNIX_READMSG_FLAGS (FI_REMOTE_CQ_DATA | FI_COMPLETION | FI_MORE | \
- FI_FENCE)
-#define GNIX_ATOMICMSG_FLAGS (FI_COMPLETION | FI_MORE | FI_INJECT | FI_FENCE)
-#define GNIX_FATOMICMSG_FLAGS (FI_COMPLETION | FI_MORE | FI_FENCE)
-#define GNIX_CATOMICMSG_FLAGS (FI_COMPLETION | FI_MORE | FI_FENCE)
+ FI_FENCE | FI_TRIGGER)
+#define GNIX_ATOMICMSG_FLAGS (FI_COMPLETION | FI_MORE | FI_INJECT | \
+ FI_FENCE | FI_TRIGGER)
+#define GNIX_FATOMICMSG_FLAGS (FI_COMPLETION | FI_MORE | FI_FENCE | \
+ FI_TRIGGER)
+#define GNIX_CATOMICMSG_FLAGS (FI_COMPLETION | FI_MORE | FI_FENCE | \
+ FI_TRIGGER)
/*
* Valid completion event flags. See fi_cq.3.
@@ -301,6 +336,7 @@ struct gnix_fid_fabric {
* GNI_PostdataProbeWaitById in gni_pub.h */
uint64_t datagram_timeout;
struct gnix_reference ref_cnt;
+ struct gnix_mr_notifier mr_notifier;
};
extern struct fi_ops_cm gnix_cm_ops;
@@ -334,10 +370,21 @@ struct gnix_fid_domain {
/* additional gni cq modes to use for this domain */
enum fi_progress control_progress;
enum fi_progress data_progress;
+ enum fi_threading thread_model;
struct gnix_reference ref_cnt;
gnix_mr_cache_attr_t mr_cache_attr;
gnix_mr_cache_t *mr_cache;
fastlock_t mr_cache_lock;
+ struct gnix_mr_ops *mr_ops;
+ int mr_cache_type;
+ /* flag to indicate that memory registration is initialized and should not
+ * be changed at this point.
+ */
+ int mr_is_init;
+ int udreg_reg_limit;
+#ifdef HAVE_UDREG
+ udreg_cache_handle_t udreg_cache;
+#endif
};
#define GNIX_CQS_PER_EP 8
@@ -353,6 +400,27 @@ struct gnix_fid_ep_ops_en {
uint32_t atomic_write_allowed: 1;
};
+#define GNIX_HTD_POOL_SIZE 128
+
+struct gnix_htd_buf {
+ struct slist_entry e;
+ uint8_t *buf;
+};
+
+struct gnix_htd_pool {
+ bool enabled;
+ fastlock_t lock;
+ struct gnix_fid_mem_desc *md;
+ struct slist sl;
+ void *buf_ptr;
+ void *sl_ptr;
+};
+
+struct gnix_addr_cache_entry {
+ fi_addr_t addr;
+ struct gnix_vc *vc;
+};
+
/*
* gnix endpoint structure
*
@@ -371,19 +439,22 @@ struct gnix_fid_ep {
struct gnix_fid_cq *recv_cq;
struct gnix_fid_cntr *send_cntr;
struct gnix_fid_cntr *recv_cntr;
- struct gnix_fid_cntr *read_cntr;
struct gnix_fid_cntr *write_cntr;
+ struct gnix_fid_cntr *read_cntr;
+ struct gnix_fid_cntr *rwrite_cntr;
+ struct gnix_fid_cntr *rread_cntr;
struct gnix_fid_av *av;
struct gnix_fid_stx *stx_ctx;
struct gnix_ep_name my_name;
struct gnix_cm_nic *cm_nic;
struct gnix_nic *nic;
- fastlock_t vc_ht_lock;
+ fastlock_t vc_lock;
union {
- struct gnix_hashtable *vc_ht;
- struct gnix_vc **vc_table; /* used for FI_AV_TABLE */
- struct gnix_vc *vc;
+ struct gnix_hashtable *vc_ht; /* FI_AV_MAP */
+ struct gnix_vector *vc_table; /* FI_AV_TABLE */
};
+ struct gnix_vc *vc; /* used for FI_EP_MSG */
+ struct dlist_entry unmapped_vcs;
/* lock for unexp and posted recv queue */
fastlock_t recv_queue_lock;
/* used for unexpected receives */
@@ -395,21 +466,22 @@ struct gnix_fid_ep {
struct gnix_tag_storage tagged_unexp_recv_queue;
struct gnix_tag_storage tagged_posted_recv_queue;
-
- fastlock_t recv_comp_lock;
- struct slist pending_recv_comp_queue;
/* pointer to tag matching engine */
- void *tag_matcher;
int (*progress_fn)(struct gnix_fid_ep *, enum gnix_progress_type);
/* RX specific progress fn */
int (*rx_progress_fn)(struct gnix_fid_ep *, gni_return_t *rc);
+ struct gnix_xpmem_handle *xpmem_hndl;
bool tx_enabled;
bool rx_enabled;
+ bool requires_lock;
+ int last_cached;
+ struct gnix_addr_cache_entry addr_cache[GNIX_ADDR_CACHE_SIZE];
int send_selective_completion;
int recv_selective_completion;
int min_multi_recv;
/* note this free list will be initialized for thread safe */
- struct gnix_s_freelist fr_freelist;
+ struct gnix_freelist fr_freelist;
+ struct gnix_htd_pool htd_pool;
struct gnix_reference ref_cnt;
struct gnix_fid_ep_ops_en ep_ops;
};
@@ -468,14 +540,25 @@ struct gnix_fid_av {
enum gnix_fab_req_type {
GNIX_FAB_RQ_SEND,
+ GNIX_FAB_RQ_SENDV,
GNIX_FAB_RQ_TSEND,
+ GNIX_FAB_RQ_TSENDV,
GNIX_FAB_RQ_RDMA_WRITE,
GNIX_FAB_RQ_RDMA_READ,
GNIX_FAB_RQ_RECV,
+ GNIX_FAB_RQ_RECVV,
GNIX_FAB_RQ_TRECV,
+ GNIX_FAB_RQ_TRECVV,
GNIX_FAB_RQ_AMO,
GNIX_FAB_RQ_FAMO,
- GNIX_FAB_RQ_CAMO
+ GNIX_FAB_RQ_CAMO,
+ GNIX_FAB_RQ_END_NON_NATIVE,
+ GNIX_FAB_RQ_START_NATIVE = GNIX_NAMO_AX,
+ GNIX_FAB_RQ_NAMO_AX = GNIX_NAMO_AX,
+ GNIX_FAB_RQ_NAMO_AX_S = GNIX_NAMO_AX_S,
+ GNIX_FAB_RQ_NAMO_FAX = GNIX_NAMO_FAX,
+ GNIX_FAB_RQ_NAMO_FAX_S = GNIX_NAMO_FAX_S,
+ GNIX_FAB_RQ_MAX_TYPES,
};
struct gnix_fab_req_rma {
@@ -491,21 +574,44 @@ struct gnix_fab_req_rma {
struct gnix_fab_req_msg {
struct gnix_tag_list_element tle;
- uint64_t send_addr;
- size_t send_len;
- struct gnix_fid_mem_desc *send_md;
+
+ struct send_info_t {
+ uint64_t send_addr;
+ size_t send_len;
+ gni_mem_handle_t mem_hndl;
+ uint32_t head;
+ uint32_t tail;
+ } send_info[GNIX_MAX_MSG_IOV_LIMIT];
+ struct gnix_fid_mem_desc *send_md[GNIX_MAX_MSG_IOV_LIMIT];
+ size_t send_iov_cnt;
uint64_t send_flags;
- uint64_t recv_addr;
- size_t recv_len;
- struct gnix_fid_mem_desc *recv_md;
+ size_t cum_send_len;
+
+ struct recv_info_t {
+ uint64_t recv_addr;
+ size_t recv_len;
+ gni_mem_handle_t mem_hndl;
+ uint32_t tail_len : 2; /* If the send len is > the recv_len, we
+ * need to fetch the unaligned tail into
+ * the txd's int buf
+ */
+ uint32_t head_len : 2;
+ } recv_info[GNIX_MAX_MSG_IOV_LIMIT];
+ struct gnix_fid_mem_desc *recv_md[GNIX_MAX_MSG_IOV_LIMIT];
+ size_t recv_iov_cnt;
uint64_t recv_flags; /* protocol, API info */
+ size_t cum_recv_len;
+
+ /* @var htd_buf->buf "head(H) tail(T) data buf" layout: '[T|T|...|H|H]' */
+ struct slist_entry *htd_buf_e;
+ uint8_t *htd_buf;
+ gni_mem_handle_t htd_mdh;
+
uint64_t tag;
uint64_t ignore;
uint64_t imm;
gni_mem_handle_t rma_mdh;
uint64_t rma_id;
- uint32_t rndzv_head;
- uint32_t rndzv_tail;
atomic_t outstanding_txds;
gni_return_t status;
};
@@ -714,14 +820,32 @@ static inline int gnix_ops_allowed(struct gnix_fid_ep *ep,
return 0;
}
-/*
+/**
* Fabric request layout, there is a one to one
* correspondence between an application's invocation of fi_send, fi_recv
* and a gnix fab_req.
+ *
+ * @var dlist a doubly linked list entry used to queue a request in
+ * either the vc's tx_queue or work_queue.
+ * @var addr the peer's gnix_address associated with this request.
+ * @var type the fabric request type
+ * @var gnix_ep the gni endpoint associated with this request
+ * @var user_context the user context, typically the receive buffer address for
+ * a send or the send buffer address for a receive.
+ * @var vc the virtual channel or connection edge between the sender
+ * and receiver.
+ * @var work_fn the function called by the nic progress loop to initiate
+ * the fabric request.
+ * @var flags a set of bit patterns that apply to all message types
+ * @var iov_txds A list of pending Rdma/CtFma GET txds.
+ * @var iov_txd_cnt The count of outstanding iov txds.
+ * @var tx_failures tx failure bits.
+ * @var rma GNI PostRdma request
+ * @var msg GNI SMSG request
+ * @var amo GNI Fma request
*/
-
struct gnix_fab_req {
- struct slist_entry slist;
+ struct dlist_entry dlist;
struct gnix_address addr;
enum gnix_fab_req_type type;
struct gnix_fid_ep *gnix_ep;
@@ -729,8 +853,15 @@ struct gnix_fab_req {
struct gnix_vc *vc;
int (*work_fn)(void *);
uint64_t flags;
- void *txd;
- uint32_t tx_failures;
+
+ /* TODO: change the size of this for unaligned data? */
+ struct gnix_tx_descriptor *iov_txds[GNIX_MAX_MSG_IOV_LIMIT];
+ /*
+ * special value of UINT_MAX is used to indicate
+ * an unrecoverable (aka non-transient) error has occurred
+ * in one of the underlying GNI transactions
+ */
+ uint32_t tx_failures;
/* common to rma/amo/msg */
union {
@@ -741,6 +872,24 @@ struct gnix_fab_req {
char inject_buf[GNIX_INJECT_SIZE];
};
+/*
+ * test whether a request is replayable
+ * or not based on the value of the tx_failures field
+ */
+
+static inline bool _gnix_req_replayable(struct gnix_fab_req *req)
+{
+ bool ret = false;
+ uint32_t tx_failures, max_retrans;
+
+ tx_failures = req->tx_failures;
+ max_retrans = req->gnix_ep->domain->params.max_retransmits;
+ if ((req->tx_failures != UINT_MAX) &&
+ (++tx_failures < max_retrans))
+ ret = true;
+
+ return ret;
+}
static inline int _gnix_req_inject_err(struct gnix_fab_req *req)
{
int err_cnt = req->gnix_ep->domain->params.err_inject_count;
@@ -845,8 +994,4 @@ int gnix_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
#define STATIC static
#endif
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
#endif /* _GNIX_H_ */
diff --git a/prov/gni/include/gnix_atomic.h b/prov/gni/include/gnix_atomic.h
index 92b4c37..d928f5b 100644
--- a/prov/gni/include/gnix_atomic.h
+++ b/prov/gni/include/gnix_atomic.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -47,5 +47,8 @@ ssize_t _gnix_atomic(struct gnix_fid_ep *ep,
size_t result_count,
uint64_t flags);
+/* SMSG callback for AMO remote counter control message. */
+int __smsg_amo_cntr(void *data, void *msg);
+
#endif /* _GNIX_ATOMIC_H_ */
diff --git a/prov/gni/include/gnix_av.h b/prov/gni/include/gnix_av.h
index 199828e..95b7f99 100644
--- a/prov/gni/include/gnix_av.h
+++ b/prov/gni/include/gnix_av.h
@@ -79,7 +79,7 @@ struct gnix_av_addr_entry {
* @return FI_SUCCESS on success, -FI_EINVAL on error
*/
int _gnix_av_lookup(struct gnix_fid_av *gnix_av, fi_addr_t fi_addr,
- struct gnix_av_addr_entry **addr);
+ struct gnix_av_addr_entry *addr);
/**
* @brief Return the FI address mapped to a given GNIX address.
@@ -93,15 +93,68 @@ int _gnix_av_reverse_lookup(struct gnix_fid_av *gnix_av,
struct gnix_address gnix_addr,
fi_addr_t *fi_addr);
+/*******************************************************************************
+ * If the caller already knows the av type they can call the lookups directly
+ * using the following functions.
+ ******************************************************************************/
+
/**
- * @brief Return the string representation of the FI address.
+ * @brief (FI_AV_TABLE) Return the gnix address using its corresponding
+ * fi_addr.
+ *
+ * @param[in] int_av The AV to use for the lookup.
+ * @param[in] fi_addr The corresponding fi_addr_t.
+ * @param[in/out] entry_ptr pointer to an av entry struct
+ *
+ * @return FI_SUCCESS on successfully looking up the entry in the entry table.
+ * @return -FI_EINVAL upon passing an invalid parameter.
+ */
+int _gnix_table_lookup(struct gnix_fid_av *int_av,
+ fi_addr_t fi_addr,
+ struct gnix_av_addr_entry *entry_ptr);
+
+/**
+ * @brief (FI_AV_MAP) Return the gnix address using its corresponding
+ * fi_addr.
+ *
+ * @param[in] int_av The AV to use for the lookup.
+ * @param[in] fi_addr The corresponding fi_addr_t.
+ * @param[in/out] entry_ptr pointer to an av entry struct
+ *
+ * @return FI_SUCCESS on successfully looking up the entry in the entry table.
+ * @return -FI_EINVAL upon passing an invalid parameter.
+ */
+int _gnix_map_lookup(struct gnix_fid_av *int_av,
+ fi_addr_t fi_addr,
+ struct gnix_av_addr_entry *entry_ptr);
+
+/**
+ * @brief (FI_AV_TABLE) Return fi_addr using its corresponding gnix address.
+ *
+ * @param[in] int_av The AV to use for the lookup.
+ * @param[in] gnix_addr The gnix address
+ * @param[in/out] fi_addr The pointer to the corresponding fi_addr.
+ *
+ * @return FI_SUCCESS on successfully looking up the entry in the entry table.
+ * @return -FI_EINVAL upon passing an invalid parameter.
+ */
+int _gnix_table_reverse_lookup(struct gnix_fid_av *int_av,
+ struct gnix_address gnix_addr,
+ fi_addr_t *fi_addr);
+
+/**
+ * @brief (FI_AV_MAP) Return fi_addr using its corresponding gnix address.
+ *
+ * @param[in] int_av The AV to use for the lookup.
+ * @param[in] gnix_addr The gnix address
+ * @param[in/out] fi_addr The pointer to the corresponding fi_addr.
*
- * @param[in] buf The buffer that contains the address string.
- * @param(out) gnix_ep The gnix_ep_name structure that contains the
- * @return FI_SUCCESS on success or -FI_EINVAL on error.
+ * @return FI_SUCCESS on successfully looking up the entry in the entry table.
+ * @return -FI_EINVAL upon passing an invalid parameter.
*/
-int gnix_av_straddr_to_ep_name(char *buf,
- struct gnix_ep_name *gnix_ep);
+int _gnix_map_reverse_lookup(struct gnix_fid_av *int_av,
+ struct gnix_address gnix_addr,
+ fi_addr_t *fi_addr);
/**
* @brief Return the string representation of the FI address.
diff --git a/prov/gni/include/gnix_buddy_allocator.h b/prov/gni/include/gnix_buddy_allocator.h
index ed6878e..7b6c7f0 100644
--- a/prov/gni/include/gnix_buddy_allocator.h
+++ b/prov/gni/include/gnix_buddy_allocator.h
@@ -68,11 +68,16 @@ static inline uint32_t __gnix_buddy_log2(uint32_t v)
}
/* Find the bitmap index for block X of size X_LEN */
-static inline size_t __gnix_buddy_bitmap_index(void *x, size_t x_len, void *base,
- size_t base_len, size_t min_len)
+static inline size_t __gnix_buddy_bitmap_index(void *_x, size_t x_len,
+ void *_base, size_t base_len,
+ size_t min_len)
{
- return (size_t) ((x - base) / (size_t) x_len) + base_len / (min_len / 2)
- - base_len / (x_len / 2);
+ /* arithmetic on void * is not part of the C standard (yet?) */
+ uint8_t *x = _x;
+ uint8_t *base = _base;
+
+ return (size_t) ((x - base) / (size_t) x_len) +
+ base_len / (min_len / 2) - base_len / (x_len / 2);
}
/* Find the address of X's buddy block:
diff --git a/prov/gni/include/gnix_cm_nic.h b/prov/gni/include/gnix_cm_nic.h
index b11abf4..5a7e1b7 100644
--- a/prov/gni/include/gnix_cm_nic.h
+++ b/prov/gni/include/gnix_cm_nic.h
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2015 Cray Inc. All rights reserved.
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC.
+ * All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -58,6 +59,8 @@ typedef int gnix_cm_nic_rcv_cb_func(struct gnix_cm_nic *cm_nic,
* @var rcv_cb_fn pointer to callback function used to process
* incoming messages received by this cm nic
* @var ptag ptag of this nic.
+ * @var poll_cnt non-atomic counter to reduce datagram polling cnt
+ * when using FI_PROGRESS_MANUAL for control progress.
* @var device_id local Aries device id associated with this nic.
*/
struct gnix_cm_nic {
@@ -72,6 +75,7 @@ struct gnix_cm_nic {
struct gnix_ep_name my_name;
gnix_cm_nic_rcv_cb_func *rcv_cb_fn;
uint8_t ptag;
+ uint32_t poll_cnt;
uint32_t device_id;
};
@@ -132,6 +136,7 @@ int _gnix_cm_nic_free(struct gnix_cm_nic *cm_nic);
*/
int _gnix_cm_nic_alloc(struct gnix_fid_domain *domain,
struct fi_info *info,
+ uint32_t cdm_id,
struct gnix_cm_nic **cm_nic);
/**
@@ -158,24 +163,20 @@ int _gnix_cm_nic_progress(struct gnix_cm_nic *cm_nic);
* value previously returned from _gnix_cm_nic_get_cdm_seed_set
*
* @param[in] domain pointer to previously allocated gnix_fid_domain struct
- * @param[in] seed seed value to be used for creating cdm_id
* @param[out] id pointer to address where the 32 bit ids will be returned
* @return FI_SUCCESS upon generation of 32 bit id.
*/
-int _gnix_cm_nic_create_cdm_id(struct gnix_fid_domain *domain, uint32_t seed,
- uint32_t *id);
+int _gnix_cm_nic_create_cdm_id(struct gnix_fid_domain *domain, uint32_t *id);
/**
- * @brief generate a set of contiguous, unique 32 bit seed values to supply
- * to _gnix_cm_nic_create_cdm_id to generate cdm_id for calls to GNI_CdmCreate
+ * @brief generate a set of contiguous, unique 32 bit cdm_ids for use with GNI_CdmCreate
*
- * @param[in] domain pointer to previously allocated gnix_fid_domain struct
- * @param[in] nseeds number of seeds to be allocated
- * @param[out] seeds pointer to address where the 32 bit seeds will be
- * returned
+ * @param domain pointer to previously allocated gnix_fid_domain struct
+ * @param nids number of ids to be allocated
+ * @param id pointer to address where the 32 bit id will be returned
* @return FI_SUCCESS upon generate ion of 32 bit id.
*/
-int _gnix_cm_nic_get_cdm_seed_set(struct gnix_fid_domain *domain, int nseeds,
- uint32_t *seeds);
+int _gnix_get_new_cdm_id_set(struct gnix_fid_domain *domain, int nids,
+ uint32_t *id);
#endif /* _GNIX_CM_NIC_H_ */
diff --git a/prov/gni/include/gnix_cntr.h b/prov/gni/include/gnix_cntr.h
index 5eaee85..2b32247 100644
--- a/prov/gni/include/gnix_cntr.h
+++ b/prov/gni/include/gnix_cntr.h
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -34,10 +34,6 @@
#ifndef _GNIX_CNTR_H_
#define _GNIX_CNTR_H_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
#include <fi.h>
#include "gnix.h"
@@ -61,6 +57,9 @@ struct gnix_fid_cntr {
atomic_t cnt;
atomic_t cnt_err;
struct gnix_reference ref_cnt;
+ struct dlist_entry trigger_list;
+ fastlock_t trigger_lock;
+ bool requires_lock;
};
/**
@@ -97,8 +96,4 @@ int _gnix_cntr_poll_nic_add(struct gnix_fid_cntr *cntr, struct gnix_nic *nic);
*/
int _gnix_cntr_poll_nic_rem(struct gnix_fid_cntr *cntr, struct gnix_nic *nic);
-#ifdef __cplusplus
-}
-#endif
-
#endif
diff --git a/prov/gni/include/gnix_cq.h b/prov/gni/include/gnix_cq.h
index 41ad600..aade2c9 100644
--- a/prov/gni/include/gnix_cq.h
+++ b/prov/gni/include/gnix_cq.h
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -34,16 +34,13 @@
#ifndef _GNIX_CQ_H_
#define _GNIX_CQ_H_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
#include <fi.h>
#include "gnix_queue.h"
#include "gnix_wait.h"
#include "gnix_util.h"
#include <fi_list.h>
+#include <stdbool.h>
#define GNIX_CQ_DEFAULT_FORMAT struct fi_cq_entry
#define GNIX_CQ_DEFAULT_SIZE 256
@@ -78,6 +75,7 @@ struct gnix_fid_cq {
struct dlist_entry poll_nics;
rwlock_t nic_lock;
+ bool requires_lock;
};
@@ -93,8 +91,4 @@ ssize_t _gnix_cq_add_error(struct gnix_fid_cq *cq, void *op_context,
int _gnix_cq_poll_nic_add(struct gnix_fid_cq *cq, struct gnix_nic *nic);
int _gnix_cq_poll_nic_rem(struct gnix_fid_cq *cq, struct gnix_nic *nic);
-#ifdef __cplusplus
-}
-#endif
-
#endif
diff --git a/prov/gni/include/gnix_datagram.h b/prov/gni/include/gnix_datagram.h
index f69856d..ab57ba2 100644
--- a/prov/gni/include/gnix_datagram.h
+++ b/prov/gni/include/gnix_datagram.h
@@ -34,10 +34,6 @@
#ifndef _GNIX_DATAGRAM_H_
#define _GNIX_DATAGRAM_H_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
#include "gnix.h"
/*
@@ -106,9 +102,7 @@ enum gnix_dgram_type {
enum gnix_dgram_state {
GNIX_DGRAM_STATE_FREE,
- GNIX_DGRAM_STATE_CONNECTING,
- GNIX_DGRAM_STATE_LISTENING,
- GNIX_DGRAM_STATE_CONNECTED
+ GNIX_DGRAM_STATE_ACTIVE
};
enum gnix_dgram_buf {
@@ -137,14 +131,6 @@ enum gnix_dgram_poll_type {
* @var type datagram type (bound or wildcard)
* @var d_hndl pointer to datagram handle this datagram is
* associated
- * @var pre_test_clbk_fn Call back function to be called prior to
- * a call to GNI_EpPostDataTestById to retrieve
- * the datagram from GNI. This callback is invoked
- * while the lock is held on the cm nic.
- * @var post_test_clbk_fn Call back function to be called following a
- * call to GNI_EpPostDataTestById to retrieve
- * the datagram from GNI. This callback is invoked
- * while the lock is held on the cm nic.
* @var pre_post_clbk_fn Call back function to be called prior to
* to the call to GNI_EpPostDataWId. This callback
* is invoked while the lock is held on the cm nic.
@@ -186,10 +172,6 @@ struct gnix_datagram {
enum gnix_dgram_state state;
enum gnix_dgram_type type;
struct gnix_dgram_hndl *d_hndl;
- int (*pre_test_clbk_fn)(struct gnix_datagram *);
- int (*post_test_clbk_fn)(struct gnix_datagram *,
- struct gnix_address,
- gni_post_state_t);
int (*pre_post_clbk_fn)(struct gnix_datagram *,
int *);
int (*post_post_clbk_fn)(struct gnix_datagram *,
@@ -351,8 +333,4 @@ int _gnix_dgram_poll(struct gnix_dgram_hndl *hndl_ptr,
enum gnix_dgram_poll_type type);
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
#endif /* _GNIX_DATAGRAM_H_ */
diff --git a/prov/gni/include/gnix_ep.h b/prov/gni/include/gnix_ep.h
index 98710c5..17e5185 100644
--- a/prov/gni/include/gnix_ep.h
+++ b/prov/gni/include/gnix_ep.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -58,7 +58,9 @@ enum {
GNIX_SMSG_T_RNDZV_RDONE,
GNIX_SMSG_T_RNDZV_START,
GNIX_SMSG_T_RNDZV_FIN,
- GNIX_SMSG_T_RMA_DATA
+ GNIX_SMSG_T_RMA_DATA,
+ GNIX_SMSG_T_AMO_CNTR,
+ GNIX_SMSG_T_RNDZV_IOV_START
};
extern smsg_completer_fn_t gnix_ep_smsg_completers[];
@@ -140,20 +142,51 @@ typedef ssize_t (*trecvmsg_func_t)(struct fid_ep *ep,
* inline functions
*/
+static inline struct slist_entry *_gnix_ep_get_htd_buf(struct gnix_fid_ep *ep)
+{
+ struct slist_entry *e;
+
+ fastlock_acquire(&ep->htd_pool.lock);
+
+ e = slist_remove_head(&ep->htd_pool.sl);
+
+ fastlock_release(&ep->htd_pool.lock);
+
+ return e;
+}
+
+static inline gni_mem_handle_t _gnix_ep_get_htd_mdh(struct gnix_fid_ep *ep)
+{
+ return ep->htd_pool.md->mem_hndl;
+}
+
+static inline void _gnix_ep_release_htd_buf(struct gnix_fid_ep *ep, struct slist_entry *e)
+{
+ fastlock_acquire(&ep->htd_pool.lock);
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "sl.head = %p, sl.tail = %p\n", ep->htd_pool.sl.head,
+ ep->htd_pool.sl.tail);
+
+ slist_insert_head(e, &ep->htd_pool.sl);
+
+ fastlock_release(&ep->htd_pool.lock);
+}
+
static inline struct gnix_fab_req *
_gnix_fr_alloc(struct gnix_fid_ep *ep)
{
- struct slist_entry *se;
+ struct dlist_entry *de;
struct gnix_fab_req *fr = NULL;
- int ret = _gnix_sfe_alloc(&se, &ep->fr_freelist);
+ int ret = _gnix_fl_alloc(&de, &ep->fr_freelist);
while (ret == -FI_EAGAIN)
- ret = _gnix_sfe_alloc(&se, &ep->fr_freelist);
+ ret = _gnix_fl_alloc(&de, &ep->fr_freelist);
if (ret == FI_SUCCESS) {
- fr = container_of(se, struct gnix_fab_req, slist);
+ fr = container_of(de, struct gnix_fab_req, dlist);
fr->gnix_ep = ep;
- fr->slist.next = NULL; /* slist stuff isn't too smart */
+ dlist_init(&fr->dlist);
+ dlist_init(&fr->msg.tle.free);
}
/* reset common fields */
@@ -167,17 +200,24 @@ static inline void
_gnix_fr_free(struct gnix_fid_ep *ep, struct gnix_fab_req *fr)
{
assert(fr->gnix_ep == ep);
- _gnix_sfe_free(&fr->slist, &ep->fr_freelist);
+
+ if (fr->msg.htd_buf_e != NULL) {
+ _gnix_ep_release_htd_buf(ep, fr->msg.htd_buf_e);
+ fr->msg.htd_buf_e = NULL;
+ fr->msg.htd_buf = NULL;
+ }
+
+ _gnix_fl_free(&fr->dlist, &ep->fr_freelist);
_gnix_ref_put(ep);
}
static inline int
-__msg_match_fab_req(struct slist_entry *item, const void *arg)
+__msg_match_fab_req(struct dlist_entry *item, const void *arg)
{
struct gnix_fab_req *req;
const struct gnix_address *addr_ptr = arg;
- req = container_of(item, struct gnix_fab_req, slist);
+ req = container_of(item, struct gnix_fab_req, dlist);
return ((GNIX_ADDR_UNSPEC(*addr_ptr)) ||
(GNIX_ADDR_EQUAL(req->addr, *addr_ptr)));
@@ -219,6 +259,6 @@ int gnix_scalable_ep_bind(fid_t fid, struct fid *bfid, uint64_t flags);
* @return -FI_ERRNO upon an error
* @return -FI_ENOSYS if this operation is not supported
*/
-DIRECT_FN int gnix_pep_bind(fid_t fid, fid_t *bfid, uint64_t flags);
+int gnix_pep_bind(fid_t fid, fid_t *bfid, uint64_t flags);
#endif /* _GNIX_EP_H_ */
diff --git a/prov/gni/include/gnix_eq.h b/prov/gni/include/gnix_eq.h
index f553e0f..2e0c7cd 100644
--- a/prov/gni/include/gnix_eq.h
+++ b/prov/gni/include/gnix_eq.h
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -34,10 +34,6 @@
#ifndef _GNIX_EQ_H_
#define _GNIX_EQ_H_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
#include <rdma/fi_eq.h>
#include "gnix_queue.h"
@@ -46,6 +42,11 @@ extern "C" {
#define GNIX_EQ_DEFAULT_SIZE 256
+ssize_t _gnix_eq_write_error(struct fid_eq *eq, fid_t fid,
+ void *context, uint64_t index, int err,
+ int prov_errno, void *err_data,
+ size_t err_size);
+
/*
* Stores events inside of the event queue.
*
@@ -82,8 +83,4 @@ struct gnix_fid_eq {
struct gnix_reference ref_cnt;
};
-#ifdef __cplusplus
-}
-#endif
-
#endif /* _GNIX_EQ_H_ */
diff --git a/prov/gni/include/gnix_freelist.h b/prov/gni/include/gnix_freelist.h
index 33521c3..b6d9c34 100644
--- a/prov/gni/include/gnix_freelist.h
+++ b/prov/gni/include/gnix_freelist.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -34,21 +34,17 @@
#ifndef _GNIX_FREELIST_H_
#define _GNIX_FREELIST_H_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
#include <fi.h>
#include <fi_list.h>
/* Number of elements to seed the freelist with */
-#define GNIX_SFL_INIT_SIZE 100
+#define GNIX_FL_INIT_SIZE 100
/* Initial refill size */
-#define GNIX_SFL_INIT_REFILL_SIZE 10
+#define GNIX_FL_INIT_REFILL_SIZE 10
/* Refill growth factor */
-#define GNIX_SFL_GROWTH_FACTOR 2
+#define GNIX_FL_GROWTH_FACTOR 2
-/** Free list based on singly linked slist
+/** Free list implementation
*
* @var freelist The free list itself
* @var chunks Memory chunks (must be saved for freeing)
@@ -56,10 +52,10 @@ extern "C" {
* @var growth_factor Factor for increasing refill size
* @var max_refill_size; Max refill size
* @var elem_size Size of element (in bytes)
- * @var offset Offset of slist_entry field (in bytes)
+ * @var offset Offset of dlist_entry field (in bytes)
*/
-struct gnix_s_freelist {
- struct slist freelist;
+struct gnix_freelist {
+ struct dlist_entry freelist;
struct slist chunks;
int refill_size;
int growth_factor;
@@ -70,69 +66,65 @@ struct gnix_s_freelist {
fastlock_t lock;
};
-/** Initializes a gnix_s_freelist
+/** Initializes a gnix_freelist
*
* @param elem_size Size of element
- * @param offset Offset of slist_entry field
+ * @param offset Offset of dlist_entry field
* @param init_size Initial freelist size
* @param refill_size Number of elements for next refill
* @param growth_factor Factor for increasing refill size
* @param max_refill_size Max refill size
- * @param fl gnix_s_freelist
+ * @param fl gnix_freelist
* @return FI_SUCCESS on success, -FI_ENOMEM on failure
*/
-int _gnix_sfl_init(int elem_size, int offset, int init_size,
+int _gnix_fl_init(int elem_size, int offset, int init_size,
int refill_size, int growth_factor,
- int max_refill_size, struct gnix_s_freelist *fl);
+ int max_refill_size, struct gnix_freelist *fl);
-/** Initializes a thread safe gnix_s_freelist
+/** Initializes a thread safe gnix_freelist
*
* @param elem_size Size of element
- * @param offset Offset of slist_entry field
+ * @param offset Offset of dlist_entry field
* @param init_size Initial freelist size
* @param refill_size Number of elements for next refill
* @param growth_factor Factor for increasing refill size
* @param max_refill_size Max refill size
- * @param fl gnix_s_freelist
+ * @param fl gnix_freelist
* @return FI_SUCCESS on success, -FI_ENOMEM on failure
*/
-int _gnix_sfl_init_ts(int elem_size, int offset, int init_size,
+int _gnix_fl_init_ts(int elem_size, int offset, int init_size,
int refill_size, int growth_factor,
- int max_refill_size, struct gnix_s_freelist *fl);
+ int max_refill_size, struct gnix_freelist *fl);
-/** Clean up a gnix_s_freelist, including deleting memory chunks
+/** Clean up a gnix_freelist, including deleting memory chunks
*
* @param fl Freelist
*/
-void _gnix_sfl_destroy(struct gnix_s_freelist *fl);
+void _gnix_fl_destroy(struct gnix_freelist *fl);
/** Return an item from the freelist
*
* @param e item
- * @param fl gnix_s_freelist
+ * @param fl gnix_freelist
* @return FI_SUCCESS on success, -FI_ENOMEM or -FI_EAGAIN on failure
*/
-int _gnix_sfe_alloc(struct slist_entry **e, struct gnix_s_freelist *fl);
+int _gnix_fl_alloc(struct dlist_entry **e, struct gnix_freelist *fl);
/** Return an item to the free list
*
* @param e item
- * @param fl gnix_s_freelist
+ * @param fl gnix_freelist
*/
-void _gnix_sfe_free(struct slist_entry *e, struct gnix_s_freelist *fl);
+void _gnix_fl_free(struct dlist_entry *e, struct gnix_freelist *fl);
/** Is freelist empty (primarily used for testing
*
- * @param fl gnix_s_freelist
+ * @param fl gnix_freelist
* @return True if list is currently empty, false otherwise
*/
-static inline int _gnix_sfl_empty(struct gnix_s_freelist *fl)
+static inline int _gnix_fl_empty(struct gnix_freelist *fl)
{
- return slist_empty(&fl->freelist);
+ return dlist_empty(&fl->freelist);
}
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
#endif /* _GNIX_FREELIST_H_ */
diff --git a/prov/gni/include/gnix_mbox_allocator.h b/prov/gni/include/gnix_mbox_allocator.h
index 6cf2738..d149350 100644
--- a/prov/gni/include/gnix_mbox_allocator.h
+++ b/prov/gni/include/gnix_mbox_allocator.h
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -34,10 +34,6 @@
#ifndef _GNIX_MBOX_ALLOCATOR_
#define _GNIX_MBOX_ALLOCATOR_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
#include <stddef.h>
#include "gni_pub.h"
@@ -194,8 +190,4 @@ int _gnix_mbox_free(struct gnix_mbox *ptr);
*/
extern atomic_t file_id_counter;
-#ifdef __cplusplus
-}
-#endif
-
#endif /* _GNIX_MBOX_ALLOCATOR_ */
diff --git a/prov/gni/include/gnix_mr.h b/prov/gni/include/gnix_mr.h
index a04e430..ed14485 100644
--- a/prov/gni/include/gnix_mr.h
+++ b/prov/gni/include/gnix_mr.h
@@ -34,6 +34,10 @@
#ifndef GNIX_MR_H_
#define GNIX_MR_H_
+#ifdef HAVE_UDREG
+#include <udreg_pub.h>
+#endif
+
/* global includes */
#include "rdma/fi_domain.h"
@@ -41,6 +45,7 @@
#include "gnix_priv.h"
#include "gnix_mr_cache.h"
+
#define GNIX_MR_PAGE_SHIFT 12
#define GNIX_MR_PFN_BITS 37
#define GNIX_MR_MDD_BITS 12
@@ -56,6 +61,15 @@ enum {
GNIX_MR_FLAG_READONLY = 1 << 0
};
+enum {
+ GNIX_MR_TYPE_INTERNAL = 0,
+ GNIX_MR_TYPE_UDREG,
+ GNIX_MR_TYPE_NONE,
+ GNIX_MR_MAX_TYPE,
+};
+
+#define GNIX_DEFAULT_CACHE_TYPE GNIX_MR_TYPE_INTERNAL
+
/* forward declarations */
struct gnix_fid_domain;
struct gnix_nic;
@@ -74,6 +88,9 @@ struct gnix_fid_mem_desc {
struct gnix_fid_domain *domain;
gni_mem_handle_t mem_hndl;
struct gnix_nic *nic;
+#ifdef HAVE_UDREG
+ udreg_entry_t *entry;
+#endif
};
/**
@@ -101,6 +118,22 @@ typedef struct gnix_mr_key {
} gnix_mr_key_t;
/**
+ *
+ */
+struct gnix_mr_ops {
+ int (*init)(struct gnix_fid_domain *domain);
+ int (*is_init)(struct gnix_fid_domain *domain);
+ int (*reg_mr)(struct gnix_fid_domain *domain, uint64_t address,
+ uint64_t length, struct _gnix_fi_reg_context *fi_reg_context,
+ void **handle);
+ int (*dereg_mr)(struct gnix_fid_domain *domain,
+ struct gnix_fid_mem_desc *md);
+ int (*destroy_cache)(struct gnix_fid_domain *domain);
+ int (*flush_cache)(struct gnix_fid_domain *domain);
+};
+
+
+/**
* @brief Converts a libfabric key to a gni memory handle, skipping memory
* handle CRC generation.
*
@@ -129,6 +162,16 @@ void _gnix_convert_key_to_mhdl(
*/
uint64_t _gnix_convert_mhdl_to_key(gni_mem_handle_t *mhdl);
+/* initializes mr cache for a given domain */
+int _gnix_open_cache(struct gnix_fid_domain *domain, int type);
+
+/* destroys mr cache for a given domain */
+int _gnix_close_cache(struct gnix_fid_domain *domain);
+
+/* flushes the memory registration cache for a given domain */
+int _gnix_flush_registration_cache(struct gnix_fid_domain *domain);
+
+
extern gnix_mr_cache_attr_t _gnix_default_mr_cache_attr;
#endif /* GNIX_MR_H_ */
diff --git a/prov/gni/include/gnix_mr_cache.h b/prov/gni/include/gnix_mr_cache.h
index 67d9b8a..d8336a3 100644
--- a/prov/gni/include/gnix_mr_cache.h
+++ b/prov/gni/include/gnix_mr_cache.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -148,6 +148,7 @@ typedef struct gnix_mr_cache_attr {
void *reg_context;
void *dereg_context;
void *destruct_context;
+ struct gnix_mr_notifier *notifier;
void *(*reg_callback)(void *handle, void *address, size_t length,
struct _gnix_fi_reg_context *fi_reg_context,
void *context);
@@ -220,7 +221,13 @@ int _gnix_mr_cache_destroy(gnix_mr_cache_t *cache);
int _gnix_mr_cache_flush(gnix_mr_cache_t *cache);
/**
- * TODO
+ * @brief Initializes the MR cache state
+ *
+ * @param[in,out] cache a gnix memory registration cache
+ * @param[in] attr cache attributes, @see gnix_mr_cache_attr_t
+ *
+ * @return FI_SUCCESS on success
+ * -FI_ENOMEM otherwise
*/
int _gnix_mr_cache_init(
gnix_mr_cache_t **cache,
diff --git a/prov/gni/include/gnix_mr_notifier.h b/prov/gni/include/gnix_mr_notifier.h
new file mode 100644
index 0000000..159b864
--- /dev/null
+++ b/prov/gni/include/gnix_mr_notifier.h
@@ -0,0 +1,183 @@
+/*
+ * Copyright (c) 2016 Cray Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _GNIX_MR_NOTIFIER_H_
+#define _GNIX_MR_NOTIFIER_H_
+
+#include <stdint.h>
+#include <stddef.h>
+#include "rdma/fi_errno.h"
+
+#ifdef HAVE_KDREG
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <kdreg_pub.h>
+
+#include "rdma/providers/fi_log.h"
+#include "gnix_util.h"
+
+#define KDREG_DEV "/dev/kdreg"
+
+typedef volatile uint64_t kdreg_user_delta_t;
+
+/**
+ * @brief memory registration notifier
+ *
+ * @var fd File descriptor for KDREG_DEV
+ * @var cntr Kernel managed counter for monitored events
+ * @var lock Only used for set up and tear down (no guarantees
+ * if reading or writing while setting up or tearing down)
+ */
+struct gnix_mr_notifier {
+ int fd;
+ kdreg_user_delta_t *cntr;
+ fastlock_t lock;
+};
+
+/**
+ * @brief initialize the gnix_mr_notifier struct
+ *
+ * @param[out] k Empty (zeroed) gnix_mr_notifier struct
+ * @return FI_SUCESSS on success
+ * -FI_EINVAL if k == NULL
+ */
+int _gnix_notifier_init(struct gnix_mr_notifier *mrn);
+
+/**
+ * @brief open the kdreg device and prepare for notifications
+ *
+ * @param[in,out] k Empty and initialized gnix_mr_notifier struct
+ * @return FI_SUCESSS on success
+ * -FI_EBUSY if device already open
+ * -FI_ENODATA if user delta unavailable
+ * -fi_errno or -errno on other failures
+ */
+int _gnix_notifier_open(struct gnix_mr_notifier *mrn);
+
+/**
+ * @brief close the kdreg device and zero the notifier
+ *
+ * @param[in] k gnix_mr_notifier struct
+ * @return FI_SUCESSS on success
+ * -fi_errno or -errno on other failures
+ */
+int _gnix_notifier_close(struct gnix_mr_notifier *mrn);
+
+/**
+ * @brief monitor a memory region
+ *
+ * @param[in] k gnix_mr_notifier struct
+ * @param[in] addr address of memory region to monitor
+ * @param[in] len length of memory region
+ * @param[in] cookie user identifier associated with the region
+ * @return FI_SUCESSS on success
+ * -fi_errno or -errno on failure
+ */
+int _gnix_notifier_monitor(struct gnix_mr_notifier *mrn, void *addr,
+ uint64_t len, uint64_t cookie);
+
+/**
+ * @brief stop monitoring a memory region
+ *
+ * @param[in] k gnix_mr_notifier struct
+ * @param[out] cookie user identifier for notification event
+ * @return FI_SUCESSS on success
+ * -fi_errno or -errno on failure
+ */
+int _gnix_notifier_unmonitor(struct gnix_mr_notifier *mrn, uint64_t cookie);
+
+/**
+ * @brief get a monitoring event
+ *
+ * @param[in] k gnix_mr_notifier struct
+ * @param[out] buf buffer to write event data
+ * @param[in] len length of buffer
+ * @return Number of bytes read on success
+ * -FI_EINVAL if invalid arguments
+ * -FI_EAGAIN if nothing to read
+ * -fi_errno or -errno on failure
+ */
+int _gnix_notifier_get_event(struct gnix_mr_notifier *mrn,
+ void* buf, size_t len);
+
+#else
+
+struct gnix_mr_notifier {
+ int dummy[0];
+};
+
+static inline int
+_gnix_notifier_init(struct gnix_mr_notifier *mrn)
+{
+ return FI_SUCCESS;
+}
+
+static inline int
+_gnix_notifier_open(struct gnix_mr_notifier *mrn)
+{
+ return FI_SUCCESS;
+}
+
+static inline int
+_gnix_notifier_close(struct gnix_mr_notifier *mrn)
+{
+ return FI_SUCCESS;
+}
+
+static inline int
+_gnix_notifier_monitor(struct gnix_mr_notifier *mrn, void *addr,
+ uint64_t len, uint64_t cookie)
+{
+ return FI_SUCCESS;
+}
+
+static inline int
+_gnix_notifier_unmonitor(struct gnix_mr_notifier *mrn, uint64_t cookie)
+{
+ return FI_SUCCESS;
+}
+
+static inline int
+_gnix_notifier_get_event(struct gnix_mr_notifier *mrn, void* buf, size_t len)
+{
+ return -FI_EAGAIN;
+}
+
+#endif /* HAVE_KDREG */
+
+#endif /* _GNIX_MR_NOTIFIER_H_ */
diff --git a/prov/gni/include/gnix_msg.h b/prov/gni/include/gnix_msg.h
index e36ee6b..9e0153f 100644
--- a/prov/gni/include/gnix_msg.h
+++ b/prov/gni/include/gnix_msg.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -37,9 +37,17 @@
ssize_t _gnix_recv(struct gnix_fid_ep *ep, uint64_t buf, size_t len, void *desc,
uint64_t src_addr, void *context, uint64_t flags,
uint64_t tag, uint64_t ignore);
+
ssize_t _gnix_send(struct gnix_fid_ep *ep, uint64_t loc_addr, size_t len,
void *mdesc, uint64_t dest_addr, void *context,
uint64_t flags, uint64_t data, uint64_t tag);
-#endif /* _GNIX_MSG_H_ */
+ssize_t _gnix_recvv(struct gnix_fid_ep *ep, const struct iovec *iov,
+ void **desc, size_t count, uint64_t src_addr, void *context,
+ uint64_t flags, uint64_t ignore, uint64_t tag);
+ssize_t _gnix_sendv(struct gnix_fid_ep *ep, const struct iovec *iov,
+ void **mdesc, size_t count, uint64_t dest_addr,
+ void *context, uint64_t flags, uint64_t tag);
+
+#endif /* _GNIX_MSG_H_ */
diff --git a/prov/gni/include/gnix_nameserver.h b/prov/gni/include/gnix_nameserver.h
index fac8a30..7956985 100644
--- a/prov/gni/include/gnix_nameserver.h
+++ b/prov/gni/include/gnix_nameserver.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -34,10 +34,6 @@
#ifndef _GNIX_NAMESERVER_H_
#define _GNIX_NAMESERVER_H_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
#include "gnix.h"
/*
@@ -52,8 +48,4 @@ int gnix_resolve_name(IN const char *node, IN const char *service,
IN uint64_t flags, INOUT struct gnix_ep_name
*resolved_addr);
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
#endif /* _GNIX_NAMESERVER_H_ */
diff --git a/prov/gni/include/gnix_nic.h b/prov/gni/include/gnix_nic.h
index 20a82ae..3dbb989 100644
--- a/prov/gni/include/gnix_nic.h
+++ b/prov/gni/include/gnix_nic.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -34,10 +34,6 @@
#ifndef _GNIX_NIC_H_
#define _GNIX_NIC_H_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
#if HAVE_CONFIG_H
#include <config.h>
#endif /* HAVE_CONFIG_H */
@@ -84,9 +80,12 @@ typedef int (*smsg_completer_fn_t)(void *desc, gni_return_t);
* @var gni_nic_hndl optional previously allocated gni_nic_hndl to
* use for allocating GNI resources (GNI CQs) for
* this nic
+ *
+ * @var gni_cdm_modes The mode bits gni_cdm_hndl was created with.
*/
struct gnix_nic_attr {
gni_cdm_handle_t gni_cdm_hndl;
+ uint32_t gni_cdm_modes;
gni_nic_handle_t gni_nic_hndl;
bool use_cdm_id;
uint32_t cdm_id;
@@ -102,6 +101,7 @@ struct gnix_nic_attr {
* gni_nic_hndl, rx_cq, and tx_cq
* @var gni_cdm_hndl handle for the GNI communication domain (CDM)
* this nic is bound to.
+ * @var gni_cdm_modes The mode bits gni_cdm_hndl was created with.
* @var gni_nic_hndl handle for the GNI nic to which this GNIX nic is bound
* @var rx_cq GNI rx cq (non-blocking) bound to this nic
* @var rx_cq_blk GNI rx cq (blocking) bound to this nic
@@ -156,6 +156,7 @@ struct gnix_nic {
fastlock_t lock;
uint32_t allocd_gni_res;
gni_cdm_handle_t gni_cdm_hndl;
+ uint32_t gni_cdm_modes;
gni_nic_handle_t gni_nic_hndl;
gni_cq_handle_t rx_cq;
gni_cq_handle_t rx_cq_blk;
@@ -184,6 +185,7 @@ struct gnix_nic {
gnix_bitmap_t vc_id_bitmap;
uint32_t mem_per_mbox;
struct gnix_mbox_alloc_handle *mbox_hndl;
+ /* TODO: gnix_buddy_alloc_handle_t *alloc_handle */
struct gnix_mbox_alloc_handle *s_rdma_buf_hndl;
struct gnix_mbox_alloc_handle *r_rdma_buf_hndl;
struct gnix_reference ref_cnt;
@@ -194,6 +196,7 @@ struct gnix_nic {
gni_mem_handle_t irq_mem_hndl;
void *irq_mmap_addr;
size_t irq_mmap_len;
+ int requires_lock;
};
@@ -241,23 +244,59 @@ struct gnix_smsg_rndzv_start_hdr {
};
/**
+ * gnix_smsg_rndzv_iov_start_hdr
+ *
+ * @var flags the sender's flags needed on the receive side.
+ * @var imm the immediate data associated with this message.
+ * @var msg_tag the tag associated with this message.
+ * @var mdh the memory handle associated with the iov buffer.
+ * @var iov_cnt the length of the scatter/gather vector.
+ * @var req_addr the sender's fabric request address.
+ * @var send_len the cumulative size (in bytes) of the client's
+ * iov base buffers.
+ *
+ * @note the actual iov base addresses and lengths are placed in the
+ * data section of the start message.
+ */
+struct gnix_smsg_rndzv_iov_start_hdr {
+ uint64_t flags;
+ uint64_t imm;
+ uint64_t msg_tag;
+ uint64_t req_addr;
+ size_t iov_cnt;
+ uint64_t send_len;
+};
+
+/**
* gnix_smsg_rndzv_fin_hdr - first part of a rendezvous send fin SMSG message
*
* @var req_addr returned local request address
*/
struct gnix_smsg_rndzv_fin_hdr {
uint64_t req_addr;
+ int status;
};
/**
* gnix_smsg_rma_data_hdr - RMA remote data message
*
- * @var flags remote CQ RMA flags
- * @var data remote CQ immediate data
+ * @var flags control flags
+ * @var user_flags remote CQ user flags
+ * @var user_data remote CQ user immediate data
*/
struct gnix_smsg_rma_data_hdr {
uint64_t flags;
- uint64_t data;
+ uint64_t user_flags;
+ uint64_t user_data;
+};
+
+/**
+ * gnix_smsg_amo_cntr_hdr - RMA remote counter message
+ *
+ * @var user_flags control flags
+ */
+struct gnix_smsg_amo_cntr_hdr {
+ uint64_t flags;
};
/**
@@ -270,9 +309,11 @@ struct gnix_smsg_rma_data_hdr {
* used for unaligned gets
* @var gnix_smsg_eager_hdr embedded header for SMSG eager protocol
* @var gnix_smsg_rndzv_start_hdr embedded header for rendezvous protocol
+ * @var gnix_smsg_rndzv_iov_start_hdr embedded header for iovec rndzv protocol
* @var gnix_smsg_rndzv_fin_hdr embedded header for rendezvous protocol
* @var gnix_smsg_rndzv_rma_data_hdr embedded header for remote notification for
* rma operations
+ * @var gnix_smsg_amo_cntr_hdr embedded header for AMO remote counter events.
* @var req pointer to fab request associated with this descriptor
* @var completer_fn call back to invoke when associated GNI CQE's are
* returned.
@@ -280,6 +321,7 @@ struct gnix_smsg_rma_data_hdr {
* from GNI_CQ_MSG_ID
* @var err_list Error TXD list entry
* @var int_buf Intermediate buffer for landing unaligned data, etc.
+ * @var tx_failures Number of times this transmission descriptor failed.
*/
struct gnix_tx_descriptor {
struct dlist_entry list;
@@ -288,10 +330,12 @@ struct gnix_tx_descriptor {
gni_post_descriptor_t gni_desc;
gni_ct_get_post_descriptor_t gni_ct_descs[2];
};
- struct gnix_smsg_eager_hdr eager_hdr;
- struct gnix_smsg_rndzv_start_hdr rndzv_start_hdr;
- struct gnix_smsg_rndzv_fin_hdr rndzv_fin_hdr;
- struct gnix_smsg_rma_data_hdr rma_data_hdr;
+ struct gnix_smsg_eager_hdr eager_hdr;
+ struct gnix_smsg_rndzv_start_hdr rndzv_start_hdr;
+ struct gnix_smsg_rndzv_iov_start_hdr rndzv_iov_start_hdr;
+ struct gnix_smsg_rndzv_fin_hdr rndzv_fin_hdr;
+ struct gnix_smsg_rma_data_hdr rma_data_hdr;
+ struct gnix_smsg_amo_cntr_hdr amo_cntr_hdr;
};
struct gnix_fab_req *req;
int (*completer_fn)(void *, gni_return_t);
@@ -418,8 +462,4 @@ static inline void *__gnix_nic_elem_by_rem_id(struct gnix_nic *nic, int rem_id)
void _gnix_nic_txd_err_inject(struct gnix_nic *nic,
struct gnix_tx_descriptor *txd);
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
#endif /* _GNIX_NIC_H_ */
diff --git a/prov/gni/include/gnix_poll.h b/prov/gni/include/gnix_poll.h
index f4b9c19..5c9c2b4 100644
--- a/prov/gni/include/gnix_poll.h
+++ b/prov/gni/include/gnix_poll.h
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2015-2016 Los Alamos National Security, LLC. All
- * rights reserved.
+ * Copyright (c) 2016 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -34,10 +34,6 @@
#ifndef _GNIX_POLL_H_
#define _GNIX_POLL_H_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
/*******************************************************************************
* API Functions
******************************************************************************/
@@ -87,8 +83,4 @@ int gnix_poll_del(struct fid_poll *pollset, struct fid *event_fid,
* Exposed internal functions.
******************************************************************************/
-#ifdef __cplusplus
-}
-#endif
-
#endif /* _GNIX_POLL_H_ */
diff --git a/prov/gni/include/gnix_priv.h b/prov/gni/include/gnix_priv.h
index 3230650..89466f7 100644
--- a/prov/gni/include/gnix_priv.h
+++ b/prov/gni/include/gnix_priv.h
@@ -35,32 +35,12 @@
#include "gnix.h"
+extern uint8_t precomputed_crc_results[256];
+
/*
* Start of code pulled from gni_priv.h
*/
-inline static uint8_t gni_crc_bits(uint8_t data)
-{
- uint8_t lcrc = 0;
-
- if(data & 1)
- lcrc ^= 0x5e;
- if(data & 2)
- lcrc ^= 0xbc;
- if(data & 4)
- lcrc ^= 0x61;
- if(data & 8)
- lcrc ^= 0xc2;
- if(data & 0x10)
- lcrc ^= 0x9d;
- if(data & 0x20)
- lcrc ^= 0x23;
- if(data & 0x40)
- lcrc ^= 0x46;
- if(data & 0x80)
- lcrc ^= 0x8c;
-
- return lcrc;
-}
+#define gni_crc_bits(data) precomputed_crc_results[(data)]
inline static uint8_t gni_memhndl_calc_crc(gni_mem_handle_t *memhndl)
{
diff --git a/prov/gni/include/gnix_queue.h b/prov/gni/include/gnix_queue.h
index 7918136..af0b403 100644
--- a/prov/gni/include/gnix_queue.h
+++ b/prov/gni/include/gnix_queue.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
+ * Copyright (c) 2016 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -33,10 +34,6 @@
#ifndef _GNIX_QUEUE_H
#define _GNIX_QUEUE_H
-#ifdef __cplusplus
-extern "C" {
-#endif
-
#include <fi_list.h>
typedef struct slist_entry *(*alloc_func)(size_t entry_size);
@@ -67,8 +64,4 @@ void _gnix_queue_enqueue(struct gnix_queue *queue, struct slist_entry *item);
void _gnix_queue_enqueue_free(struct gnix_queue *queue,
struct slist_entry *item);
-#ifdef __cplusplus
-}
-#endif
-
#endif /* #define _GNIX_QUEUE_H */
diff --git a/prov/gni/include/gnix_rma.h b/prov/gni/include/gnix_rma.h
index 4ba3710..55ebe77 100644
--- a/prov/gni/include/gnix_rma.h
+++ b/prov/gni/include/gnix_rma.h
@@ -53,6 +53,8 @@ ssize_t _gnix_rma(struct gnix_fid_ep *ep, enum gnix_fab_req_type fr_type,
*/
int _gnix_rma_post_irq(struct gnix_vc *vc);
+/* SMSG callback for RMA data control message. */
+int __smsg_rma_data(void *data, void *msg);
#endif /* _GNIX_RMA_H_ */
diff --git a/prov/gni/test/common.c b/prov/gni/include/gnix_trigger.h
similarity index 71%
rename from prov/gni/test/common.c
rename to prov/gni/include/gnix_trigger.h
index e42bccc..9adb83d 100644
--- a/prov/gni/test/common.c
+++ b/prov/gni/include/gnix_trigger.h
@@ -1,5 +1,6 @@
/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2016 Cray Inc. All rights reserved.
+ * Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -30,16 +31,23 @@
* SOFTWARE.
*/
-#include "common.h"
+/*
+ * Triggered operations handling.
+ */
+
+#ifndef GNIX_TRIGGER_H_
+#define GNIX_TRIGGER_H_
-void calculate_time_difference(struct timeval *start, struct timeval *end,
- int *secs_out, int *usec_out)
+#include "gnix.h"
+#include "gnix_cntr.h"
+#include "gnix_vc.h"
+
+int _gnix_trigger_queue_req(struct gnix_fab_req *req);
+void _gnix_trigger_check_cntr(struct gnix_fid_cntr *cntr);
+
+static inline int _gnix_trigger_pending(struct gnix_fid_cntr *cntr)
{
- *secs_out = end->tv_sec - start->tv_sec;
- if (end->tv_usec < start->tv_usec) {
- *secs_out = *secs_out - 1;
- *usec_out = (1000000 + end->tv_usec) - start->tv_usec;
- } else {
- *usec_out = end->tv_usec - start->tv_usec;
- }
+ return dlist_empty(&cntr->trigger_list) ? 0 : 1;
}
+
+#endif /* GNIX_TRIGGER_H */
diff --git a/prov/gni/include/gnix_util.h b/prov/gni/include/gnix_util.h
index 0dec49b..41106f9 100644
--- a/prov/gni/include/gnix_util.h
+++ b/prov/gni/include/gnix_util.h
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
* Copyright (c) 2015-2016 Cray Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -40,7 +41,6 @@
#include <stdio.h>
#include <fi.h>
-#include "rdma/fi_log.h"
extern struct fi_provider gnix_prov;
@@ -127,7 +127,7 @@ extern atomic_t gnix_debug_next_tid;
} while (0)
#endif
-/* dlist utilities */
+/* slist and dlist utilities */
#include "fi_list.h"
static inline void dlist_node_init(struct dlist_entry *e)
@@ -141,6 +141,8 @@ static inline void dlist_remove_init(struct dlist_entry *e)
e->prev = e->next = e;
}
+#define DLIST_IN_LIST(e) e.prev != e.next
+
#define DLIST_HEAD(dlist) struct dlist_entry dlist = { &(dlist), &(dlist) }
#define dlist_entry(e, type, member) container_of(e, type, member)
@@ -227,11 +229,13 @@ int _gnix_job_disable_unassigned_cpus(void);
int _gnix_job_enable_affinity_apply(void);
int _gnix_job_disable_affinity_apply(void);
-void _gnix_alps_cleanup(void);
+void _gnix_app_cleanup(void);
int _gnix_job_fma_limit(uint32_t dev_id, uint8_t ptag, uint32_t *limit);
int _gnix_job_cq_limit(uint32_t dev_id, uint8_t ptag, uint32_t *limit);
int _gnix_pes_on_node(uint32_t *num_pes);
int _gnix_nics_per_rank(uint32_t *nics_per_rank);
+void _gnix_dump_gni_res(uint8_t ptag);
+int _gnix_get_num_corespec_cpus(uint32_t *num_core_spec_cpus);
struct gnix_reference {
atomic_t references;
@@ -272,6 +276,34 @@ static inline void _gnix_ref_init(
ref->destruct = destruct;
}
+#define __STRINGIFY(expr) #expr
+#define STRINGIFY(expr) __STRINGIFY(expr)
+
+#define __COND_FUNC(cond, lock, func) \
+ do { \
+ if ((cond)) { \
+ func(lock); \
+ } \
+ } while (0)
+
+#define COND_ACQUIRE(cond, lock) \
+ __COND_FUNC((cond), (lock), fastlock_acquire)
+#define COND_READ_ACQUIRE(cond, lock) \
+ __COND_FUNC((cond), (lock), rwlock_rdlock)
+#define COND_WRITE_ACQUIRE(cond, lock) \
+ __COND_FUNC((cond), (lock), rwlock_wrlock)
+
+#define COND_RELEASE(cond, lock) \
+ __COND_FUNC((cond), (lock), fastlock_release)
+#define COND_RW_RELEASE(cond, lock) \
+ __COND_FUNC((cond), (lock), rwlock_unlock)
+#ifdef __GNUC__
+#define __PREFETCH(addr, rw, locality) __builtin_prefetch(addr, rw, locality)
+#else
+#define __PREFETCH(addr, rw, locality) ((void *) 0)
+#endif
+#define READ_PREFETCH(addr) __PREFETCH(addr, 0, 3)
+#define WRITE_PREFETCH(addr) __PREFETCH(addr, 1, 3)
#endif
diff --git a/prov/gni/include/gnix_vc.h b/prov/gni/include/gnix_vc.h
index 2c38100..2636e75 100644
--- a/prov/gni/include/gnix_vc.h
+++ b/prov/gni/include/gnix_vc.h
@@ -34,10 +34,6 @@
#ifndef _GNIX_VC_H_
#define _GNIX_VC_H_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
#if HAVE_CONFIG_H
#include <config.h>
#endif /* HAVE_CONFIG_H */
@@ -45,6 +41,7 @@ extern "C" {
#include "gnix.h"
#include "gnix_bitmap.h"
#include "gnix_av.h"
+#include "gnix_xpmem.h"
/*
* mode bits
@@ -54,6 +51,8 @@ extern "C" {
#define GNIX_VC_MODE_DG_POSTED (1U << 2)
#define GNIX_VC_MODE_PENDING_MSGS (1U << 3)
#define GNIX_VC_MODE_PEER_CONNECTED (1U << 4)
+#define GNIX_VC_MODE_IN_TABLE (1U << 5)
+#define GNIX_VC_MODE_XPMEM (1U << 6)
/* VC flags */
#define GNIX_VC_FLAG_RX_SCHEDULED 0
@@ -119,15 +118,15 @@ enum gnix_vc_conn_req_type {
struct gnix_vc {
struct dlist_entry rx_list; /* RX VC list entry */
- struct slist work_queue; /* Work reqs */
+ struct dlist_entry work_queue; /* Work reqs */
fastlock_t work_queue_lock; /* Work req lock */
struct dlist_entry work_list; /* Work VC list entry */
- struct slist tx_queue; /* TX reqs */
+ struct dlist_entry tx_queue; /* TX reqs */
fastlock_t tx_queue_lock; /* TX reqs lock */
struct dlist_entry tx_list; /* TX VC list entry */
- struct dlist_entry entry;
+ struct dlist_entry list; /* General purpose list */
fi_addr_t peer_fi_addr;
struct gnix_address peer_addr;
struct gnix_address peer_cm_nic_addr;
@@ -143,6 +142,7 @@ struct gnix_vc {
int modes;
gnix_bitmap_t flags; /* We're missing regular bit ops */
gni_mem_handle_t peer_irq_mem_hndl;
+ xpmem_apid_t peer_apid;
uint64_t peer_caps;
};
@@ -342,4 +342,5 @@ static inline enum gnix_vc_conn_state _gnix_vc_state(struct gnix_vc *vc)
return vc->conn_state;
}
+
#endif /* _GNIX_VC_H_ */
diff --git a/prov/gni/include/gnix_vector.h b/prov/gni/include/gnix_vector.h
index d7873e5..0ab0839 100644
--- a/prov/gni/include/gnix_vector.h
+++ b/prov/gni/include/gnix_vector.h
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2015-2016 Los Alamos National Security, LLC.
* All rights reserved.
+ * Copyright (c) 2016 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -58,7 +59,7 @@ typedef enum gnix_vec_lock {
GNIX_VEC_LOCKED,
} gnix_vec_lock_e;
-typedef uint64_t gnix_vec_index_t;
+typedef uint32_t gnix_vec_index_t;
typedef void * gnix_vec_entry_t;
/**
@@ -71,11 +72,11 @@ typedef void * gnix_vec_entry_t;
* @var creator fn required to properly alloc the vector element
*/
typedef struct gnix_vec_attr {
- uint64_t vec_initial_size;
- uint64_t cur_size;
- uint64_t vec_maximum_size;
+ uint32_t vec_initial_size;
+ uint32_t cur_size;
+ uint32_t vec_maximum_size;
- uint64_t vec_increase_step;
+ uint32_t vec_increase_step;
gnix_vec_increase_e vec_increase_type;
@@ -84,35 +85,49 @@ typedef struct gnix_vec_attr {
struct gnix_vector;
+struct gnix_vector_iter {
+ struct gnix_vector *vec;
+ uint32_t cur_idx;
+};
+
+#define GNIX_VECTOR_ITERATOR(_vec, _iter) \
+ struct gnix_vector_iter _iter = { \
+ .vec = (_vec), \
+ .cur_idx = 0, \
+ }
+
+/* Returns the current index of the iterator */
+#define GNIX_VECTOR_ITERATOR_IDX(_iter) ((_iter).cur_idx - 1)
+
/**
* Vector operations
*
- * @var insert_first Insert an entry into first index of the vector.
* @var insert_last Insert an entry into the last index of the vector.
* @var insert_at Insert an entry into the vector at the given index.
*
- * @var remove_first Removes the first element from the vector.
* @var remove_last Removes the last element from the vector.
* @var remove_at Removes the element at index from the vector.
*
- * @var first Return the first element of the vector.
* @var last Return the last element of the vector.
* @var at Return the element at the specified index.
+ *
+ * @var iter_next Return the element at the current index and move them
+ * index to the next element.
*/
typedef struct gnix_vector_ops {
- int (*insert_first)(struct gnix_vector *, gnix_vec_entry_t *);
+ int (*resize)(struct gnix_vector *, uint32_t);
+
int (*insert_last)(struct gnix_vector *, gnix_vec_entry_t *);
int (*insert_at)(struct gnix_vector *, gnix_vec_entry_t *,
gnix_vec_index_t);
- int (*remove_first)(struct gnix_vector *);
int (*remove_last)(struct gnix_vector *);
int (*remove_at)(struct gnix_vector *, gnix_vec_index_t);
- int (*first)(struct gnix_vector *, void **);
int (*last)(struct gnix_vector *, void **);
int (*at)(struct gnix_vector *, void **, gnix_vec_index_t);
- /* TODO: void *(*iter_next)(struct gnix_vector_iter *); */
+
+ gnix_vec_entry_t *(*iter_next)(struct gnix_vector_iter *);
} gnix_vector_ops_t;
/**
@@ -160,6 +175,9 @@ int _gnix_vec_init(struct gnix_vector *vec, gnix_vec_attr_t *attr);
*/
int _gnix_vec_close(gnix_vector_t *vec);
+/*******************************************************************************
+ * INLINE OPS FNS
+ ******************************************************************************/
/**
* Resize the vector to size.
*
@@ -172,12 +190,19 @@ int _gnix_vec_close(gnix_vector_t *vec);
* than the maximum vector size
* @return -FI_ENOMEM Upon running out of memory
*/
-int _gnix_vec_resize(gnix_vector_t *vec, uint64_t size);
+static inline int _gnix_vec_resize(gnix_vector_t *vec, uint32_t size)
+{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+ if (unlikely(!vec)) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "Invalid parameter to "
+ "_gnix_vec_resize.\n");
+ return -FI_EINVAL;
+ } else {
+ return vec->ops->resize(vec, size);
+ }
+}
-/*******************************************************************************
- * INLINE OPS FNS
- ******************************************************************************/
/**
* Get the element at index in the vector.
*
@@ -189,7 +214,19 @@ int _gnix_vec_resize(gnix_vector_t *vec, uint64_t size);
* @return -FI_EINVAL Upon passing a NULL or dead vector
* @return -FI_ECANCLED Upon attempting to get an empty element
*/
-int _gnix_vec_at(gnix_vector_t *vec, void **element, gnix_vec_index_t index);
+static inline int _gnix_vec_at(gnix_vector_t *vec, void **element,
+ gnix_vec_index_t index)
+{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+
+ if (unlikely(!vec || !element)) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "Invalid parameter to "
+ "_gnix_vec_at\n");
+ return -FI_EINVAL;
+ } else {
+ return vec->ops->at(vec, element, index);
+ }
+}
/**
* Get the first element in the vector.
@@ -201,7 +238,18 @@ int _gnix_vec_at(gnix_vector_t *vec, void **element, gnix_vec_index_t index);
* @return -FI_EINVAL Upon passing a NULL or dead vector
* @return -FI_ECANCLED Upon attempting to get an empty element
*/
-int _gnix_vec_last(gnix_vector_t *vec, void **element);
+static inline int _gnix_vec_last(gnix_vector_t *vec, void **element)
+{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+
+ if (unlikely(!vec || !element)) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "Invalid parameter to "
+ "_gnix_vec_last\n");
+ return -FI_EINVAL;
+ } else {
+ return vec->ops->last(vec, element);
+ }
+}
/**
* Get the first element in the vector.
@@ -213,7 +261,10 @@ int _gnix_vec_last(gnix_vector_t *vec, void **element);
* @return -FI_EINVAL Upon passing a NULL or dead vector
* @return -FI_ECANCLED Upon attempting to get an empty element
*/
-int _gnix_vec_first(gnix_vector_t *vec, void **element);
+static inline int _gnix_vec_first(gnix_vector_t *vec, void **element)
+{
+ return _gnix_vec_at(vec, element, 0);
+}
/**
* Removes the element at index from the vector. Note that
@@ -227,7 +278,19 @@ int _gnix_vec_first(gnix_vector_t *vec, void **element);
* @return -FI_EINVAL Upon passing a dead vector
* @return -FI_ECANCELED Upon attempting to remove an empty entry
*/
-int _gnix_vec_remove_at(gnix_vector_t *vec, gnix_vec_index_t index);
+static inline int _gnix_vec_remove_at(gnix_vector_t *vec,
+ gnix_vec_index_t index)
+{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+
+ if (unlikely(!vec)) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "Invalid parameter to "
+ "_gnix_vec_remove_at\n");
+ return -FI_EINVAL;
+ } else {
+ return vec->ops->remove_at(vec, index);
+ }
+}
/**
* Removes the last element from the vector. Note that
@@ -240,7 +303,18 @@ int _gnix_vec_remove_at(gnix_vector_t *vec, gnix_vec_index_t index);
* @return -FI_EINVAL Upon passing a dead entry
* @return -FI_ECANCELED Upon attempting to remove an empty entry
*/
-int _gnix_vec_remove_last(gnix_vector_t *vec);
+static inline int _gnix_vec_remove_last(gnix_vector_t *vec)
+{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+
+ if (unlikely(!vec)) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "Invalid parameter to "
+ "_gnix_vec_remove_at\n");
+ return -FI_EINVAL;
+ } else {
+ return vec->ops->remove_last(vec);
+ }
+}
/**
* Removes the first element from the vector. Note that
@@ -253,7 +327,10 @@ int _gnix_vec_remove_last(gnix_vector_t *vec);
* @return -FI_EINVAL Upon passing a dead entry
* @return -FI_ECANCELED Upon attempting to remove an empty entry
*/
-int _gnix_vec_remove_first(gnix_vector_t *vec);
+static inline int _gnix_vec_remove_first(gnix_vector_t *vec)
+{
+ return _gnix_vec_remove_at(vec, 0);
+}
/**
* Inserts an entry into the vector at the given index. If the current size
@@ -272,8 +349,18 @@ int _gnix_vec_remove_first(gnix_vector_t *vec);
* @return -FI_ECANCELED Upon an existing non-empty entry being found at index
* or reaching the maximum vector size.
*/
-int _gnix_vec_insert_at(gnix_vector_t *vec, gnix_vec_entry_t *entry,
- gnix_vec_index_t index);
+static inline int _gnix_vec_insert_at(gnix_vector_t *vec,
+ gnix_vec_entry_t *entry,
+ gnix_vec_index_t index)
+{
+ if (unlikely(!vec || !entry)) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "Invalid parameter to "
+ "_gnix_vec_insert_at\n");
+ return -FI_EINVAL;
+ } else {
+ return vec->ops->insert_at(vec, entry, index);
+ }
+}
/**
* Inserts an entry into the last index of the vector. If the entry at the
@@ -288,7 +375,17 @@ int _gnix_vec_insert_at(gnix_vector_t *vec, gnix_vec_entry_t *entry,
* @return -FI_ECANCELED Upon an existing non-empty entry being found at the
* last index
*/
-int _gnix_vec_insert_last(gnix_vector_t *vec, gnix_vec_entry_t *entry);
+static inline int _gnix_vec_insert_last(gnix_vector_t *vec,
+ gnix_vec_entry_t *entry)
+{
+ if (unlikely(!vec || !entry)) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "Invalid parameter to "
+ "_gnix_vec_insert_last\n");
+ return -FI_EINVAL;
+ } else {
+ return vec->ops->insert_last(vec, entry);
+ }
+}
/**
* Inserts an entry into the first index of the vector. If the entry at the
@@ -302,6 +399,29 @@ int _gnix_vec_insert_last(gnix_vector_t *vec, gnix_vec_entry_t *entry);
* entry
* @return -FI_ECANCELED Upon an existing non-empty entry being found at index 0
*/
-int _gnix_vec_insert_first(gnix_vector_t *vec, gnix_vec_entry_t *entry);
+static inline int _gnix_vec_insert_first(gnix_vector_t *vec,
+ gnix_vec_entry_t *entry)
+{
+ return _gnix_vec_insert_at(vec, entry, 0);
+}
+
+/**
+ * Return the current element in the vector iterator and move
+ * the iterator to the next element.
+ *
+ * @param iter pointer to the vector iterator
+ * @return pointer to current element in the vector
+ */
+static inline
+gnix_vec_entry_t *_gnix_vec_iterator_next(struct gnix_vector_iter *iter)
+{
+ if (iter == NULL) {
+ GNIX_WARN(FI_LOG_EP_DATA, "Invalid parameter to"
+ "_gnix_vec_iterator_next\n");
+ return NULL;
+ } else {
+ return iter->vec->ops->iter_next(iter);
+ }
+}
#endif /* GNIX_VECTOR_H_ */
diff --git a/prov/gni/include/gnix_wait.h b/prov/gni/include/gnix_wait.h
index 88871ac..3a29ff0 100644
--- a/prov/gni/include/gnix_wait.h
+++ b/prov/gni/include/gnix_wait.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
+ * Copyright (c) 2016 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -33,10 +34,6 @@
#ifndef _GNIX_WAIT_H_
#define _GNIX_WAIT_H_
-#ifdef __cplusplus
-extern "C" {
-#endif
-
#include <pthread.h>
#include <rdma/fi_eq.h>
#include <fi_list.h>
@@ -87,8 +84,4 @@ int _gnix_wait_set_remove(struct fid_wait *wait, struct fid *wait_obj);
int _gnix_get_wait_obj(struct fid_wait *wait, void *arg);
void _gnix_signal_wait_obj(struct fid_wait *wait);
-#ifdef __cplusplus
-}
-#endif
-
#endif
diff --git a/prov/gni/include/gnix_xpmem.h b/prov/gni/include/gnix_xpmem.h
new file mode 100644
index 0000000..423d654
--- /dev/null
+++ b/prov/gni/include/gnix_xpmem.h
@@ -0,0 +1,207 @@
+/*
+ * Copyright (c) 2016 Los Alamos National Security, LLC.
+ * All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef GNIX_XPMEM_H_
+#define GNIX_XPMEM_H_
+
+#include "gnix.h"
+#if HAVE_XPMEM
+#include <xpmem.h>
+#else
+typedef int64_t xpmem_apid_t;
+typedef int64_t xpmem_segid_t;
+#endif
+
+struct gnix_xpmem_handle {
+ struct gnix_reference ref_cnt;
+ struct gnix_hashtable *apid_ht;
+ fastlock_t lock;
+};
+
+struct gnix_xpmem_access_handle {
+ struct gnix_xpmem_handle *xp_hndl;
+ struct gnix_xpmem_ht_entry *entry;
+ void *attach_addr;
+ void *remote_base_addr;
+ size_t access_len;
+};
+
+extern bool gnix_xpmem_disabled;
+
+/*******************************************************************************
+ * API Prototypes
+ ******************************************************************************/
+
+/**
+ * @brief create an xpmem handle to use for subsequent
+ * xpmem operations
+ *
+ * @param [in] dom pointer to a previously allocated
+ * gnix_fid_domain struct
+ * @param [out] handle pointer to a memory location where
+ * a pointer to an xpmem_handle will be
+ * returned
+ *
+ * @return FI_SUCCESS xpmem handle successfully allocated
+ * @return -FI_EINVAL Upon receiving an invalid parameter
+ */
+int _gnix_xpmem_handle_create(struct gnix_fid_domain *dom,
+ struct gnix_xpmem_handle **handle);
+
+/**
+ * @brief destroy an xpmem handle
+ *
+ * @param [in] handle pointer to a previously allocated
+ * xpmem_handle
+ * @return FI_SUCCESS xpmem handle successfully destroyed
+ * @return -FI_EINVAL Upon receiving an invalid parameter
+ */
+int _gnix_xpmem_handle_destroy(struct gnix_xpmem_handle *hndl);
+
+/**
+ * @brief get an access handle to a address range a peer's
+ * address space
+ *
+ * @param[in] xp_handle pointer to previously created
+ * xpmem handle
+ * @param[in] peer_apid xpmem apid for peer
+ * @param[in] remote_vaddr virtual address in process associated
+ * with the target EP
+ * @param[in] len length in bytes of the region to
+ * to be accessed in the target process
+ * @param[out] access_hndl access handle to be used to copy data
+ * from the peer process in to the local
+ * address space
+ *
+ * @return FI_SUCCESS Upon xpmem successfully initialized
+ * @return -FI_EINVAL Upon receiving an invalid parameter
+ * @return -FI_ENOSYS Target EP can't be attached to local process
+ * address space
+ */
+int _gnix_xpmem_access_hndl_get(struct gnix_xpmem_handle *xp_hndl,
+ xpmem_apid_t peer_apid,
+ uint64_t remote_vaddr,
+ size_t len,
+ struct gnix_xpmem_access_handle **access_hndl);
+
+
+/**
+ * @brief release an access handle
+ *
+ * @param[in] access_handle pointer to previously created
+ * access handle
+ *
+ * @return FI_SUCCESS Upon xpmem successfully initialized
+ * @return -FI_EINVAL Upon receiving an invalid parameter
+ */
+int _gnix_xpmem_access_hndl_put(struct gnix_xpmem_access_handle *access_hndl);
+
+/**
+ * @brief memcpy from previously accessed memory in peer's
+ * virtual address space
+ *
+ * @param[in] access_hndl pointer to previously created
+ * xpmem access handle
+ * @param[in] dst_addr starting virtual address in the calling
+ * process address space where data
+ * will be copied
+ * @param[in] remote_start_addr starting virtual address in the target
+ * address space from which data will be copied
+ * @param[in] len copy length in bytes
+ *
+ * @return FI_SUCCESS Upon successful copy
+ * @return -FI_EINVAL Invalid argument
+ */
+int _gnix_xpmem_copy(struct gnix_xpmem_access_handle *access_hndl,
+ void *dst_addr,
+ void *remote_start_addr,
+ size_t len);
+
+/**
+ * @brief get the xpmem segid associated with an xpmem_handle
+ *
+ * @param[in] xp_handle pointer to previously created
+ * will be copied
+ * @param[out] seg_id pointer to memory location where
+ * the segid value will be returned
+ *
+ * @return FI_SUCCESS Upon success
+ * @return -FI_EINVAL Invalid argument
+ */
+int _gnix_xpmem_get_my_segid(struct gnix_xpmem_handle *xp_hndl,
+ xpmem_segid_t *seg_id);
+
+/**
+ * @brief get the xpmem apid associated with an xpmem_handle
+ * and input segid
+ *
+ * @param[in] xp_handle pointer to previously created
+ * will be copied
+ * @param[in] seg_id seg_id obtained from process
+ * whose memory is to be accessed
+ * via xpmem.
+ * @param[out] peer_apid pointer to memory location where
+ * the apid value to use for accessing
+ * the address space of the peer
+ * process.
+ *
+ * @return FI_SUCCESS Upon success
+ * @return -FI_EINVAL Invalid argument
+ */
+int _gnix_xpmem_get_apid(struct gnix_xpmem_handle *xp_hndl,
+ xpmem_segid_t segid,
+ xpmem_apid_t *peer_apid);
+
+/**
+ * @brief determine if a process at a given gnix_address can
+ * be accessed using xpmem
+ *
+ * @param[in] ep pointer to a previously allocated
+ * gnix_fid_ep structure
+ * @param[in] addr address used by an endpoint of the
+ * peer process
+ * @param[out] accessible set to true if endpoint with
+ * gnix_address addr can be accessed
+ * using xpmem, otherwise false
+ *
+ * @return FI_SUCCESS Upon success
+ * @return -FI_EINVAL Invalid argument
+ */
+int _gnix_xpmem_accessible(struct gnix_fid_ep *ep,
+ struct gnix_address addr,
+ bool *accessible);
+
+
+
+
+#endif /* GNIX_XPMEM_H_ */
diff --git a/prov/gni/include/rdma/fi_direct.h b/prov/gni/include/rdma/fi_direct.h
deleted file mode 100644
index be977ea..0000000
--- a/prov/gni/include/rdma/fi_direct.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright (c) 2013-2014 Intel Corporation. All rights reserved.
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All
- * rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _FI_DIRECT_H_
-#define _FI_DIRECT_H_
-
-#define FABRIC_DIRECT_ 1
-
-struct fi_context {
- void *internal[4];
-};
-
-#endif /* _FI_DIRECT_H_ */
diff --git a/prov/gni/include/rdma/fi_direct_atomic.h b/prov/gni/include/rdma/fi_direct_atomic.h
deleted file mode 100644
index bd55998..0000000
--- a/prov/gni/include/rdma/fi_direct_atomic.h
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All
- * rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _FI_DIRECT_ATOMIC_H_
-#define _FI_DIRECT_ATOMIC_H_
-
-#define FABRIC_DIRECT_ATOMIC 1
-
-/*******************************************************************************
- * GNI API Functions
- ******************************************************************************/
-extern ssize_t gnix_ep_atomic_write(struct fid_ep *ep, const void *buf,
- size_t count, void *desc,
- fi_addr_t dest_addr, uint64_t addr,
- uint64_t key, enum fi_datatype datatype,
- enum fi_op op, void *context);
-
-extern ssize_t gnix_ep_atomic_writev(struct fid_ep *ep,
- const struct fi_ioc *iov, void **desc,
- size_t count, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key,
- enum fi_datatype datatype, enum fi_op op,
- void *context);
-
-extern ssize_t gnix_ep_atomic_writemsg(struct fid_ep *ep,
- const struct fi_msg_atomic *msg,
- uint64_t flags);
-
-extern ssize_t gnix_ep_atomic_inject(struct fid_ep *ep, const void *buf,
- size_t count, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key,
- enum fi_datatype datatype, enum fi_op op);
-
-extern ssize_t gnix_ep_atomic_readwrite(struct fid_ep *ep, const void *buf,
- size_t count, void *desc, void *result,
- void *result_desc, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key,
- enum fi_datatype datatype,
- enum fi_op op, void *context);
-
-extern ssize_t gnix_ep_atomic_readwritev(struct fid_ep *ep,
- const struct fi_ioc *iov,
- void **desc, size_t count,
- struct fi_ioc *resultv,
- void **result_desc,
- size_t result_count,
- fi_addr_t dest_addr, uint64_t addr,
- uint64_t key,
- enum fi_datatype datatype,
- enum fi_op op, void *context);
-
-extern ssize_t gnix_ep_atomic_readwritemsg(struct fid_ep *ep,
- const struct fi_msg_atomic *msg,
- struct fi_ioc *resultv,
- void **result_desc,
- size_t result_count, uint64_t flags);
-
-extern ssize_t gnix_ep_atomic_compwrite(struct fid_ep *ep, const void *buf,
- size_t count, void *desc,
- const void *compare, void *compare_desc,
- void *result, void *result_desc,
- fi_addr_t dest_addr, uint64_t addr,
- uint64_t key, enum fi_datatype datatype,
- enum fi_op op, void *context);
-
-extern ssize_t gnix_ep_atomic_compwritev(struct fid_ep *ep,
- const struct fi_ioc *iov, void **desc,
- size_t count,
- const struct fi_ioc *comparev,
- void **compare_desc,
- size_t compare_count,
- struct fi_ioc *resultv,
- void **result_desc,
- size_t result_count,
- fi_addr_t dest_addr, uint64_t addr,
- uint64_t key,
- enum fi_datatype datatype,
- enum fi_op op, void *context);
-
-extern ssize_t
-gnix_ep_atomic_compwritemsg(struct fid_ep *ep, const struct fi_msg_atomic *msg,
- const struct fi_ioc *comparev, void **compare_desc,
- size_t compare_count, struct fi_ioc *resultv,
- void **result_desc, size_t result_count,
- uint64_t flags);
-
-extern ssize_t gnix_ep_atomic_valid(struct fid_ep *ep,
- enum fi_datatype datatype, enum fi_op op,
- size_t *);
-
-extern int gnix_ep_fetch_atomic_valid(struct fid_ep *ep,
- enum fi_datatype datatype, enum fi_op op,
- size_t *count);
-
-extern ssize_t gnix_ep_cmp_atomic_valid(struct fid_ep *ep,
- enum fi_datatype datatype,
- enum fi_op op, size_t *count);
-
-/*******************************************************************************
- * Libfabric API Functions
- ******************************************************************************/
-static inline ssize_t fi_atomic(struct fid_ep *ep, const void *buf,
- size_t count, void *desc, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key,
- enum fi_datatype datatype, enum fi_op op,
- void *context)
-{
- return gnix_ep_atomic_write(ep, buf, count, desc, dest_addr, addr, key,
- datatype, op, context);
-}
-
-static inline ssize_t fi_atomicv(struct fid_ep *ep, const struct fi_ioc *iov,
- void **desc, size_t count, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key,
- enum fi_datatype datatype, enum fi_op op,
- void *context)
-{
- return gnix_ep_atomic_writev(ep, iov, desc, count, dest_addr, addr, key,
- datatype, op, context);
-}
-
-static inline ssize_t
-fi_atomicmsg(struct fid_ep *ep, const struct fi_msg_atomic *msg, uint64_t flags)
-{
- return gnix_ep_atomic_writemsg(ep, msg, flags);
-}
-
-static inline ssize_t fi_inject_atomic(struct fid_ep *ep, const void *buf,
- size_t count, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key,
- enum fi_datatype datatype, enum fi_op op)
-{
- return gnix_ep_atomic_inject(ep, buf, count, dest_addr, addr, key,
- datatype, op);
-}
-
-static inline ssize_t fi_fetch_atomic(struct fid_ep *ep, const void *buf,
- size_t count, void *desc, void *result,
- void *result_desc, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key,
- enum fi_datatype datatype, enum fi_op op,
- void *context)
-{
- return gnix_ep_atomic_readwrite(ep, buf, count, desc, result,
- result_desc, dest_addr, addr, key,
- datatype, op, context);
-}
-
-static inline ssize_t fi_fetch_atomicv(struct fid_ep *ep,
- const struct fi_ioc *iov, void **desc,
- size_t count, struct fi_ioc *resultv,
- void **result_desc, size_t result_count,
- fi_addr_t dest_addr, uint64_t addr,
- uint64_t key, enum fi_datatype datatype,
- enum fi_op op, void *context)
-{
- return gnix_ep_atomic_readwritev(ep, iov, desc, count, resultv,
- result_desc, result_count, dest_addr,
- addr, key, datatype, op, context);
-}
-
-static inline ssize_t fi_fetch_atomicmsg(struct fid_ep *ep,
- const struct fi_msg_atomic *msg,
- struct fi_ioc *resultv,
- void **result_desc,
- size_t result_count, uint64_t flags)
-{
- return gnix_ep_atomic_readwritemsg(ep, msg, resultv, result_desc,
- result_count, flags);
-}
-
-static inline ssize_t fi_compare_atomic(struct fid_ep *ep, const void *buf,
- size_t count, void *desc,
- const void *compare, void *compare_desc,
- void *result, void *result_desc,
- fi_addr_t dest_addr, uint64_t addr,
- uint64_t key, enum fi_datatype datatype,
- enum fi_op op, void *context)
-{
- return gnix_ep_atomic_compwrite(
- ep, buf, count, desc, compare, compare_desc, result, result_desc,
- dest_addr, addr, key, datatype, op, context);
-}
-
-static inline ssize_t fi_compare_atomicv(struct fid_ep *ep,
- const struct fi_ioc *iov,
- void **desc, size_t count,
- const struct fi_ioc *comparev,
- void **compare_desc,
- size_t compare_count,
- struct fi_ioc *resultv,
- void **result_desc,
- size_t result_count,
- fi_addr_t dest_addr, uint64_t addr,
- uint64_t key,
- enum fi_datatype datatype,
- enum fi_op op, void *context)
-{
- return gnix_ep_atomic_compwritev(ep, iov, desc, count, comparev,
- compare_desc, compare_count, resultv,
- result_desc, result_count, dest_addr,
- addr, key, datatype, op, context);
-}
-
-static inline ssize_t
-fi_compare_atomicmsg(struct fid_ep *ep, const struct fi_msg_atomic *msg,
- const struct fi_ioc *comparev, void **compare_desc,
- size_t compare_count, struct fi_ioc *resultv,
- void **result_desc, size_t result_count, uint64_t flags)
-{
- return gnix_ep_atomic_compwritemsg(ep, msg, comparev, compare_desc,
- compare_count, resultv, result_desc,
- result_count, flags);
-}
-
-static inline int fi_atomicvalid(struct fid_ep *ep, enum fi_datatype datatype,
- enum fi_op op, size_t *count)
-{
- return gnix_ep_atomic_valid(ep, datatype, op, count);
-}
-
-static inline int fi_fetch_atomicvalid(struct fid_ep *ep,
- enum fi_datatype datatype, enum fi_op op,
- size_t *count)
-{
- return gnix_ep_fetch_atomic_valid(ep, datatype, op, count);
-}
-
-static inline int fi_compare_atomicvalid(struct fid_ep *ep,
- enum fi_datatype datatype,
- enum fi_op op, size_t *count)
-{
- return gnix_ep_cmp_atomic_valid(ep, datatype, op, count);
-}
-
-#endif /* _FI_DIRECT_ATOMIC_H_ */
diff --git a/prov/gni/include/rdma/fi_direct_atomic_def.h b/prov/gni/include/rdma/fi_direct_atomic_def.h
deleted file mode 100644
index 1bf521f..0000000
--- a/prov/gni/include/rdma/fi_direct_atomic_def.h
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC.
- * All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _FI_DIRECT_ATOMIC_DEF_H_
-#define _FI_DIRECT_ATOMIC_DEF_H_
-
-#define FABRIC_DIRECT_ATOMIC_DEF 1
-
-enum fi_datatype {
- FI_INT8,
- FI_UINT8,
- FI_INT16,
- FI_UINT16,
- FI_INT32,
- FI_UINT32,
- FI_INT64,
- FI_UINT64,
- FI_FLOAT,
- FI_DOUBLE,
- FI_FLOAT_COMPLEX,
- FI_DOUBLE_COMPLEX,
- FI_LONG_DOUBLE,
- FI_LONG_DOUBLE_COMPLEX,
- FI_DATATYPE_LAST
-};
-
-enum fi_op {
- FI_MIN,
- FI_MAX,
- FI_SUM,
- FI_PROD,
- FI_LOR,
- FI_LAND,
- FI_BOR,
- FI_BAND,
- FI_LXOR,
- FI_BXOR,
- FI_ATOMIC_READ,
- FI_ATOMIC_WRITE,
- FI_CSWAP,
- FI_CSWAP_NE,
- FI_CSWAP_LE,
- FI_CSWAP_LT,
- FI_CSWAP_GE,
- FI_CSWAP_GT,
- FI_MSWAP,
- FI_ATOMIC_OP_LAST
-};
-
-#endif /* _FI_DIRECT_ATOMIC_DEF_H_ */
diff --git a/prov/gni/include/rdma/fi_direct_cm.h b/prov/gni/include/rdma/fi_direct_cm.h
deleted file mode 100644
index 301a184..0000000
--- a/prov/gni/include/rdma/fi_direct_cm.h
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All
- * rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _FI_DIRECT_CM_H_
-#define _FI_DIRECT_CM_H_
-
-#define FABRIC_DIRECT_CM 1
-
-/*******************************************************************************
- * GNI API Functions
- ******************************************************************************/
-extern int gnix_setname(fid_t fid, void *addr, size_t addrlen);
-
-extern int gnix_getname(fid_t fid, void *addr, size_t *addrlen);
-
-extern int gnix_getpeer(struct fid_ep *ep, void *addr, size_t *addrlen);
-
-extern int gnix_listen(struct fid_pep *pep);
-
-extern int gnix_connect(struct fid_ep *ep, const void *addr, const void *param,
- size_t paramlen);
-
-extern int gnix_accept(struct fid_ep *ep, const void *param, size_t paramlen);
-
-extern int gnix_reject(struct fid_pep *pep, fid_t handle, const void *param,
- size_t paramlen);
-
-extern int gnix_shutdown(struct fid_ep *ep, uint64_t flags);
-
-/*******************************************************************************
- * Libfabric API Functions
- ******************************************************************************/
-static inline int fi_setname(fid_t fid, void *addr, size_t addrlen)
-{
- return gnix_setname(fid, addr, addrlen);
-}
-
-static inline int fi_getname(fid_t fid, void *addr, size_t *addrlen)
-{
- return gnix_getname(fid, addr, addrlen);
-}
-
-static inline int fi_getpeer(struct fid_ep *ep, void *addr, size_t *addrlen)
-{
- return gnix_getpeer(ep, addr, addrlen);
-}
-
-static inline int fi_listen(struct fid_pep *pep)
-{
- return gnix_listen(pep);
-}
-
-static inline int fi_connect(struct fid_ep *ep, const void *addr,
- const void *param, size_t paramlen)
-{
- return gnix_connect(ep, addr, param, paramlen);
-}
-
-static inline int fi_accept(struct fid_ep *ep, const void *param,
- size_t paramlen)
-{
- return gnix_accept(ep, param, paramlen);
-}
-
-static inline int fi_reject(struct fid_pep *pep, fid_t handle,
- const void *param, size_t paramlen)
-{
- return gnix_reject(pep, handle, param, paramlen);
-}
-
-static inline int fi_shutdown(struct fid_ep *ep, uint64_t flags)
-{
- return gnix_shutdown(ep, flags);
-}
-
-#endif /* _FI_DIRECT_CM_H_ */
diff --git a/prov/gni/include/rdma/fi_direct_domain.h b/prov/gni/include/rdma/fi_direct_domain.h
deleted file mode 100644
index 9ad8507..0000000
--- a/prov/gni/include/rdma/fi_direct_domain.h
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All
- * rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _FI_DIRECT_DOMAIN_H_
-#define _FI_DIRECT_DOMAIN_H_
-
-#define FABRIC_DIRECT_DOMAIN 1
-
-/*******************************************************************************
- * GNI API Functions
- ******************************************************************************/
-extern int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info,
- struct fid_domain **dom, void *context);
-
-extern int gnix_domain_bind(struct fid_domain *domain, struct fid *fid,
- uint64_t flags);
-
-extern int gnix_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
- struct fid_cq **cq, void *context);
-
-extern int gnix_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
- struct fid_cntr **cntr, void *context);
-
-extern int gnix_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
- struct fid_wait **waitset);
-
-extern int gnix_poll_open(struct fid_domain *domain, struct fi_poll_attr *attr,
- struct fid_poll **pollset);
-
-extern int gnix_mr_reg(struct fid *fid, const void *buf, size_t len,
- uint64_t access, uint64_t offset, uint64_t requested_key,
- uint64_t flags, struct fid_mr **mr_o, void *context);
-
-extern int gnix_mr_bind(fid_t fid, struct fid *bfid, uint64_t flags);
-
-extern int gnix_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
- struct fid_av **av, void *context);
-
-extern int gnix_av_bind(struct fid_av *av, struct fid *fid, uint64_t flags);
-
-extern int gnix_av_insert(struct fid_av *av, const void *addr, size_t count,
- fi_addr_t *fi_addr, uint64_t flags, void *context);
-
-extern int gnix_av_insertsvc(struct fid_av *av, const char *node,
- const char *service, fi_addr_t *fi_addr,
- uint64_t flags, void *context);
-
-extern int gnix_av_insertsym(struct fid_av *av, const char *node,
- size_t nodecnt, const char *service, size_t svccnt,
- fi_addr_t *fi_addr, uint64_t flags, void *context);
-
-extern int gnix_av_remove(struct fid_av *av, fi_addr_t *fi_addr, size_t count,
- uint64_t flags);
-
-extern int gnix_av_lookup(struct fid_av *av, fi_addr_t fi_addr, void *addr,
- size_t *addrlen);
-
-extern const char *gnix_av_straddr(struct fid_av *av, const void *addr,
- char *buf, size_t *len);
-
-/*******************************************************************************
- * Libfabric API Functions
- ******************************************************************************/
-static inline int fi_domain(struct fid_fabric *fabric, struct fi_info *info,
- struct fid_domain **domain, void *context)
-{
- return gnix_domain_open(fabric, info, domain, context);
-}
-
-static inline int fi_domain_bind(struct fid_domain *domain, struct fid *fid,
- uint64_t flags)
-{
- return gnix_domain_bind(domain, fid, flags);
-}
-
-static inline int fi_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
- struct fid_cq **cq, void *context)
-{
- return gnix_cq_open(domain, attr, cq, context);
-}
-
-static inline int fi_cntr_open(struct fid_domain *domain,
- struct fi_cntr_attr *attr,
- struct fid_cntr **cntr, void *context)
-{
- return gnix_cntr_open(domain, attr, cntr, context);
-}
-
-static inline int fi_wait_open(struct fid_fabric *fabric,
- struct fi_wait_attr *attr,
- struct fid_wait **waitset)
-{
- return gnix_wait_open(fabric, attr, waitset);
-}
-
-static inline int fi_poll_open(struct fid_domain *domain,
- struct fi_poll_attr *attr,
- struct fid_poll **pollset)
-{
- return gnix_poll_open(domain, attr, pollset);
-}
-
-static inline int fi_mr_reg(struct fid_domain *domain, const void *buf,
- size_t len, uint64_t access, uint64_t offset,
- uint64_t requested_key, uint64_t flags,
- struct fid_mr **mr, void *context)
-{
- return gnix_mr_reg(&domain->fid, buf, len, access, offset,
- requested_key, flags, mr, context);
-}
-
-static inline void *fi_mr_desc(struct fid_mr *mr)
-{
- return mr->mem_desc;
-}
-
-static inline uint64_t fi_mr_key(struct fid_mr *mr)
-{
- return mr->key;
-}
-
-static inline int fi_mr_bind(struct fid_mr *mr, struct fid *bfid,
- uint64_t flags)
-{
- return gnix_mr_bind(&mr->fid, bfid, flags);
-}
-
-static inline int fi_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
- struct fid_av **av, void *context)
-{
- return gnix_av_open(domain, attr, av, context);
-}
-
-static inline int fi_av_bind(struct fid_av *av, struct fid *fid, uint64_t flags)
-{
- return gnix_av_bind(av, fid, flags);
-}
-
-static inline int fi_av_insert(struct fid_av *av, const void *addr,
- size_t count, fi_addr_t *fi_addr, uint64_t flags,
- void *context)
-{
- return gnix_av_insert(av, addr, count, fi_addr, flags, context);
-}
-
-static inline int fi_av_insertsvc(struct fid_av *av, const char *node,
- const char *service, fi_addr_t *fi_addr,
- uint64_t flags, void *context)
-{
- return gnix_av_insertsvc(av, node, service, fi_addr, flags, context);
-}
-
-static inline int fi_av_insertsym(struct fid_av *av, const char *node,
- size_t nodecnt, const char *service,
- size_t svccnt, fi_addr_t *fi_addr,
- uint64_t flags, void *context)
-{
- return gnix_av_insertsym(av, node, nodecnt, service, svccnt, fi_addr,
- flags, context);
-}
-
-static inline int fi_av_remove(struct fid_av *av, fi_addr_t *fi_addr,
- size_t count, uint64_t flags)
-{
- return gnix_av_remove(av, fi_addr, count, flags);
-}
-
-static inline int fi_av_lookup(struct fid_av *av, fi_addr_t fi_addr, void *addr,
- size_t *addrlen)
-{
- return gnix_av_lookup(av, fi_addr, addr, addrlen);
-}
-
-static inline const char *fi_av_straddr(struct fid_av *av, const void *addr,
- char *buf, size_t *len)
-{
- return gnix_av_straddr(av, addr, buf, len);
-}
-
-static inline fi_addr_t fi_rx_addr(fi_addr_t fi_addr, int rx_index,
- int rx_ctx_bits)
-{
- return (fi_addr_t)(((uint64_t)rx_index << (64 - rx_ctx_bits)) |
- fi_addr);
-}
-
-#endif /* _FI_DIRECT_DOMAIN_H_ */
diff --git a/prov/gni/include/rdma/fi_direct_endpoint.h b/prov/gni/include/rdma/fi_direct_endpoint.h
deleted file mode 100644
index df4c3bf..0000000
--- a/prov/gni/include/rdma/fi_direct_endpoint.h
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All
- * rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _FI_DIRECT_ENDPOINT_H_
-#define _FI_DIRECT_ENDPOINT_H_
-
-#define FABRIC_DIRECT_ENDPOINT 1
-
-/*******************************************************************************
- * GNI API Functions
- ******************************************************************************/
-extern int gnix_passive_ep_open(struct fid_fabric *fabric, struct fi_info *info,
- struct fid_pep **pep, void *context);
-
-extern int gnix_ep_open(struct fid_domain *domain, struct fi_info *info,
- struct fid_ep **ep, void *context);
-
-extern int gnix_scalable_ep_open(struct fid_domain *domain,
- struct fi_info *info, struct fid_ep **ep,
- void *context);
-
-extern int gnix_ep_bind(fid_t fid, fid_t bfid, uint64_t flags);
-
-extern int gnix_pep_bind(fid_t pep, fid_t bfid, uint64_t flags);
-
-extern int gnix_scalable_ep_bind(fid_t sep, fid_t bfid, uint64_t flags);
-
-extern int gnix_ep_control(fid_t fid, int command, void *arg);
-
-extern ssize_t gnix_ep_cancel(fid_t fid, void *context);
-
-extern int gnix_ep_setopt(fid_t fid, int level, int optname, const void *optval,
- size_t optlen);
-
-extern int gnix_getopt(fid_t fid, int level, int optname, void *optval,
- size_t *optlen);
-
-extern int gnix_tx_context(struct fid_ep *ep, int index,
- struct fi_tx_attr *attr, struct fid_ep **tx_ep,
- void *context);
-
-extern int gnix_rx_context(struct fid_ep *ep, int index,
- struct fi_rx_attr *attr, struct fid_ep **rx_ep,
- void *context);
-
-extern ssize_t gnix_ep_tx_size_left(struct fid_ep *ep);
-
-extern ssize_t gnix_ep_rx_size_left(struct fid_ep *ep);
-
-extern int gnix_stx_open(struct fid_domain *dom, struct fi_tx_attr *tx_attr,
- struct fid_stx **stx, void *context);
-
-extern int gnix_srx_context(struct fid_domain *domain, struct fi_rx_attr *attr,
- struct fid_ep **rx_ep, void *context);
-
-extern ssize_t gnix_ep_recv(struct fid_ep *ep, void *buf, size_t len,
- void *desc, fi_addr_t src_addr, void *context);
-
-extern ssize_t gnix_ep_recvv(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t src_addr,
- void *context);
-
-extern ssize_t gnix_ep_recvmsg(struct fid_ep *ep, const struct fi_msg *msg,
- uint64_t flags);
-
-extern ssize_t gnix_ep_send(struct fid_ep *ep, const void *buf, size_t len,
- void *desc, fi_addr_t dest_addr, void *context);
-
-extern ssize_t gnix_ep_sendv(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t dest_addr,
- void *context);
-
-extern ssize_t gnix_ep_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
- uint64_t flags);
-
-extern ssize_t gnix_ep_msg_inject(struct fid_ep *ep, const void *buf,
- size_t len, fi_addr_t dest_addr);
-
-extern ssize_t gnix_ep_senddata(struct fid_ep *ep, const void *buf, size_t len,
- void *desc, uint64_t data, fi_addr_t dest_addr,
- void *context);
-
-extern ssize_t gnix_ep_msg_injectdata(struct fid_ep *ep, const void *buf,
- size_t len, uint64_t data,
- fi_addr_t dest_addr);
-
-/*******************************************************************************
- * Libfabric API Functions
- ******************************************************************************/
-static inline int fi_passive_ep(struct fid_fabric *fabric, struct fi_info *info,
- struct fid_pep **pep, void *context)
-{
- return gnix_passive_ep_open(fabric, info, pep, context);
-}
-
-static inline int fi_endpoint(struct fid_domain *domain, struct fi_info *info,
- struct fid_ep **ep, void *context)
-{
- return gnix_ep_open(domain, info, ep, context);
-}
-
-static inline int fi_scalable_ep(struct fid_domain *domain,
- struct fi_info *info, struct fid_ep **sep,
- void *context)
-{
- return gnix_scalable_ep_open(domain, info, sep, context);
-}
-
-static inline int fi_ep_bind(struct fid_ep *ep, fid_t bfid, uint64_t flags)
-{
- return gnix_ep_bind(&ep->fid, bfid, flags);
-}
-
-static inline int fi_pep_bind(struct fid_pep *pep, fid_t bfid, uint64_t flags)
-{
- return gnix_pep_bind(&pep->fid, bfid, flags);
-}
-
-static inline int fi_scalable_ep_bind(struct fid_ep *sep, fid_t bfid,
- uint64_t flags)
-{
- return gnix_scalable_ep_bind(&sep->fid, bfid, flags);
-}
-
-static inline int fi_enable(struct fid_ep *ep)
-{
- return gnix_ep_control(&ep->fid, FI_ENABLE, NULL);
-}
-
-static inline ssize_t fi_cancel(fid_t fid, void *context)
-{
- struct fid_ep *ep = container_of(fid, struct fid_ep, fid);
- return gnix_ep_cancel(&ep->fid, context);
-}
-
-static inline int fi_setopt(fid_t fid, int level, int optname,
- const void *optval, size_t optlen)
-{
- return gnix_ep_setopt(fid, level, optname, optval, optlen);
-}
-
-static inline int fi_getopt(fid_t fid, int level, int optname, void *optval,
- size_t *optlen)
-{
- return gnix_getopt(fid, level, optname, optval, optlen);
-}
-
-static inline int fi_tx_context(struct fid_ep *ep, int index,
- struct fi_tx_attr *attr, struct fid_ep **tx_ep,
- void *context)
-{
- return gnix_tx_context(ep, index, attr, tx_ep, context);
-}
-
-static inline int fi_rx_context(struct fid_ep *ep, int index,
- struct fi_rx_attr *attr, struct fid_ep **rx_ep,
- void *context)
-{
- return gnix_rx_context(ep, index, attr, rx_ep, context);
-}
-
-static inline ssize_t fi_rx_size_left(struct fid_ep *ep)
-{
- return gnix_ep_rx_size_left(ep);
-}
-
-static inline ssize_t fi_tx_size_left(struct fid_ep *ep)
-{
- return gnix_ep_tx_size_left(ep);
-}
-
-static inline int fi_stx_context(struct fid_domain *domain,
- struct fi_tx_attr *attr, struct fid_stx **stx,
- void *context)
-{
- return gnix_stx_open(domain, attr, stx, context);
-}
-
-static inline int fi_srx_context(struct fid_domain *domain,
- struct fi_rx_attr *attr, struct fid_ep **rx_ep,
- void *context)
-{
- return gnix_srx_context(domain, attr, rx_ep, context);
-}
-
-static inline ssize_t fi_recv(struct fid_ep *ep, void *buf, size_t len,
- void *desc, fi_addr_t src_addr, void *context)
-{
- return gnix_ep_recv(ep, buf, len, desc, src_addr, context);
-}
-
-static inline ssize_t fi_recvv(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t src_addr,
- void *context)
-{
- return gnix_ep_recvv(ep, iov, desc, count, src_addr, context);
-}
-
-static inline ssize_t fi_recvmsg(struct fid_ep *ep, const struct fi_msg *msg,
- uint64_t flags)
-{
- return gnix_ep_recvmsg(ep, msg, flags);
-}
-
-static inline ssize_t fi_send(struct fid_ep *ep, const void *buf, size_t len,
- void *desc, fi_addr_t dest_addr, void *context)
-{
- return gnix_ep_send(ep, buf, len, desc, dest_addr, context);
-}
-
-static inline ssize_t fi_sendv(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t dest_addr,
- void *context)
-{
- return gnix_ep_sendv(ep, iov, desc, count, dest_addr, context);
-}
-
-static inline ssize_t fi_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
- uint64_t flags)
-{
- return gnix_ep_sendmsg(ep, msg, flags);
-}
-
-static inline ssize_t fi_inject(struct fid_ep *ep, const void *buf, size_t len,
- fi_addr_t dest_addr)
-{
- return gnix_ep_msg_inject(ep, buf, len, dest_addr);
-}
-
-static inline ssize_t fi_senddata(struct fid_ep *ep, const void *buf,
- size_t len, void *desc, uint64_t data,
- fi_addr_t dest_addr, void *context)
-{
- return gnix_ep_senddata(ep, buf, len, desc, data, dest_addr, context);
-}
-
-static inline ssize_t fi_injectdata(struct fid_ep *ep, const void *buf,
- size_t len, uint64_t data,
- fi_addr_t dest_addr)
-{
- return gnix_ep_msg_injectdata(ep, buf, len, data, dest_addr);
-}
-
-#endif /* _FI_DIRECT_ENDPOINT_H_ */
diff --git a/prov/gni/include/rdma/fi_direct_eq.h b/prov/gni/include/rdma/fi_direct_eq.h
deleted file mode 100644
index f6fbcda..0000000
--- a/prov/gni/include/rdma/fi_direct_eq.h
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All
- * rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _FI_DIRECT_EQ_H_
-#define _FI_DIRECT_EQ_H_
-
-#define FABRIC_DIRECT_EQ 1
-
-/*******************************************************************************
- * GNI API Functions
- ******************************************************************************/
-extern int gnix_fabric_trywait(struct fid_fabric *fabric, struct fid **fids,
- int count);
-
-extern int gnix_wait_wait(struct fid_wait *wait, int timeout);
-
-extern int gnix_poll_poll(struct fid_poll *pollset, void **context, int count);
-
-extern int gnix_poll_add(struct fid_poll *pollset, struct fid *event_fid,
- uint64_t flags);
-
-extern int gnix_poll_del(struct fid_poll *pollset, struct fid *event_fid,
- uint64_t flags);
-
-extern int gnix_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr,
- struct fid_eq **eq, void *context);
-
-extern int gnix_eq_read(struct fid_eq *eq, uint32_t *event, void *buf,
- size_t len, uint64_t flags);
-
-extern int gnix_eq_readerr(struct fid_eq *eq, struct fi_eq_err_entry *buf,
- uint64_t flags);
-
-extern int gnix_eq_write(struct fid_eq *eq, uint32_t event, const void *buf,
- size_t len, uint64_t flags);
-
-extern ssize_t gnix_eq_sread(struct fid_eq *eq, uint32_t *event, void *buf,
- size_t len, int timeout, uint64_t flags);
-
-extern const char *gnix_eq_strerror(struct fid_eq *eq, int prov_errno,
- const void *err_data, char *buf,
- size_t len);
-
-extern int gnix_cq_read(struct fid_cq *cq, void *buf, size_t count);
-
-extern int gnix_cq_readfrom(struct fid_cq *cq, void *buf, size_t count,
- fi_addr_t *src_addr);
-
-extern ssize_t gnix_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf,
- uint64_t flags);
-
-extern int gnix_cq_sread(struct fid_cq *cq, void *buf, size_t count,
- const void *cond, int timeout);
-
-extern int gnix_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count,
- fi_addr_t *src_addr, const void *cond,
- int timeout);
-
-extern int gnix_cq_signal(struct fid_cq *cq);
-
-extern const char *gnix_cq_strerror(struct fid_cq *cq, int prov_errno,
- const void *err_data, char *buf,
- size_t len);
-
-extern int gnix_cntr_read(struct fid_cntr *cntr);
-
-extern int gnix_cntr_readerr(struct fid_cntr *cntr);
-
-extern int gnix_cntr_add(struct fid_cntr *cntr, uint64_t value);
-
-extern int gnix_cntr_set(struct fid_cntr *cntr, uint64_t value);
-
-extern int gnix_cntr_wait(struct fid_cntr *cntr, uint64_t threshold,
- int timeout);
-
-/*******************************************************************************
- * Libfabric API Functions
- ******************************************************************************/
-static inline int
-fi_trywait(struct fid_fabric *fabric, struct fid **fids, int count)
-{
- return gnix_fabric_trywait(fabric, fids, count);
-}
-
-static inline int fi_wait(struct fid_wait *waitset, int timeout)
-{
- return gnix_wait_wait(waitset, timeout);
-}
-
-static inline int fi_poll(struct fid_poll *pollset, void **context, int count)
-{
- return gnix_poll_poll(pollset, context, count);
-}
-
-static inline int fi_poll_add(struct fid_poll *pollset, struct fid *event_fid,
- uint64_t flags)
-{
- return gnix_poll_add(pollset, event_fid, flags);
-}
-
-static inline int fi_poll_del(struct fid_poll *pollset, struct fid *event_fid,
- uint64_t flags)
-{
- return gnix_poll_del(pollset, event_fid, flags);
-}
-
-static inline int fi_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr,
- struct fid_eq **eq, void *context)
-{
- return gnix_eq_open(fabric, attr, eq, context);
-}
-
-static inline ssize_t fi_eq_read(struct fid_eq *eq, uint32_t *event, void *buf,
- size_t len, uint64_t flags)
-{
- return gnix_eq_read(eq, event, buf, len, flags);
-}
-
-static inline ssize_t fi_eq_readerr(struct fid_eq *eq,
- struct fi_eq_err_entry *buf, uint64_t flags)
-{
- return gnix_eq_readerr(eq, buf, flags);
-}
-
-static inline ssize_t fi_eq_write(struct fid_eq *eq, uint32_t event,
- const void *buf, size_t len, uint64_t flags)
-{
- return gnix_eq_write(eq, event, buf, len, flags);
-}
-
-static inline ssize_t fi_eq_sread(struct fid_eq *eq, uint32_t *event, void *buf,
- size_t len, int timeout, uint64_t flags)
-{
- return gnix_eq_sread(eq, event, buf, len, timeout, flags);
-}
-
-static inline const char *fi_eq_strerror(struct fid_eq *eq, int prov_errno,
- const void *err_data, char *buf,
- size_t len)
-{
- return gnix_eq_strerror(eq, prov_errno, err_data, buf, len);
-}
-
-static inline ssize_t fi_cq_read(struct fid_cq *cq, void *buf, size_t count)
-{
- return gnix_cq_read(cq, buf, count);
-}
-
-static inline ssize_t fi_cq_readfrom(struct fid_cq *cq, void *buf, size_t count,
- fi_addr_t *src_addr)
-{
- return gnix_cq_readfrom(cq, buf, count, src_addr);
-}
-
-static inline ssize_t fi_cq_readerr(struct fid_cq *cq,
- struct fi_cq_err_entry *buf, uint64_t flags)
-{
- return gnix_cq_readerr(cq, buf, flags);
-}
-
-static inline ssize_t fi_cq_sread(struct fid_cq *cq, void *buf, size_t count,
- const void *cond, int timeout)
-{
- return gnix_cq_sread(cq, buf, count, cond, timeout);
-}
-
-static inline ssize_t fi_cq_sreadfrom(struct fid_cq *cq, void *buf,
- size_t count, fi_addr_t *src_addr,
- const void *cond, int timeout)
-{
- return gnix_cq_sreadfrom(cq, buf, count, src_addr, cond, timeout);
-}
-
-static inline int fi_cq_signal(struct fid_cq *cq)
-{
- return gnix_cq_signal(cq);
-}
-
-static inline const char *fi_cq_strerror(struct fid_cq *cq, int prov_errno,
- const void *err_data, char *buf,
- size_t len)
-{
- return gnix_cq_strerror(cq, prov_errno, err_data, buf, len);
-}
-
-static inline uint64_t fi_cntr_read(struct fid_cntr *cntr)
-{
- return gnix_cntr_read(cntr);
-}
-
-static inline uint64_t fi_cntr_readerr(struct fid_cntr *cntr)
-{
- return gnix_cntr_readerr(cntr);
-}
-
-static inline int fi_cntr_add(struct fid_cntr *cntr, uint64_t value)
-{
- return gnix_cntr_add(cntr, value);
-}
-
-static inline int fi_cntr_set(struct fid_cntr *cntr, uint64_t value)
-{
- return gnix_cntr_set(cntr, value);
-}
-
-static inline int fi_cntr_wait(struct fid_cntr *cntr, uint64_t threshold,
- int timeout)
-{
- return gnix_cntr_wait(cntr, threshold, timeout);
-}
-
-#endif /* _FI_DIRECT_EQ_H_ */
diff --git a/prov/gni/include/rdma/fi_direct_rma.h b/prov/gni/include/rdma/fi_direct_rma.h
deleted file mode 100644
index 9c6265e..0000000
--- a/prov/gni/include/rdma/fi_direct_rma.h
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All
- * rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _FI_DIRECT_RMA_H_
-#define _FI_DIRECT_RMA_H_
-
-#define FABRIC_DIRECT_RMA 1
-
-/*******************************************************************************
- * GNI API Functions
- ******************************************************************************/
-extern ssize_t gnix_ep_read(struct fid_ep *ep, void *buf, size_t len,
- void *desc, fi_addr_t src_addr, uint64_t addr,
- uint64_t key, void *context);
-
-extern ssize_t gnix_ep_readv(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t src_addr,
- uint64_t addr, uint64_t key, void *context);
-
-extern ssize_t gnix_ep_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
- uint64_t flags);
-
-extern ssize_t gnix_ep_write(struct fid_ep *ep, const void *buf, size_t len,
- void *desc, fi_addr_t dest_addr, uint64_t addr,
- uint64_t key, void *context);
-
-extern ssize_t gnix_ep_writev(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key, void *context);
-
-extern ssize_t gnix_ep_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
- uint64_t flags);
-
-extern ssize_t gnix_ep_rma_inject(struct fid_ep *ep, const void *buf,
- size_t len, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key);
-
-extern ssize_t gnix_ep_writedata(struct fid_ep *ep, const void *buf, size_t len,
- void *desc, uint64_t data, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key, void *context);
-
-extern ssize_t gnix_ep_rma_injectdata(struct fid_ep *ep, const void *buf,
- size_t len, uint64_t data,
- fi_addr_t dest_addr, uint64_t addr,
- uint64_t key);
-
-/*******************************************************************************
- * Libfabric API Functions
- ******************************************************************************/
-static inline ssize_t fi_read(struct fid_ep *ep, void *buf, size_t len,
- void *desc, fi_addr_t src_addr, uint64_t addr,
- uint64_t key, void *context)
-{
- return gnix_ep_read(ep, buf, len, desc, src_addr, addr, key, context);
-}
-
-static inline ssize_t fi_readv(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t src_addr,
- uint64_t addr, uint64_t key, void *context)
-{
- return gnix_ep_readv(ep, iov, desc, count, src_addr, addr, key,
- context);
-}
-
-static inline ssize_t fi_readmsg(struct fid_ep *ep,
- const struct fi_msg_rma *msg, uint64_t flags)
-{
- return gnix_ep_readmsg(ep, msg, flags);
-}
-
-static inline ssize_t fi_write(struct fid_ep *ep, const void *buf, size_t len,
- void *desc, fi_addr_t dest_addr, uint64_t addr,
- uint64_t key, void *context)
-{
- return gnix_ep_write(ep, buf, len, desc, dest_addr, addr, key, context);
-}
-
-static inline ssize_t fi_writev(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key, void *context)
-{
- return gnix_ep_writev(ep, iov, desc, count, dest_addr, addr, key,
- context);
-}
-
-static inline ssize_t fi_writemsg(struct fid_ep *ep,
- const struct fi_msg_rma *msg, uint64_t flags)
-{
- return gnix_ep_writemsg(ep, msg, flags);
-}
-
-static inline ssize_t fi_inject_write(struct fid_ep *ep, const void *buf,
- size_t len, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key)
-{
- return gnix_ep_rma_inject(ep, buf, len, dest_addr, addr, key);
-}
-
-static inline ssize_t fi_writedata(struct fid_ep *ep, const void *buf,
- size_t len, void *desc, uint64_t data,
- fi_addr_t dest_addr, uint64_t addr,
- uint64_t key, void *context)
-{
- return gnix_ep_writedata(ep, buf, len, desc, data, dest_addr, addr, key,
- context);
-}
-
-static inline ssize_t fi_inject_writedata(struct fid_ep *ep, const void *buf,
- size_t len, uint64_t data,
- fi_addr_t dest_addr, uint64_t addr,
- uint64_t key)
-{
- return gnix_ep_rma_injectdata(ep, buf, len, data, dest_addr, addr, key);
-}
-
-#endif /* _FI_DIRECT_RMA_H_ */
diff --git a/prov/gni/include/rdma/fi_direct_tagged.h b/prov/gni/include/rdma/fi_direct_tagged.h
deleted file mode 100644
index b355a47..0000000
--- a/prov/gni/include/rdma/fi_direct_tagged.h
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All
- * rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _FI_DIRECT_TAGGED_H_
-#define _FI_DIRECT_TAGGED_H_
-
-#define FABRIC_DIRECT_TAGGED 1
-
-/*******************************************************************************
- * GNI API Functions
- ******************************************************************************/
-extern ssize_t gnix_ep_trecv(struct fid_ep *ep, void *buf, size_t len,
- void *desc, fi_addr_t src_addr, uint64_t tag,
- uint64_t ignore, void *context);
-
-extern ssize_t gnix_ep_trecvv(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t src_addr,
- uint64_t tag, uint64_t ignore, void *context);
-
-extern ssize_t gnix_ep_trecvmsg(struct fid_ep *ep,
- const struct fi_msg_tagged *msg,
- uint64_t flags);
-
-extern ssize_t gnix_ep_tsend(struct fid_ep *ep, const void *buf, size_t len,
- void *desc, fi_addr_t dest_addr, uint64_t tag,
- void *context);
-
-extern ssize_t gnix_ep_tsendv(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t dest_addr,
- uint64_t tag, void *context);
-
-extern ssize_t gnix_ep_tsendmsg(struct fid_ep *ep,
- const struct fi_msg_tagged *msg,
- uint64_t flags);
-
-extern ssize_t gnix_ep_tinject(struct fid_ep *ep, const void *buf, size_t len,
- fi_addr_t dest_addr, uint64_t tag);
-
-extern ssize_t gnix_ep_tsenddata(struct fid_ep *ep, const void *buf, size_t len,
- void *desc, uint64_t data, fi_addr_t dest_addr,
- uint64_t tag, void *context);
-
-extern ssize_t gnix_ep_tinjectdata(struct fid_ep *ep, const void *buf,
- size_t len, uint64_t data,
- fi_addr_t dest_addr, uint64_t tag);
-
-/*******************************************************************************
- * Libfabric API Functions
- ******************************************************************************/
-static inline ssize_t fi_trecv(struct fid_ep *ep, void *buf, size_t len,
- void *desc, fi_addr_t src_addr, uint64_t tag,
- uint64_t ignore, void *context)
-{
- return gnix_ep_trecv(ep, buf, len, desc, src_addr, tag, ignore,
- context);
-}
-
-static inline ssize_t fi_trecvv(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t src_addr,
- uint64_t tag, uint64_t ignore, void *context)
-{
- return gnix_ep_trecvv(ep, iov, desc, count, src_addr, tag, ignore,
- context);
-}
-
-static inline ssize_t
-fi_trecvmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg, uint64_t flags)
-{
- return gnix_ep_trecvmsg(ep, msg, flags);
-}
-
-static inline ssize_t fi_tsend(struct fid_ep *ep, const void *buf, size_t len,
- void *desc, fi_addr_t dest_addr, uint64_t tag,
- void *context)
-{
- return gnix_ep_tsend(ep, buf, len, desc, dest_addr, tag, context);
-}
-
-static inline ssize_t fi_tsendv(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t dest_addr,
- uint64_t tag, void *context)
-{
- return gnix_ep_tsendv(ep, iov, desc, count, dest_addr, tag, context);
-}
-
-static inline ssize_t
-fi_tsendmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg, uint64_t flags)
-{
- return gnix_ep_tsendmsg(ep, msg, flags);
-}
-
-static inline ssize_t fi_tinject(struct fid_ep *ep, const void *buf, size_t len,
- fi_addr_t dest_addr, uint64_t tag)
-{
- return gnix_ep_tinject(ep, buf, len, dest_addr, tag);
-}
-
-static inline ssize_t fi_tsenddata(struct fid_ep *ep, const void *buf,
- size_t len, void *desc, uint64_t data,
- fi_addr_t dest_addr, uint64_t tag,
- void *context)
-{
- return gnix_ep_tsenddata(ep, buf, len, desc, data, dest_addr, tag,
- context);
-}
-
-static inline ssize_t fi_tinjectdata(struct fid_ep *ep, const void *buf,
- size_t len, uint64_t data,
- fi_addr_t dest_addr, uint64_t tag)
-{
- return gnix_ep_tinjectdata(ep, buf, len, data, dest_addr, tag);
-}
-
-#endif /* _FI_DIRECT_TAGGED_H_ */
diff --git a/prov/gni/include/rdma/fi_direct_trigger.h b/prov/gni/include/rdma/fi_direct_trigger.h
deleted file mode 100644
index 678cabe..0000000
--- a/prov/gni/include/rdma/fi_direct_trigger.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All
- * rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _FI_DIRECT_TRIGGER_H_
-#define _FI_DIRECT_TRIGGER_H_
-
-#define FABRIC_DIRECT_TRIGGER 1
-
-/* Size must match struct fi_context */
-struct fi_triggered_context {
- enum fi_trigger_event event_type;
- union {
- struct fi_trigger_threshold threshold;
- void *internal[3];
- } trigger;
-};
-
-#endif /* _FI_DIRECT_TRIGGER_H_ */
diff --git a/prov/gni/provider_FABRIC_1.0.map b/prov/gni/provider_FABRIC_1.0.map
deleted file mode 100644
index 14d9a2f..0000000
--- a/prov/gni/provider_FABRIC_1.0.map
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * used for exporting GNI provider
- * symbols when building to support FI_DIRECT
- */
- gnix_av_straddr;
- gnix_cq_strerror;
- gnix_eq_strerror;
- gnix_accept;
- gnix_av_bind;
- gnix_av_insertsvc;
- gnix_av_insertsym;
- gnix_av_open;
- gnix_cntr_open;
- gnix_cntr_wait;
- gnix_connect;
- gnix_cq_open;
- gnix_domain_bind;
- gnix_domain_open;
- gnix_ep_atomic_compwrite;
- gnix_ep_atomic_compwritemsg;
- gnix_ep_atomic_compwritev;
- gnix_ep_atomic_inject;
- gnix_ep_atomic_readwrite;
- gnix_ep_atomic_readwritemsg;
- gnix_ep_atomic_readwritev;
- gnix_ep_atomic_write;
- gnix_ep_atomic_writemsg;
- gnix_ep_atomic_writev;
- gnix_ep_msg_injectdata;
- gnix_ep_open;
- gnix_ep_send;
- gnix_ep_senddata;
- gnix_ep_tinjectdata;
- gnix_ep_tsenddata;
- gnix_eq_open;
- gnix_eq_read;
- gnix_eq_sread;
- gnix_fabric_trywait;
- gnix_getpeer;
- gnix_listen;
- gnix_mr_bind;
- gnix_mr_reg;
- gnix_passive_ep_open;
- gnix_pep_bind;
- gnix_poll_add;
- gnix_poll_del;
- gnix_poll_open;
- gnix_poll_poll;
- gnix_reject;
- gnix_scalable_ep_bind;
- gnix_scalable_ep_open;
- gnix_setname;
- gnix_shutdown;
- gnix_srx_context;
- gnix_stx_open;
- gnix_wait_open;
- gnix_wait_wait;
- gnix_av_insert;
- gnix_av_lookup;
- gnix_av_remove;
- gnix_cntr_add;
- gnix_cntr_set;
- gnix_cq_signal;
- gnix_ep_write;
- gnix_ep_rma_inject;
- gnix_ep_rma_injectdata;
- gnix_ep_writedata;
- gnix_ep_readmsg;
- gnix_ep_readv;
- gnix_ep_writedata;
- gnix_ep_writemsg;
- gnix_ep_writev;
- gnix_ep_atomic_valid;
- gnix_ep_bind;
- gnix_ep_cmp_atomic_valid;
- gnix_ep_control;
- gnix_ep_fetch_atomic_valid;
- gnix_ep_getopt;
- gnix_ep_setopt;
- gnix_eq_close;
- gnix_eq_control;
- gnix_getname;
- gnix_rx_context;
- gnix_tx_context;
- gnix_cq_read;
- gnix_cq_readerr;
- gnix_cq_readfrom;
- gnix_cq_sread;
- gnix_cq_sreadfrom;
- gnix_ep_cancel;
- gnix_ep_msg_inject;
- gnix_ep_read;
- gnix_ep_readv;
- gnix_ep_recv;
- gnix_ep_recvmsg;
- gnix_ep_recvv;
- gnix_ep_rx_size_left;
- gnix_ep_sendmsg;
- gnix_ep_sendv;
- gnix_ep_tinject;
- gnix_ep_trecv;
- gnix_ep_trecvmsg;
- gnix_ep_trecvv;
- gnix_ep_tsend;
- gnix_ep_tsendmsg;
- gnix_ep_tsendv;
- gnix_ep_tx_size_left;
- gnix_eq_readerr;
- gnix_eq_write;
- gnix_cntr_read;
- gnix_cntr_readerr;
diff --git a/prov/gni/src/gnix_atomic.c b/prov/gni/src/gnix_atomic.c
index 4c9969a..d60e76a 100644
--- a/prov/gni/src/gnix_atomic.c
+++ b/prov/gni/src/gnix_atomic.c
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -60,11 +60,15 @@ static int __gnix_amo_send_err(struct gnix_fid_ep *ep,
}
}
- if ((req->type == GNIX_FAB_RQ_AMO) &&
+ if (((req->type == GNIX_FAB_RQ_AMO) ||
+ (req->type == GNIX_FAB_RQ_NAMO_AX) ||
+ (req->type == GNIX_FAB_RQ_NAMO_AX_S)) &&
ep->write_cntr) {
cntr = ep->write_cntr;
} else if ((req->type == GNIX_FAB_RQ_FAMO ||
- req->type == GNIX_FAB_RQ_CAMO) &&
+ req->type == GNIX_FAB_RQ_CAMO ||
+ req->type == GNIX_FAB_RQ_NAMO_FAX ||
+ req->type == GNIX_FAB_RQ_NAMO_FAX_S) &&
ep->read_cntr) {
cntr = ep->read_cntr;
}
@@ -95,11 +99,15 @@ static int __gnix_amo_send_completion(struct gnix_fid_ep *ep,
}
}
- if ((req->type == GNIX_FAB_RQ_AMO) &&
+ if ((req->type == GNIX_FAB_RQ_AMO ||
+ req->type == GNIX_FAB_RQ_NAMO_AX ||
+ req->type == GNIX_FAB_RQ_NAMO_AX_S) &&
ep->write_cntr) {
cntr = ep->write_cntr;
} else if ((req->type == GNIX_FAB_RQ_FAMO ||
- req->type == GNIX_FAB_RQ_CAMO) &&
+ req->type == GNIX_FAB_RQ_CAMO ||
+ req->type == GNIX_FAB_RQ_NAMO_FAX ||
+ req->type == GNIX_FAB_RQ_NAMO_FAX_S) &&
ep->read_cntr) {
cntr = ep->read_cntr;
}
@@ -114,8 +122,7 @@ static int __gnix_amo_send_completion(struct gnix_fid_ep *ep,
return FI_SUCCESS;
}
-static void __gnix_amo_fr_complete(struct gnix_fab_req *req,
- struct gnix_tx_descriptor *txd)
+static void __gnix_amo_fr_complete(struct gnix_fab_req *req)
{
if (req->flags & FI_LOCAL_MR) {
GNIX_INFO(FI_LOG_EP_DATA, "freeing auto-reg MR: %p\n",
@@ -124,7 +131,6 @@ static void __gnix_amo_fr_complete(struct gnix_fab_req *req,
}
atomic_dec(&req->vc->outstanding_tx_reqs);
- _gnix_nic_tx_free(req->vc->ep->nic, txd);
/* Schedule VC TX queue in case the VC is 'fenced'. */
_gnix_vc_tx_schedule(req->vc);
@@ -132,9 +138,8 @@ static void __gnix_amo_fr_complete(struct gnix_fab_req *req,
_gnix_fr_free(req->vc->ep, req);
}
-static int __gnix_amo_post_err(struct gnix_tx_descriptor *txd, int error)
+static int __gnix_amo_post_err(struct gnix_fab_req *req, int error)
{
- struct gnix_fab_req *req = txd->req;
int rc;
rc = __gnix_amo_send_err(req->vc->ep, req, error);
@@ -143,10 +148,127 @@ static int __gnix_amo_post_err(struct gnix_tx_descriptor *txd, int error)
"__gnix_amo_send_err() failed: %d\n",
rc);
- __gnix_amo_fr_complete(req, txd);
+ __gnix_amo_fr_complete(req);
return FI_SUCCESS;
}
+/* SMSG callback for AMO remote counter control message. */
+int __smsg_amo_cntr(void *data, void *msg)
+{
+ int ret = FI_SUCCESS;
+ struct gnix_vc *vc = (struct gnix_vc *)data;
+ struct gnix_smsg_amo_cntr_hdr *hdr =
+ (struct gnix_smsg_amo_cntr_hdr *)msg;
+ struct gnix_fid_ep *ep = vc->ep;
+ gni_return_t status;
+
+ if (hdr->flags & FI_REMOTE_WRITE && ep->rwrite_cntr) {
+ ret = _gnix_cntr_inc(ep->rwrite_cntr);
+ if (ret != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_cntr_inc() failed: %d\n",
+ ret);
+ }
+
+ if (hdr->flags & FI_REMOTE_READ && ep->rread_cntr) {
+ ret = _gnix_cntr_inc(ep->rread_cntr);
+ if (ret != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_cntr_inc() failed: %d\n",
+ ret);
+ }
+
+ status = GNI_SmsgRelease(vc->gni_ep);
+ if (unlikely(status != GNI_RC_SUCCESS)) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "GNI_SmsgRelease returned %s\n",
+ gni_err_str[status]);
+ ret = gnixu_to_fi_errno(status);
+ }
+
+ return ret;
+}
+
+static int __gnix_amo_txd_cntr_complete(void *arg, gni_return_t tx_status)
+{
+ struct gnix_tx_descriptor *txd = (struct gnix_tx_descriptor *)arg;
+ struct gnix_fab_req *req = txd->req;
+ int rc;
+
+ _gnix_nic_tx_free(req->gnix_ep->nic, txd);
+
+ if (tx_status != GNI_RC_SUCCESS)
+ return __gnix_amo_post_err(req, FI_ECANCELED);
+
+ /* Successful data delivery. Generate local completion. */
+ rc = __gnix_amo_send_completion(req->vc->ep, req);
+ if (rc != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "__gnix_amo_send_completion() failed: %d\n",
+ rc);
+
+ __gnix_amo_fr_complete(req);
+
+ return FI_SUCCESS;
+}
+
+static int __gnix_amo_send_cntr_req(void *arg)
+{
+ struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
+ struct gnix_fid_ep *ep = req->gnix_ep;
+ struct gnix_nic *nic = ep->nic;
+ struct gnix_tx_descriptor *txd;
+ gni_return_t status;
+ int rc;
+ int inject_err = _gnix_req_inject_err(req);
+
+ rc = _gnix_nic_tx_alloc(nic, &txd);
+ if (rc) {
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "_gnix_nic_tx_alloc() failed: %d\n",
+ rc);
+ return -FI_ENOSPC;
+ }
+
+ txd->req = req;
+ txd->completer_fn = __gnix_amo_txd_cntr_complete;
+
+ if (req->type == GNIX_FAB_RQ_AMO) {
+ txd->amo_cntr_hdr.flags = FI_REMOTE_WRITE;
+ } else {
+ txd->amo_cntr_hdr.flags = FI_REMOTE_READ;
+ }
+
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
+ if (inject_err) {
+ _gnix_nic_txd_err_inject(nic, txd);
+ status = GNI_RC_SUCCESS;
+ } else {
+ status = GNI_SmsgSendWTag(req->vc->gni_ep,
+ &txd->amo_cntr_hdr,
+ sizeof(txd->amo_cntr_hdr),
+ NULL, 0, txd->id,
+ GNIX_SMSG_T_AMO_CNTR);
+ }
+ COND_RELEASE(nic->requires_lock, &nic->lock);
+
+ if (status == GNI_RC_NOT_DONE) {
+ _gnix_nic_tx_free(nic, txd);
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "GNI_SmsgSendWTag returned %s\n",
+ gni_err_str[status]);
+ } else if (status != GNI_RC_SUCCESS) {
+ _gnix_nic_tx_free(nic, txd);
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "GNI_SmsgSendWTag returned %s\n",
+ gni_err_str[status]);
+ } else {
+ GNIX_INFO(FI_LOG_EP_DATA, "Sent RMA CQ data, req: %p\n", req);
+ }
+
+ return gnixu_to_fi_errno(status);
+}
+
static int __gnix_amo_txd_complete(void *arg, gni_return_t tx_status)
{
struct gnix_tx_descriptor *txd = (struct gnix_tx_descriptor *)arg;
@@ -155,8 +277,10 @@ static int __gnix_amo_txd_complete(void *arg, gni_return_t tx_status)
uint32_t read_data32;
uint64_t read_data64;
+ _gnix_nic_tx_free(req->vc->ep->nic, txd);
+
if (tx_status != GNI_RC_SUCCESS) {
- return __gnix_amo_post_err(txd, FI_ECANCELED);
+ return __gnix_amo_post_err(req, FI_ECANCELED);
}
/* FI_ATOMIC_READ data is delivered to operand buffer in addition to
@@ -178,14 +302,20 @@ static int __gnix_amo_txd_complete(void *arg, gni_return_t tx_status)
}
}
- /* complete request */
- rc = __gnix_amo_send_completion(req->vc->ep, req);
- if (rc != FI_SUCCESS)
- GNIX_WARN(FI_LOG_EP_DATA,
- "__gnix_amo_send_completion() failed: %d\n",
- rc);
+ if (req->vc->peer_caps & FI_RMA_EVENT) {
+ /* control message needed for a counter event. */
+ req->work_fn = __gnix_amo_send_cntr_req;
+ _gnix_vc_queue_work_req(req);
+ } else {
+ /* complete request */
+ rc = __gnix_amo_send_completion(req->vc->ep, req);
+ if (rc != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "__gnix_amo_send_completion() failed: %d\n",
+ rc);
- __gnix_amo_fr_complete(req, txd);
+ __gnix_amo_fr_complete(req);
+ }
return FI_SUCCESS;
}
@@ -246,7 +376,11 @@ static int __gnix_cmp_amo_cmds[FI_ATOMIC_OP_LAST][FI_DATATYPE_LAST] = {
int _gnix_atomic_cmd(enum fi_datatype dt, enum fi_op op,
enum gnix_fab_req_type fr_type)
{
- if (dt >= FI_DATATYPE_LAST || op >= FI_ATOMIC_OP_LAST) {
+ if (!((fr_type == GNIX_FAB_RQ_NAMO_AX) ||
+ (fr_type == GNIX_FAB_RQ_NAMO_FAX) ||
+ (fr_type == GNIX_FAB_RQ_NAMO_AX_S) ||
+ (fr_type == GNIX_FAB_RQ_NAMO_FAX_S)) &&
+ (dt >= FI_DATATYPE_LAST || op >= FI_ATOMIC_OP_LAST)) {
return -FI_ENOENT;
}
@@ -257,6 +391,14 @@ int _gnix_atomic_cmd(enum fi_datatype dt, enum fi_op op,
return __gnix_fetch_amo_cmds[op][dt] ?: -FI_ENOENT;
case GNIX_FAB_RQ_CAMO:
return __gnix_cmp_amo_cmds[op][dt] ?: -FI_ENOENT;
+ case GNIX_FAB_RQ_NAMO_AX:
+ return GNI_FMA_ATOMIC2_AX;
+ case GNIX_FAB_RQ_NAMO_AX_S:
+ return GNI_FMA_ATOMIC2_AX_S;
+ case GNIX_FAB_RQ_NAMO_FAX:
+ return GNI_FMA_ATOMIC2_FAX;
+ case GNIX_FAB_RQ_NAMO_FAX_S:
+ return GNI_FMA_ATOMIC2_FAX_S;
default:
break;
}
@@ -276,16 +418,6 @@ int _gnix_amo_post_req(void *data)
int rc;
int inject_err = _gnix_req_inject_err(fab_req);
- rc = _gnix_nic_tx_alloc(nic, &txd);
- if (rc) {
- GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n",
- rc);
- return -FI_ENOSPC;
- }
-
- txd->completer_fn = __gnix_amo_txd_complete;
- txd->req = fab_req;
-
if (!gnix_ops_allowed(ep, fab_req->vc->peer_caps, fab_req->flags)) {
GNIX_DEBUG(FI_LOG_EP_DATA, "flags:0x%llx, %s\n", fab_req->flags,
fi_tostr(&fab_req->flags, FI_TYPE_OP_FLAGS));
@@ -294,9 +426,24 @@ int _gnix_amo_post_req(void *data)
GNIX_DEBUG(FI_LOG_EP_DATA, "peer_caps:0x%llx, %s\n",
fab_req->vc->peer_caps,
fi_tostr(&fab_req->vc->peer_caps, FI_TYPE_OP_FLAGS));
- return __gnix_amo_post_err(txd, FI_EOPNOTSUPP);
+
+ rc = __gnix_amo_post_err(fab_req, FI_EOPNOTSUPP);
+ if (rc != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "__gnix_amo_post_err() failed: %d\n", rc);
+ return -FI_ECANCELED;
+ }
+
+ rc = _gnix_nic_tx_alloc(nic, &txd);
+ if (rc) {
+ GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n",
+ rc);
+ return -FI_ENOSPC;
}
+ txd->completer_fn = __gnix_amo_txd_complete;
+ txd->req = fab_req;
+
/* Mem handle CRC is not validated during FMA operations. Skip this
* costly calculation. */
_gnix_convert_key_to_mhdl_no_crc(
@@ -323,9 +470,13 @@ int _gnix_amo_post_req(void *data)
txd->gni_desc.first_operand = fab_req->amo.first_operand;
txd->gni_desc.second_operand = fab_req->amo.second_operand;
+ GNIX_DEBUG(FI_LOG_EP_DATA, "fo:%016lx so:%016lx\n",
+ txd->gni_desc.first_operand, txd->gni_desc.second_operand);
+ GNIX_DEBUG(FI_LOG_EP_DATA, "amo_cmd:%x\n",
+ txd->gni_desc.amo_cmd);
GNIX_LOG_DUMP_TXD(txd);
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
if (unlikely(inject_err)) {
_gnix_nic_txd_err_inject(nic, txd);
@@ -334,7 +485,7 @@ int _gnix_amo_post_req(void *data)
status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc);
}
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
@@ -366,15 +517,34 @@ ssize_t _gnix_atomic(struct gnix_fid_ep *ep,
void *loc_addr = NULL;
int dt_len, dt_align;
+ if (!(flags & FI_INJECT) && !ep->send_cq &&
+ (((fr_type == GNIX_FAB_RQ_AMO ||
+ fr_type == GNIX_FAB_RQ_NAMO_AX ||
+ fr_type == GNIX_FAB_RQ_NAMO_AX_S) &&
+ !ep->write_cntr) ||
+ ((fr_type == GNIX_FAB_RQ_FAMO ||
+ fr_type == GNIX_FAB_RQ_CAMO ||
+ fr_type == GNIX_FAB_RQ_NAMO_FAX ||
+ fr_type == GNIX_FAB_RQ_NAMO_FAX_S) &&
+ !ep->read_cntr))) {
+ return -FI_ENOCQ;
+ }
+
+
if (!ep || !msg || !msg->msg_iov ||
!msg->msg_iov[0].addr ||
msg->msg_iov[0].count != 1 ||
- msg->iov_count != 1 ||
+ msg->iov_count != GNIX_MAX_ATOMIC_IOV_LIMIT ||
!msg->rma_iov || !msg->rma_iov[0].addr)
return -FI_EINVAL;
- if (!ep->send_cq && !(flags & FI_INJECT)) {
- return -FI_ENOCQ;
+ if (flags & FI_TRIGGER) {
+ struct fi_triggered_context *trigger_context =
+ (struct fi_triggered_context *)msg->context;
+ if ((trigger_context->event_type != FI_TRIGGER_THRESHOLD) ||
+ (flags & FI_INJECT)) {
+ return -FI_EINVAL;
+ }
}
if (fr_type == GNIX_FAB_RQ_CAMO) {
@@ -396,7 +566,10 @@ ssize_t _gnix_atomic(struct gnix_fid_ep *ep,
}
/* need a memory descriptor for all fetching and comparison AMOs */
- if (fr_type == GNIX_FAB_RQ_FAMO || fr_type == GNIX_FAB_RQ_CAMO) {
+ if (fr_type == GNIX_FAB_RQ_FAMO ||
+ fr_type == GNIX_FAB_RQ_CAMO ||
+ fr_type == GNIX_FAB_RQ_NAMO_FAX ||
+ fr_type == GNIX_FAB_RQ_NAMO_FAX_S) {
if (!resultv || !resultv[0].addr || result_count != 1)
return -FI_EINVAL;
@@ -457,7 +630,15 @@ ssize_t _gnix_atomic(struct gnix_fid_ep *ep,
req->amo.loc_md = (void *)md;
req->amo.loc_addr = (uint64_t)loc_addr;
- if (msg->op == FI_ATOMIC_READ) {
+ if ((fr_type == GNIX_FAB_RQ_NAMO_AX) ||
+ (fr_type == GNIX_FAB_RQ_NAMO_FAX) ||
+ (fr_type == GNIX_FAB_RQ_NAMO_AX_S) ||
+ (fr_type == GNIX_FAB_RQ_NAMO_FAX_S)) {
+ req->amo.first_operand =
+ *(uint64_t *)msg->msg_iov[0].addr;
+ req->amo.second_operand =
+ *((uint64_t *)(msg->msg_iov[0].addr) + 1);
+ } else if (msg->op == FI_ATOMIC_READ) {
/* Atomic reads are the only AMO which write to the operand
* buffer. It's assumed that this is in addition to writing
* fetched data to the result buffer. Make the NIC write to
diff --git a/prov/gni/src/gnix_av.c b/prov/gni/src/gnix_av.c
index 58a711f..570a6a0 100644
--- a/prov/gni/src/gnix_av.c
+++ b/prov/gni/src/gnix_av.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -66,7 +66,7 @@ static struct fi_ops gnix_fi_av_ops;
* Helper functions.
******************************************************************************/
/*
- * TODO: Check for named AV creation and check RX CTX bits.
+ * TODO: Check RX CTX bits.
*/
static int gnix_verify_av_attr(struct fi_av_attr *attr)
{
@@ -75,139 +75,17 @@ static int gnix_verify_av_attr(struct fi_av_attr *attr)
switch (attr->type) {
case FI_AV_TABLE:
case FI_AV_MAP:
+ case FI_AV_UNSPEC:
break;
default:
ret = -FI_EINVAL;
break;
}
- return ret;
-}
-
-/*
- * Given a string that contains the string representation of gnix_ep_name.
- * Parse this string and return the values in a gnix_ep_name structure.
- */
-int gnix_av_straddr_to_ep_name(char *buf,
- struct gnix_ep_name *gnix_ep)
-{
- uint32_t cdm_id;
- uint32_t cm_nic_cdm_id;
- uint32_t cookie;
- uint32_t device_addr;
- char *int_buf;
- uint32_t name_type;
- int ret = FI_SUCCESS;
- char *save_ptr = NULL;
- char temp_buf[GNIX_AV_MAX_STR_ADDR_LEN];
- uint32_t version;
-
- if ((buf == NULL) || (gnix_ep == NULL)) {
- ret = -FI_EINVAL;
- goto err;
- }
-
- memset(temp_buf, 0, GNIX_AV_MAX_STR_ADDR_LEN);
- strncpy(temp_buf, buf, (GNIX_AV_MAX_STR_ADDR_LEN - 1));
-
- errno = 0;
- gnix_ep->gnix_addr.device_addr = 0;
- gnix_ep->gnix_addr.cdm_id = 0;
- gnix_ep->name_type = 0;
- gnix_ep->cm_nic_cdm_id = 0;
- gnix_ep->cookie = 0;
-
- /* extract the version component */
- int_buf = strtok_r(temp_buf, ":", &save_ptr);
- if (int_buf == NULL) {
- ret = -FI_EINVAL;
- goto err;
- }
-
- version = (uint32_t) strtol(int_buf, NULL, 10);
- if (errno != 0) {
- ret = -FI_EINVAL;
- goto err;
- }
-
- if ((version == 0) || (version > GNIX_AV_STR_ADDR_VERSION)) {
- ret = -FI_EINVAL;
- goto err;
- }
-
- /* extract the device_addr component */
- int_buf = strtok_r(NULL, ":", &save_ptr);
- if (int_buf == NULL) {
- ret = -FI_EINVAL;
- goto err;
- }
-
- device_addr = (uint32_t) strtol(int_buf, NULL, 16);
- if (errno != 0) {
- ret = -FI_EINVAL;
- goto err;
- }
-
- /* extract the cdm_id component */
- int_buf = strtok_r(NULL, ":", &save_ptr);
- if (int_buf == NULL) {
- ret = -FI_EINVAL;
- goto err;
- }
-
- cdm_id = (uint32_t) strtol(int_buf, NULL, 16);
- if (errno != 0) {
- ret = -FI_EINVAL;
- goto err;
- }
-
- /* extract the name_type component */
- int_buf = strtok_r(NULL, ":", &save_ptr);
- if (int_buf == NULL) {
- ret = -FI_EINVAL;
- goto err;
- }
-
- name_type = (uint32_t) strtol(int_buf, NULL, 10);
- if (errno != 0) {
- ret = -FI_EINVAL;
- goto err;
+ if (attr->name != NULL) {
+ ret = -FI_ENOSYS;
}
- /* extract the cm_nic_cdm_id component */
- int_buf = strtok_r(NULL, ":", &save_ptr);
- if (int_buf == NULL) {
- ret = -FI_EINVAL;
- goto err;
- }
-
- cm_nic_cdm_id = (uint32_t) strtol(int_buf, NULL, 16);
- if (errno != 0) {
- ret = -FI_EINVAL;
- goto err;
- }
-
- /* extract the cookie component */
- int_buf = strtok_r(NULL, ":", &save_ptr);
- if (int_buf == NULL) {
- ret = -FI_EINVAL;
- goto err;
- }
-
- cookie = (uint32_t) strtol(int_buf, NULL, 16);
- if (errno != 0) {
- ret = -FI_EINVAL;
- goto err;
- }
-
- /* check to see if additional component are specified */
- gnix_ep->gnix_addr.device_addr = device_addr;
- gnix_ep->gnix_addr.cdm_id = cdm_id;
- gnix_ep->name_type = name_type;
- gnix_ep->cm_nic_cdm_id = cm_nic_cdm_id;
- gnix_ep->cookie = cookie;
-
-err:
return ret;
}
@@ -271,51 +149,34 @@ static int gnix_check_capacity(struct gnix_fid_av *av, size_t count)
/*******************************************************************************
* FI_AV_TABLE specific implementations.
******************************************************************************/
-static int table_insert(struct gnix_fid_av *int_av, const void *addr,
+static int table_insert(struct gnix_fid_av *av_priv, const void *addr,
size_t count, fi_addr_t *fi_addr, uint64_t flags,
void *context)
{
- struct gnix_ep_name temp;
- char *temp_addr = (char *) addr;
+ struct gnix_ep_name *temp = NULL;
int ret = count;
- int rc;
size_t index;
size_t i;
- if (gnix_check_capacity(int_av, count)) {
+ if (gnix_check_capacity(av_priv, count)) {
ret = -FI_ENOMEM;
goto err;
}
- assert(int_av->table);
- for (index = int_av->count, i = 0; i < count; index++, i++) {
- temp.gnix_addr.device_addr = 0;
- temp.gnix_addr.cdm_id = 0;
- temp.name_type = 0;
- temp.cm_nic_cdm_id = 0;
- temp.cookie = 0;
-
- rc = gnix_av_straddr_to_ep_name(temp_addr, &temp);
- if (rc != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_AV,
- "table_insert: gnix_av_straddr_to_ep_name failed %d\n",
- ret);
- return ret;
- }
-
- int_av->table[index].gnix_addr = temp.gnix_addr;
- int_av->valid_entry_vec[index] = 1;
- int_av->table[index].name_type = temp.name_type;
- int_av->table[index].cookie = temp.cookie;
- int_av->table[index].cm_nic_cdm_id =
- temp.cm_nic_cdm_id;
+ assert(av_priv->table);
+ for (index = av_priv->count, i = 0; i < count; index++, i++) {
+ temp = &((struct gnix_ep_name *)addr)[i];
+ av_priv->table[index].gnix_addr = temp->gnix_addr;
+ av_priv->valid_entry_vec[index] = 1;
+ av_priv->table[index].name_type = temp->name_type;
+ av_priv->table[index].cookie = temp->cookie;
+ av_priv->table[index].cm_nic_cdm_id =
+ temp->cm_nic_cdm_id;
if (fi_addr)
fi_addr[i] = index;
-
- temp_addr += GNIX_AV_MAX_STR_ADDR_LEN;
}
- int_av->count += count;
+ av_priv->count += count;
err:
return ret;
@@ -326,7 +187,7 @@ err:
* If any of the given address fail to be removed (are already marked removed)
* then the return value will be -FI_EINVAL.
*/
-static int table_remove(struct gnix_fid_av *int_av, fi_addr_t *fi_addr,
+static int table_remove(struct gnix_fid_av *av_priv, fi_addr_t *fi_addr,
size_t count, uint64_t flags)
{
int ret = FI_SUCCESS;
@@ -335,12 +196,12 @@ static int table_remove(struct gnix_fid_av *int_av, fi_addr_t *fi_addr,
for (i = 0; i < count; i++) {
index = (size_t) fi_addr[i];
- if (index < int_av->count) {
- if (int_av->valid_entry_vec[index] == 0) {
+ if (index < av_priv->count) {
+ if (av_priv->valid_entry_vec[index] == 0) {
ret = -FI_EINVAL;
break;
} else {
- int_av->valid_entry_vec[index] = 0;
+ av_priv->valid_entry_vec[index] = 0;
}
} else {
ret = -FI_EINVAL;
@@ -354,39 +215,39 @@ static int table_remove(struct gnix_fid_av *int_av, fi_addr_t *fi_addr,
/*
* table_lookup(): Translate fi_addr_t to struct gnix_address.
*/
-static int table_lookup(struct gnix_fid_av *int_av, fi_addr_t fi_addr,
- struct gnix_av_addr_entry **entry_ptr)
+static int table_lookup(struct gnix_fid_av *av_priv, fi_addr_t fi_addr,
+ struct gnix_av_addr_entry *entry_ptr)
{
size_t index;
struct gnix_av_addr_entry *entry = NULL;
index = (size_t)fi_addr;
- if (index >= int_av->count)
+ if (index >= av_priv->count)
return -FI_EINVAL;
- assert(int_av->table);
- entry = &int_av->table[index];
+ assert(av_priv->table);
+ entry = &av_priv->table[index];
if (entry == NULL)
return -FI_EINVAL;
- if (int_av->valid_entry_vec[index] == 0)
+ if (av_priv->valid_entry_vec[index] == 0)
return -FI_EINVAL;
- *entry_ptr = entry;
+ memcpy(entry_ptr, entry, sizeof(*entry));
return FI_SUCCESS;
}
-static int table_reverse_lookup(struct gnix_fid_av *int_av,
+static int table_reverse_lookup(struct gnix_fid_av *av_priv,
struct gnix_address gnix_addr,
fi_addr_t *fi_addr)
{
struct gnix_av_addr_entry *entry;
int i;
- for (i = 0; i < int_av->count; i++) {
- entry = &int_av->table[i];
+ for (i = 0; i < av_priv->count; i++) {
+ entry = &av_priv->table[i];
if (GNIX_ADDR_EQUAL(entry->gnix_addr, gnix_addr)) {
*fi_addr = i;
return FI_SUCCESS;
@@ -400,19 +261,18 @@ static int table_reverse_lookup(struct gnix_fid_av *int_av,
* FI_AV_MAP specific implementations.
******************************************************************************/
-static int map_insert(struct gnix_fid_av *int_av, const void *addr,
+static int map_insert(struct gnix_fid_av *av_priv, const void *addr,
size_t count, fi_addr_t *fi_addr, uint64_t flags,
void *context)
{
int ret;
- struct gnix_ep_name temp;
- char *temp_addr = (char *) addr;
+ struct gnix_ep_name *temp = NULL;
struct gnix_av_addr_entry *the_entry;
gnix_ht_key_t key;
size_t i;
struct gnix_av_block *blk = NULL;
- assert(int_av->map_ht != NULL);
+ assert(av_priv->map_ht != NULL);
if (count == 0)
return 0;
@@ -427,32 +287,19 @@ static int map_insert(struct gnix_fid_av *int_av, const void *addr,
return -FI_ENOMEM;
}
- slist_insert_tail(&blk->slist, &int_av->block_list);
+ slist_insert_tail(&blk->slist, &av_priv->block_list);
for (i = 0; i < count; i++) {
- temp.gnix_addr.device_addr = 0;
- temp.gnix_addr.cdm_id = 0;
- temp.name_type = 0;
- temp.cm_nic_cdm_id = 0;
- temp.cookie = 0;
-
- ret = gnix_av_straddr_to_ep_name(temp_addr, &temp);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_AV,
- "map_insert: gnix_av_straddr_to_ep_name failed %d\n",
- ret);
- return ret;
- }
-
- ((struct gnix_address *)fi_addr)[i] = temp.gnix_addr;
+ temp = &((struct gnix_ep_name *)addr)[i];
+ ((struct gnix_address *)fi_addr)[i] = temp->gnix_addr;
the_entry = &blk->base[i];
- memcpy(&the_entry->gnix_addr, &temp.gnix_addr,
+ memcpy(&the_entry->gnix_addr, &temp->gnix_addr,
sizeof(struct gnix_address));
- the_entry->name_type = temp.name_type;
- the_entry->cm_nic_cdm_id = temp.cm_nic_cdm_id;
- the_entry->cookie = temp.cookie;
- memcpy(&key, &temp.gnix_addr, sizeof(gnix_ht_key_t));
- ret = _gnix_ht_insert(int_av->map_ht,
+ the_entry->name_type = temp->name_type;
+ the_entry->cm_nic_cdm_id = temp->cm_nic_cdm_id;
+ the_entry->cookie = temp->cookie;
+ memcpy(&key, &temp->gnix_addr, sizeof(gnix_ht_key_t));
+ ret = _gnix_ht_insert(av_priv->map_ht,
key,
the_entry);
/*
@@ -461,12 +308,11 @@ static int map_insert(struct gnix_fid_av *int_av, const void *addr,
*/
if ((ret != FI_SUCCESS) && (ret != -FI_ENOSPC)) {
GNIX_WARN(FI_LOG_AV,
- "map_insert: _gnix_ht_insert failed %d\n",
+ "_gnix_ht_insert failed %d\n",
ret);
return ret;
}
- temp_addr += GNIX_AV_MAX_STR_ADDR_LEN;
}
return count;
@@ -477,7 +323,7 @@ static int map_insert(struct gnix_fid_av *int_av, const void *addr,
* or as an alternative, have a free list for slabs so they can be
* reused if new fi_av_insert operations are performed.
*/
-static int map_remove(struct gnix_fid_av *int_av, fi_addr_t *fi_addr,
+static int map_remove(struct gnix_fid_av *av_priv, fi_addr_t *fi_addr,
size_t count, uint64_t flags)
{
int i,ret = FI_SUCCESS;
@@ -493,37 +339,37 @@ static int map_remove(struct gnix_fid_av *int_av, fi_addr_t *fi_addr,
* TODO: is there a race condition here for multi-threaded?
*/
- the_entry = _gnix_ht_lookup(int_av->map_ht, key);
+ the_entry = _gnix_ht_lookup(av_priv->map_ht, key);
if (the_entry == NULL)
return -FI_ENOENT;
- ret = _gnix_ht_remove(int_av->map_ht, key);
+ ret = _gnix_ht_remove(av_priv->map_ht, key);
}
return ret;
}
-static int map_lookup(struct gnix_fid_av *int_av, fi_addr_t fi_addr,
- struct gnix_av_addr_entry **entry_ptr)
+static int map_lookup(struct gnix_fid_av *av_priv, fi_addr_t fi_addr,
+ struct gnix_av_addr_entry *entry_ptr)
{
gnix_ht_key_t *key = (gnix_ht_key_t *)&fi_addr;
struct gnix_av_addr_entry *entry;
- entry = _gnix_ht_lookup(int_av->map_ht, *key);
+ entry = _gnix_ht_lookup(av_priv->map_ht, *key);
if (entry == NULL)
return -FI_ENOENT;
- *entry_ptr = entry;
+ memcpy(entry_ptr, entry, sizeof(*entry));
return FI_SUCCESS;
}
-static int map_reverse_lookup(struct gnix_fid_av *int_av,
+static int map_reverse_lookup(struct gnix_fid_av *av_priv,
struct gnix_address gnix_addr,
fi_addr_t *fi_addr)
{
- GNIX_HASHTABLE_ITERATOR(int_av->map_ht, iter);
+ GNIX_HASHTABLE_ITERATOR(av_priv->map_ht, iter);
struct gnix_av_addr_entry *entry;
while ((entry = _gnix_ht_iterator_next(&iter))) {
@@ -533,15 +379,43 @@ static int map_reverse_lookup(struct gnix_fid_av *int_av,
}
}
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
return -FI_ENOENT;
}
/*******************************************************************************
* FI_AV API implementations.
******************************************************************************/
+int _gnix_table_lookup(struct gnix_fid_av *av_priv,
+ fi_addr_t fi_addr,
+ struct gnix_av_addr_entry *entry_ptr)
+{
+ return table_lookup(av_priv, fi_addr, entry_ptr);
+}
+
+int _gnix_table_reverse_lookup(struct gnix_fid_av *av_priv,
+ struct gnix_address gnix_addr,
+ fi_addr_t *fi_addr)
+{
+ return table_reverse_lookup(av_priv, gnix_addr, fi_addr);
+}
+
+int _gnix_map_lookup(struct gnix_fid_av *av_priv,
+ fi_addr_t fi_addr,
+ struct gnix_av_addr_entry *entry_ptr)
+{
+ return map_lookup(av_priv, fi_addr, entry_ptr);
+}
+
+int _gnix_map_reverse_lookup(struct gnix_fid_av *av_priv,
+ struct gnix_address gnix_addr,
+ fi_addr_t *fi_addr)
+{
+ return map_reverse_lookup(av_priv, gnix_addr, fi_addr);
+}
int _gnix_av_lookup(struct gnix_fid_av *gnix_av, fi_addr_t fi_addr,
- struct gnix_av_addr_entry **entry_ptr)
+ struct gnix_av_addr_entry *entry_ptr)
{
int ret = FI_SUCCESS;
@@ -594,6 +468,7 @@ int _gnix_av_reverse_lookup(struct gnix_fid_av *gnix_av,
}
err:
+ GNIX_TRACE(FI_LOG_AV, "\n");
return ret;
}
@@ -607,7 +482,7 @@ DIRECT_FN STATIC int gnix_av_lookup(struct fid_av *av, fi_addr_t fi_addr,
{
struct gnix_fid_av *gnix_av;
struct gnix_ep_name ep_name = { {0} };
- struct gnix_av_addr_entry *entry = NULL;
+ struct gnix_av_addr_entry entry;
int rc;
GNIX_TRACE(FI_LOG_AV, "\n");
@@ -615,8 +490,8 @@ DIRECT_FN STATIC int gnix_av_lookup(struct fid_av *av, fi_addr_t fi_addr,
if (!av || !addrlen)
return -FI_EINVAL;
- if (*addrlen < GNIX_AV_MAX_STR_ADDR_LEN) {
- *addrlen = GNIX_AV_MAX_STR_ADDR_LEN;
+ if (*addrlen < sizeof(ep_name)) {
+ *addrlen = sizeof(ep_name);
return -FI_ETOOSMALL;
}
@@ -636,13 +511,13 @@ DIRECT_FN STATIC int gnix_av_lookup(struct fid_av *av, fi_addr_t fi_addr,
return rc;
}
- memcpy(&ep_name.gnix_addr, &entry->gnix_addr,
- sizeof(struct gnix_address));
- ep_name.name_type = entry->name_type;
- ep_name.cm_nic_cdm_id = entry->cm_nic_cdm_id;
- ep_name.cookie = entry->cookie;
+ ep_name.gnix_addr = entry.gnix_addr;
+ ep_name.name_type = entry.name_type;
+ ep_name.cm_nic_cdm_id = entry.cm_nic_cdm_id;
+ ep_name.cookie = entry.cookie;
- gnix_av_straddr(NULL, (void *) &ep_name, (char *) addr, addrlen);
+ memcpy(addr, (void *)&ep_name, MIN(*addrlen, sizeof(ep_name)));
+ *addrlen = sizeof(ep_name);
return FI_SUCCESS;
}
@@ -651,7 +526,7 @@ DIRECT_FN STATIC int gnix_av_insert(struct fid_av *av, const void *addr,
size_t count, fi_addr_t *fi_addr,
uint64_t flags, void *context)
{
- struct gnix_fid_av *int_av = NULL;
+ struct gnix_fid_av *av_priv = NULL;
int ret = FI_SUCCESS;
GNIX_TRACE(FI_LOG_AV, "\n");
@@ -661,20 +536,20 @@ DIRECT_FN STATIC int gnix_av_insert(struct fid_av *av, const void *addr,
goto err;
}
- int_av = container_of(av, struct gnix_fid_av, av_fid);
+ av_priv = container_of(av, struct gnix_fid_av, av_fid);
- if (!int_av) {
+ if (!av_priv) {
ret = -FI_EINVAL;
goto err;
}
- switch (int_av->type) {
+ switch (av_priv->type) {
case FI_AV_TABLE:
ret =
- table_insert(int_av, addr, count, fi_addr, flags, context);
+ table_insert(av_priv, addr, count, fi_addr, flags, context);
break;
case FI_AV_MAP:
- ret = map_insert(int_av, addr, count, fi_addr, flags, context);
+ ret = map_insert(av_priv, addr, count, fi_addr, flags, context);
break;
default:
ret = -FI_EINVAL;
@@ -703,7 +578,7 @@ DIRECT_FN STATIC int gnix_av_insertsym(struct fid_av *av, const char *node,
DIRECT_FN STATIC int gnix_av_remove(struct fid_av *av, fi_addr_t *fi_addr,
size_t count, uint64_t flags)
{
- struct gnix_fid_av *int_av = NULL;
+ struct gnix_fid_av *av_priv = NULL;
int ret = FI_SUCCESS;
GNIX_TRACE(FI_LOG_AV, "\n");
@@ -713,19 +588,19 @@ DIRECT_FN STATIC int gnix_av_remove(struct fid_av *av, fi_addr_t *fi_addr,
goto err;
}
- int_av = container_of(av, struct gnix_fid_av, av_fid);
+ av_priv = container_of(av, struct gnix_fid_av, av_fid);
- if (!int_av) {
+ if (!av_priv) {
ret = -FI_EINVAL;
goto err;
}
- switch (int_av->type) {
+ switch (av_priv->type) {
case FI_AV_TABLE:
- ret = table_remove(int_av, fi_addr, count, flags);
+ ret = table_remove(av_priv, fi_addr, count, flags);
break;
case FI_AV_MAP:
- ret = map_remove(int_av, fi_addr, count, flags);
+ ret = map_remove(av_priv, fi_addr, count, flags);
break;
default:
ret = -FI_EINVAL;
@@ -743,12 +618,11 @@ err:
* hexadecimal. And name_type is represented as an integer.
*/
DIRECT_FN const char *gnix_av_straddr(struct fid_av *av,
- const void *addr, char *buf,
- size_t *len)
+ const void *addr, char *buf,
+ size_t *len)
{
char int_buf[GNIX_AV_MAX_STR_ADDR_LEN];
int size;
-
const struct gnix_ep_name *gnix_ep = addr;
/*
@@ -757,15 +631,14 @@ DIRECT_FN const char *gnix_av_straddr(struct fid_av *av,
* GNIX_AV_STR_ADDR_VERSION, increment this value
* GNIX_AV_MAX_STR_ADDR_LEN, to be the number of characters printed
*/
- size =
- snprintf(int_buf, sizeof(int_buf), "%04i:0x%08" PRIx32 ":0x%08"
- PRIx32 ":%02i:0x%06" PRIx32 ":0x%08" PRIx32,
- GNIX_AV_STR_ADDR_VERSION,
- gnix_ep->gnix_addr.device_addr,
- gnix_ep->gnix_addr.cdm_id,
- gnix_ep->name_type,
- gnix_ep->cm_nic_cdm_id,
- gnix_ep->cookie);
+ size = snprintf(int_buf, sizeof(int_buf), "%04i:0x%08" PRIx32 ":0x%08"
+ PRIx32 ":%02i:0x%06" PRIx32 ":0x%08" PRIx32,
+ GNIX_AV_STR_ADDR_VERSION,
+ gnix_ep->gnix_addr.device_addr,
+ gnix_ep->gnix_addr.cdm_id,
+ gnix_ep->name_type,
+ gnix_ep->cm_nic_cdm_id,
+ gnix_ep->cookie);
/*
* snprintf returns the number of character written
@@ -863,7 +736,7 @@ DIRECT_FN int gnix_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
struct fid_av **av, void *context)
{
struct gnix_fid_domain *int_dom = NULL;
- struct gnix_fid_av *int_av = NULL;
+ struct gnix_fid_av *av_priv = NULL;
struct gnix_hashtable_attr ht_attr;
enum fi_av_type type = FI_AV_TABLE;
@@ -883,50 +756,52 @@ DIRECT_FN int gnix_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
goto err;
}
- int_av = calloc(1, sizeof(*int_av));
- if (!int_av) {
+ av_priv = calloc(1, sizeof(*av_priv));
+ if (!av_priv) {
ret = -FI_ENOMEM;
goto err;
}
if (attr) {
- if (gnix_verify_av_attr(attr)) {
- ret = -FI_EINVAL;
+ ret = gnix_verify_av_attr(attr);
+ if (ret) {
goto cleanup;
}
- type = attr->type;
+ if (attr->type != FI_AV_UNSPEC) {
+ type = attr->type;
+ }
count = attr->count;
}
- int_av->domain = int_dom;
- int_av->type = type;
- int_av->addrlen = sizeof(struct gnix_address);
+ av_priv->domain = int_dom;
+ av_priv->type = type;
+ av_priv->addrlen = sizeof(struct gnix_address);
- int_av->capacity = count;
+ av_priv->capacity = count;
if (type == FI_AV_TABLE) {
- int_av->table = calloc(count,
+ av_priv->table = calloc(count,
sizeof(struct gnix_av_addr_entry));
- if (!int_av->table) {
+ if (!av_priv->table) {
ret = -FI_ENOMEM;
goto cleanup;
}
}
- int_av->valid_entry_vec = calloc(count, sizeof(int));
- if (!int_av->valid_entry_vec) {
+ av_priv->valid_entry_vec = calloc(count, sizeof(int));
+ if (!av_priv->valid_entry_vec) {
ret = -FI_ENOMEM;
goto cleanup;
}
- int_av->av_fid.fid.fclass = FI_CLASS_AV;
- int_av->av_fid.fid.context = context;
- int_av->av_fid.fid.ops = &gnix_fi_av_ops;
- int_av->av_fid.ops = &gnix_av_ops;
+ av_priv->av_fid.fid.fclass = FI_CLASS_AV;
+ av_priv->av_fid.fid.context = context;
+ av_priv->av_fid.fid.ops = &gnix_fi_av_ops;
+ av_priv->av_fid.ops = &gnix_av_ops;
if (type == FI_AV_MAP) {
- int_av->map_ht = calloc(1, sizeof(struct gnix_hashtable));
- if (int_av->map_ht == NULL)
+ av_priv->map_ht = calloc(1, sizeof(struct gnix_hashtable));
+ if (av_priv->map_ht == NULL)
goto cleanup;
/*
@@ -942,22 +817,22 @@ DIRECT_FN int gnix_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
ht_attr.ht_internal_locking = 1;
ht_attr.destructor = NULL;
- ret = _gnix_ht_init(int_av->map_ht,
+ ret = _gnix_ht_init(av_priv->map_ht,
&ht_attr);
- slist_init(&int_av->block_list);
+ slist_init(&av_priv->block_list);
}
- _gnix_ref_init(&int_av->ref_cnt, 1, __av_destruct);
+ _gnix_ref_init(&av_priv->ref_cnt, 1, __av_destruct);
- *av = &int_av->av_fid;
+ *av = &av_priv->av_fid;
return ret;
cleanup:
- if (int_av->table != NULL)
- free(int_av->table);
- if (int_av->valid_entry_vec != NULL)
- free(int_av->valid_entry_vec);
- free(int_av);
+ if (av_priv->table != NULL)
+ free(av_priv->table);
+ if (av_priv->valid_entry_vec != NULL)
+ free(av_priv->valid_entry_vec);
+ free(av_priv);
err:
return ret;
}
diff --git a/prov/gni/src/gnix_buddy_allocator.c b/prov/gni/src/gnix_buddy_allocator.c
index d8b6eb9..0304142 100644
--- a/prov/gni/src/gnix_buddy_allocator.c
+++ b/prov/gni/src/gnix_buddy_allocator.c
@@ -143,7 +143,8 @@ static inline int __gnix_buddy_create_lists(gnix_buddy_alloc_handle_t
/* Insert free blocks of size max in sorted order into last list */
for (i = 0; i < alloc_handle->len / alloc_handle->max; i++) {
- dlist_insert_tail(alloc_handle->base + offset,
+ dlist_insert_tail((void *) ((uint8_t *) alloc_handle->base +
+ offset),
alloc_handle->lists +
alloc_handle->nlists - 1);
offset += alloc_handle->max;
@@ -171,7 +172,8 @@ static inline void __gnix_buddy_split(gnix_buddy_alloc_handle_t *alloc_handle,
alloc_handle->len,
MIN_BLOCK_SIZE));
- dlist_insert_tail(tmp + OFFSET(MIN_BLOCK_SIZE, j - 1),
+ dlist_insert_tail((void *) ((uint8_t *) tmp +
+ OFFSET(MIN_BLOCK_SIZE, j - 1)),
alloc_handle->lists + j - 1);
}
@@ -366,7 +368,8 @@ int _gnix_buddy_free(gnix_buddy_alloc_handle_t *alloc_handle, void *ptr,
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
if (unlikely(!alloc_handle || !len || len > alloc_handle->max ||
- ptr >= alloc_handle->base + alloc_handle->len ||
+ ptr >= (void *) ((uint8_t *) alloc_handle->base +
+ alloc_handle->len) ||
ptr < alloc_handle->base)) {
GNIX_WARN(FI_LOG_EP_CTRL,
diff --git a/prov/gni/src/gnix_cm.c b/prov/gni/src/gnix_cm.c
index 07d98b3..8a14491 100644
--- a/prov/gni/src/gnix_cm.c
+++ b/prov/gni/src/gnix_cm.c
@@ -34,8 +34,7 @@
#include "gnix.h"
#include "gnix_util.h"
#include "gnix_nic.h"
-#include "gnix_nic.h"
-#include "gnix_av.h"
+#include "gnix_cm_nic.h"
/*******************************************************************************
* API function implementations.
@@ -57,32 +56,33 @@ DIRECT_FN STATIC int gnix_getname(fid_t fid, void *addr, size_t *addrlen)
struct gnix_ep_name name = {{0}};
struct gnix_fid_ep *ep = NULL;
int ret = FI_SUCCESS;
- size_t copy_size = GNIX_AV_MAX_STR_ADDR_LEN;
+ size_t copy_size;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
- if (!addrlen) {
- ret = -FI_EINVAL;
- goto err;
- }
+ copy_size = sizeof(struct gnix_ep_name);
- if (*addrlen < GNIX_AV_MIN_STR_ADDR_LEN) {
- /* return the maximum length of the string. */
- *addrlen = GNIX_AV_MAX_STR_ADDR_LEN;
+ /*
+ * If addrlen is less than the size necessary then continue copying with
+ * truncation and return error value -FI_ETOOSMALL.
+ */
+ if (*addrlen < copy_size) {
+ copy_size = *addrlen;
ret = -FI_ETOOSMALL;
- goto err;
}
+ /* copy the address length */
+ *addrlen = sizeof(struct gnix_ep_name);
+
if (!addr) {
- /* return the maximum length of the string. */
- *addrlen = GNIX_AV_MAX_STR_ADDR_LEN;
- ret = -FI_EINVAL;
+ if (copy_size >= *addrlen) {
+ ret = -FI_EINVAL;
+ }
+
goto err;
}
if (!fid) {
- /* return the maximum length of the string. */
- *addrlen = GNIX_AV_MAX_STR_ADDR_LEN;
ret = -FI_EINVAL;
goto err;
}
@@ -103,31 +103,7 @@ DIRECT_FN STATIC int gnix_getname(fid_t fid, void *addr, size_t *addrlen)
return -FI_ENOSYS; /*TODO: need to implement FI_EP_MSG */
}
- /*
- * If addrlen is less than the size necessary then continue copying
- * with the size of the receiving buffer.
- */
- if (*addrlen < copy_size) {
- copy_size = *addrlen;
- }
-
- /*
- * Retrieve the cdm_id, device_addr and other information
- * from the gnix_ep_name structure.
- */
-
- gnix_av_straddr(NULL, (void *) &name, addr, ©_size);
-
- /*
- * If the copy size is less then the string addr length,
- * truncation has occurred so return the error value -FI_ETOOSMALL.
- */
- if (copy_size < GNIX_AV_MAX_STR_ADDR_LEN) {
- ret = -FI_ETOOSMALL;
- }
-
- /* return the copied length of the string. */
- *addrlen = copy_size;
+ memcpy(addr, &name, copy_size);
err:
return ret;
diff --git a/prov/gni/src/gnix_cm_nic.c b/prov/gni/src/gnix_cm_nic.c
index f0e89de..f6d7e87 100644
--- a/prov/gni/src/gnix_cm_nic.c
+++ b/prov/gni/src/gnix_cm_nic.c
@@ -1,5 +1,6 @@
/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC.
+ * All rights reserved.
* Copyright (c) 2015 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -43,7 +44,6 @@
#include "gnix_cm_nic.h"
#include "gnix_nic.h"
#include "gnix_hashtable.h"
-#include "gnix_av.h"
#define GNIX_CM_NIC_BND_TAG (100)
@@ -192,34 +192,31 @@ err:
* Internal API functions
******************************************************************************/
-int _gnix_cm_nic_create_cdm_id(struct gnix_fid_domain *domain, uint32_t seed,
- uint32_t *id)
+int _gnix_cm_nic_create_cdm_id(struct gnix_fid_domain *domain, uint32_t *id)
{
uint32_t cdm_id;
+ int v;
- /*
- * the format of the cdm_id here is governed
- * by the fact that currently the cdm_id_seed
- * is the pid of the process/thread that created
- * the domain. So the most likely bits to be
- * unique using this method is the 16 LSBs
- * hence these are included, upper 16 MSBs are
- * masked off and overwritten by the supplied
- * seed value.
- */
- cdm_id = (domain->cdm_id_seed & 0x0000FFFF) |
- ((uint32_t)seed << 16);
+/*
+ * generate a cdm_id, use the 16 LSB of base_id from domain
+ * with 16 MSBs being obtained from atomic increment of
+ * a local variable.
+ */
+ v = atomic_inc(&gnix_id_counter);
+ cdm_id = ((domain->cdm_id_seed & 0xFFF) << 12) | v;
*id = cdm_id;
return FI_SUCCESS;
}
-int _gnix_cm_nic_get_cdm_seed_set(struct gnix_fid_domain *domain, int nseeds,
- uint32_t *seed)
+int _gnix_get_new_cdm_id_set(struct gnix_fid_domain *domain, int nids,
+ uint32_t *id)
{
- uint32_t seed_base;
+ uint32_t cdm_id;
+ int v;
- seed_base = atomic_add(&gnix_id_counter, nseeds);
- *seed = seed_base;
+ v = atomic_add(&gnix_id_counter, nids);
+ cdm_id = ((domain->cdm_id_seed & 0xFFF) << 12) | v;
+ *id = cdm_id;
return FI_SUCCESS;
}
@@ -236,13 +233,17 @@ int _gnix_cm_nic_progress(struct gnix_cm_nic *cm_nic)
*/
if (cm_nic->ctrl_progress == FI_PROGRESS_MANUAL) {
- ret = _gnix_dgram_poll(cm_nic->dgram_hndl,
- GNIX_DGRAM_NOBLOCK);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "_gnix_dgram_poll returned %s\n",
- fi_strerror(-ret));
- goto err;
+ ++cm_nic->poll_cnt;
+ if (((cm_nic->poll_cnt % 512) == 0) ||
+ !dlist_empty(&cm_nic->cm_nic_wq)) {
+ ret = _gnix_dgram_poll(cm_nic->dgram_hndl,
+ GNIX_DGRAM_NOBLOCK);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gnix_dgram_poll returned %s\n",
+ fi_strerror(-ret));
+ goto err;
+ }
}
}
@@ -311,9 +312,6 @@ static void __cm_nic_destruct(void *obj)
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
- if (cm_nic->nic != NULL)
- _gnix_ref_put(cm_nic->nic);
-
if (cm_nic->dgram_hndl != NULL) {
ret = _gnix_dgram_hndl_free(cm_nic->dgram_hndl);
if (ret != FI_SUCCESS)
@@ -332,9 +330,33 @@ static void __cm_nic_destruct(void *obj)
cm_nic->addr_to_ep_ht = NULL;
}
+ if (cm_nic->nic != NULL)
+ _gnix_ref_put(cm_nic->nic);
+
free(cm_nic);
}
+static int __gnix_cm_nic_intra_progress_fn(void *data, int *complete_ptr)
+{
+ struct gnix_datagram *dgram;
+ struct gnix_cm_nic *cm_nic;
+ int ret;
+
+ GNIX_INFO(FI_LOG_EP_CTRL, "\n");
+
+ dgram = (struct gnix_datagram *)data;
+ cm_nic = (struct gnix_cm_nic *)dgram->cache;
+ ret = __process_datagram(dgram,
+ cm_nic->my_name.gnix_addr,
+ GNI_POST_COMPLETED);
+ if (ret == FI_SUCCESS) {
+ GNIX_INFO(FI_LOG_EP_CTRL, "Intra-CM NIC dgram completed\n");
+ *complete_ptr = 1;
+ }
+
+ return FI_SUCCESS;
+}
+
int _gnix_cm_nic_send(struct gnix_cm_nic *cm_nic,
char *sbuf, size_t len,
struct gnix_address target_addr)
@@ -343,6 +365,7 @@ int _gnix_cm_nic_send(struct gnix_cm_nic *cm_nic,
struct gnix_datagram *dgram = NULL;
ssize_t __attribute__((unused)) plen;
uint8_t tag;
+ struct gnix_work_req *work_req;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
@@ -373,10 +396,39 @@ int _gnix_cm_nic_send(struct gnix_cm_nic *cm_nic,
sbuf, len);
assert (plen == len);
- ret = _gnix_dgram_bnd_post(dgram);
- if (ret == -FI_EBUSY) {
- ret = -FI_EAGAIN;
- _gnix_dgram_free(dgram);
+ /* If connecting with the same CM NIC, skip datagram exchange. The
+ * caller could be holding an endpoint lock, so schedule connection
+ * completion for later. */
+ if (GNIX_ADDR_EQUAL(target_addr, cm_nic->my_name.gnix_addr)) {
+ char tmp_buf[GNIX_CM_NIC_MAX_MSG_SIZE];
+
+ /* Pack output buffer with input data. */
+ _gnix_dgram_unpack_buf(dgram, GNIX_DGRAM_IN_BUF, tmp_buf,
+ GNIX_CM_NIC_MAX_MSG_SIZE);
+ _gnix_dgram_pack_buf(dgram, GNIX_DGRAM_OUT_BUF, tmp_buf,
+ GNIX_CM_NIC_MAX_MSG_SIZE);
+
+ work_req = calloc(1, sizeof(*work_req));
+ if (work_req == NULL) {
+ _gnix_dgram_free(dgram);
+ return -FI_ENOMEM;
+ }
+
+ work_req->progress_fn = __gnix_cm_nic_intra_progress_fn;
+ work_req->data = dgram;
+ work_req->completer_fn = NULL;
+
+ fastlock_acquire(&cm_nic->wq_lock);
+ dlist_insert_before(&work_req->list, &cm_nic->cm_nic_wq);
+ fastlock_release(&cm_nic->wq_lock);
+
+ GNIX_INFO(FI_LOG_EP_CTRL, "Initiated intra-CM NIC connect\n");
+ } else {
+ ret = _gnix_dgram_bnd_post(dgram);
+ if (ret == -FI_EBUSY) {
+ ret = -FI_EAGAIN;
+ _gnix_dgram_free(dgram);
+ }
}
exit:
@@ -473,13 +525,12 @@ int _gnix_cm_nic_free(struct gnix_cm_nic *cm_nic)
int _gnix_cm_nic_alloc(struct gnix_fid_domain *domain,
struct fi_info *info,
+ uint32_t cdm_id,
struct gnix_cm_nic **cm_nic_ptr)
{
int ret = FI_SUCCESS;
struct gnix_cm_nic *cm_nic = NULL;
- uint32_t cdm_id, seed;
gnix_hashtable_attr_t gnix_ht_attr = {0};
- struct gnix_ep_name name;
uint32_t name_type = GNIX_EPN_TYPE_UNBOUND;
struct gnix_nic_attr nic_attr = {0};
@@ -496,32 +547,9 @@ int _gnix_cm_nic_alloc(struct gnix_fid_domain *domain,
*/
if (info->src_addr &&
- info->src_addrlen == GNIX_AV_MAX_STR_ADDR_LEN) {
- gnix_av_straddr_to_ep_name(info->src_addr, &name);
- if (name.name_type == GNIX_EPN_TYPE_BOUND) {
- /* EP name includes user specified service/port */
- cdm_id = name.gnix_addr.cdm_id;
- name_type = name.name_type;
- }
- }
-
- if (name_type == GNIX_EPN_TYPE_UNBOUND) {
- ret = _gnix_cm_nic_get_cdm_seed_set(domain, 1,
- &seed);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "gnix_cm_nic_get_cdm_seed_set returned %s\n",
- fi_strerror(-ret));
- goto err;
- }
- ret = _gnix_cm_nic_create_cdm_id(domain, seed,
- &cdm_id);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "gnix_cm_nic_create_cdm_id returned %s\n",
- fi_strerror(-ret));
- goto err;
- }
+ info->src_addrlen == sizeof(struct gnix_ep_name)) {
+ name_type = ((struct gnix_ep_name *)(info->src_addr))->
+ name_type;
}
GNIX_INFO(FI_LOG_EP_CTRL, "creating cm_nic for %u/0x%x/%u\n",
@@ -552,6 +580,7 @@ int _gnix_cm_nic_alloc(struct gnix_fid_domain *domain,
cm_nic->fabric = domain->fabric;
cm_nic->ctrl_progress = domain->control_progress;
cm_nic->my_name.name_type = name_type;
+ cm_nic->poll_cnt = 0;
fastlock_init(&cm_nic->wq_lock);
dlist_init(&cm_nic->cm_nic_wq);
diff --git a/prov/gni/src/gnix_cntr.c b/prov/gni/src/gnix_cntr.c
index faf82da..b857954 100644
--- a/prov/gni/src/gnix_cntr.c
+++ b/prov/gni/src/gnix_cntr.c
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC.
+ * All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -41,6 +42,7 @@
#include "gnix.h"
#include "gnix_cntr.h"
#include "gnix_nic.h"
+#include "gnix_trigger.h"
/*******************************************************************************
* Forward declarations for filling functions.
@@ -71,9 +73,11 @@ static int __verify_cntr_attr(struct fi_cntr_attr *attr)
return -FI_EINVAL;
}
- /*
- * TODO: need to support wait objects on cntr
- */
+ /* TODO: Wait objects are not yet implemented. */
+ if (attr->wait_obj != FI_WAIT_NONE) {
+ return -FI_ENOSYS;
+ }
+
switch (attr->wait_obj) {
case FI_WAIT_UNSPEC:
case FI_WAIT_NONE:
@@ -97,7 +101,7 @@ static int __gnix_cntr_progress(struct gnix_fid_cntr *cntr)
struct gnix_cq_poll_nic *pnic, *tmp;
int rc = FI_SUCCESS;
- rwlock_rdlock(&cntr->nic_lock);
+ COND_READ_ACQUIRE(cntr->requires_lock, &cntr->nic_lock);
dlist_for_each_safe(&cntr->poll_nics, pnic, tmp, list) {
rc = _gnix_nic_progress(pnic->nic);
@@ -107,7 +111,7 @@ static int __gnix_cntr_progress(struct gnix_fid_cntr *cntr)
}
}
- rwlock_unlock(&cntr->nic_lock);
+ COND_RW_RELEASE(cntr->requires_lock, &cntr->nic_lock);
return rc;
}
@@ -126,6 +130,9 @@ int _gnix_cntr_inc(struct gnix_fid_cntr *cntr)
if (cntr->wait)
_gnix_signal_wait_obj(cntr->wait);
+ if (_gnix_trigger_pending(cntr))
+ _gnix_trigger_check_cntr(cntr);
+
return FI_SUCCESS;
}
@@ -146,12 +153,12 @@ int _gnix_cntr_poll_nic_add(struct gnix_fid_cntr *cntr, struct gnix_nic *nic)
{
struct gnix_cntr_poll_nic *pnic, *tmp;
- rwlock_wrlock(&cntr->nic_lock);
+ COND_WRITE_ACQUIRE(cntr->requires_lock, &cntr->nic_lock);
dlist_for_each_safe(&cntr->poll_nics, pnic, tmp, list) {
if (pnic->nic == nic) {
pnic->ref_cnt++;
- rwlock_unlock(&cntr->nic_lock);
+ COND_RW_RELEASE(cntr->requires_lock, &cntr->nic_lock);
return FI_SUCCESS;
}
}
@@ -159,7 +166,7 @@ int _gnix_cntr_poll_nic_add(struct gnix_fid_cntr *cntr, struct gnix_nic *nic)
pnic = malloc(sizeof(struct gnix_cntr_poll_nic));
if (!pnic) {
GNIX_WARN(FI_LOG_CQ, "Failed to add NIC to CNTR poll list.\n");
- rwlock_unlock(&cntr->nic_lock);
+ COND_RW_RELEASE(cntr->requires_lock, &cntr->nic_lock);
return -FI_ENOMEM;
}
@@ -169,7 +176,7 @@ int _gnix_cntr_poll_nic_add(struct gnix_fid_cntr *cntr, struct gnix_nic *nic)
dlist_init(&pnic->list);
dlist_insert_tail(&pnic->list, &cntr->poll_nics);
- rwlock_unlock(&cntr->nic_lock);
+ COND_RW_RELEASE(cntr->requires_lock, &cntr->nic_lock);
GNIX_INFO(FI_LOG_CQ, "Added NIC(%p) to CNTR(%p) poll list\n",
nic, cntr);
@@ -181,7 +188,7 @@ int _gnix_cntr_poll_nic_rem(struct gnix_fid_cntr *cntr, struct gnix_nic *nic)
{
struct gnix_cntr_poll_nic *pnic, *tmp;
- rwlock_wrlock(&cntr->nic_lock);
+ COND_WRITE_ACQUIRE(cntr->requires_lock, &cntr->nic_lock);
dlist_for_each_safe(&cntr->poll_nics, pnic, tmp, list) {
if (pnic->nic == nic) {
@@ -192,12 +199,12 @@ int _gnix_cntr_poll_nic_rem(struct gnix_fid_cntr *cntr, struct gnix_nic *nic)
"Removed NIC(%p) from CNTR(%p) poll list\n",
nic, cntr);
}
- rwlock_unlock(&cntr->nic_lock);
+ COND_RW_RELEASE(cntr->requires_lock, &cntr->nic_lock);
return FI_SUCCESS;
}
}
- rwlock_unlock(&cntr->nic_lock);
+ COND_RW_RELEASE(cntr->requires_lock, &cntr->nic_lock);
GNIX_WARN(FI_LOG_CQ, "NIC not found on CNTR poll list.\n");
return -FI_EINVAL;
@@ -207,31 +214,38 @@ int _gnix_cntr_poll_nic_rem(struct gnix_fid_cntr *cntr, struct gnix_nic *nic)
* API functions.
******************************************************************************/
+static int gnix_cntr_wait_sleep(struct gnix_fid_cntr *cntr_priv,
+ uint64_t threshold, int timeout)
+{
+ int ret = FI_SUCCESS;
+
+ while (atomic_get(&cntr_priv->cnt) < threshold) {
+ ret = gnix_wait_wait((struct fid_wait *)cntr_priv->wait,
+ timeout);
+ if (ret == -FI_ETIMEDOUT)
+ break;
+ if (ret) {
+ GNIX_WARN(FI_LOG_CQ,
+ " gnix_wait_wait returned %d.\n",
+ ret);
+ break;
+ }
+ }
+
+ return ret;
+}
+
+
DIRECT_FN STATIC int gnix_cntr_wait(struct fid_cntr *cntr, uint64_t threshold,
int timeout)
{
struct gnix_fid_cntr *cntr_priv;
- struct timespec ts0, ts;
- int msec_passed = 0;
int ret = 0;
cntr_priv = container_of(cntr, struct gnix_fid_cntr, cntr_fid);
- clock_gettime(CLOCK_REALTIME, &ts0);
-
- while (atomic_get(&cntr_priv->cnt) < threshold) {
- if (cntr_priv->wait) {
- ret = gnix_wait_wait((struct fid_wait *)cntr_priv->wait,
- timeout - msec_passed);
- if (ret == -FI_ETIMEDOUT)
- break;
- if (ret) {
- GNIX_WARN(FI_LOG_CQ,
- " gnix_wait_wait returned %d.\n",
- ret);
- break;
- }
- } else {
+ if (!cntr_priv->wait) {
+ while (atomic_get(&cntr_priv->cnt) < threshold) {
ret = __gnix_cntr_progress(cntr_priv);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_CQ,
@@ -239,22 +253,13 @@ DIRECT_FN STATIC int gnix_cntr_wait(struct fid_cntr *cntr, uint64_t threshold,
ret);
break;
}
- }
- if (atomic_get(&cntr_priv->cnt) >= threshold)
- break;
-
- if (timeout < 0)
- continue;
-
- clock_gettime(CLOCK_REALTIME, &ts);
- msec_passed = (ts.tv_sec - ts0.tv_sec) * 1000 +
- (ts.tv_nsec - ts0.tv_nsec) / 1000000;
+ if (atomic_get(&cntr_priv->cnt) >= threshold)
+ break;
- if (msec_passed >= timeout) {
- ret = -FI_ETIMEDOUT;
- break;
}
+ } else {
+ ret = gnix_cntr_wait_sleep(cntr_priv, threshold, timeout);
}
return ret;
@@ -358,6 +363,8 @@ DIRECT_FN STATIC int gnix_cntr_add(struct fid_cntr *cntr, uint64_t value)
if (cntr_priv->wait)
_gnix_signal_wait_obj(cntr_priv->wait);
+ _gnix_trigger_check_cntr(cntr_priv);
+
return FI_SUCCESS;
}
@@ -374,6 +381,8 @@ DIRECT_FN STATIC int gnix_cntr_set(struct fid_cntr *cntr, uint64_t value)
if (cntr_priv->wait)
_gnix_signal_wait_obj(cntr_priv->wait);
+ _gnix_trigger_check_cntr(cntr_priv);
+
return FI_SUCCESS;
}
@@ -433,6 +442,9 @@ DIRECT_FN int gnix_cntr_open(struct fid_domain *domain,
goto err;
}
+ cntr_priv->requires_lock = (domain_priv->thread_model !=
+ FI_THREAD_COMPLETION);
+
cntr_priv->domain = domain_priv;
cntr_priv->attr = *attr;
/* ref count is initialized to one to show that the counter exists */
@@ -445,6 +457,8 @@ DIRECT_FN int gnix_cntr_open(struct fid_domain *domain,
_gnix_ref_get(cntr_priv->domain);
dlist_init(&cntr_priv->poll_nics);
rwlock_init(&cntr_priv->nic_lock);
+ dlist_init(&cntr_priv->trigger_list);
+ fastlock_init(&cntr_priv->trigger_lock);
cntr_priv->cntr_fid.fid.fclass = FI_CLASS_CNTR;
cntr_priv->cntr_fid.fid.context = context;
diff --git a/prov/gni/src/gnix_cq.c b/prov/gni/src/gnix_cq.c
index c00c6aa..2f024e7 100644
--- a/prov/gni/src/gnix_cq.c
+++ b/prov/gni/src/gnix_cq.c
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC.
+ * All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -41,6 +42,7 @@
#include "gnix.h"
#include "gnix_cq.h"
#include "gnix_nic.h"
+#include "gnix_cm_nic.h"
/*******************************************************************************
* Function pointer for filling specific entry format type.
@@ -156,6 +158,11 @@ static int verify_cq_attr(struct fi_cq_attr *attr, struct fi_ops_cq *ops,
return -FI_EINVAL;
}
+ /* TODO: Wait objects are not yet implemented. */
+ if (attr->wait_obj != FI_WAIT_NONE) {
+ return -FI_ENOSYS;
+ }
+
switch (attr->wait_obj) {
case FI_WAIT_NONE:
ops->sread = fi_no_cq_sread;
@@ -249,14 +256,12 @@ err:
return NULL;
}
-extern int _gnix_cm_nic_progress(struct gnix_cm_nic *cm_nic);
-
static int __gnix_cq_progress(struct gnix_fid_cq *cq)
{
struct gnix_cq_poll_nic *pnic, *tmp;
int rc;
- rwlock_rdlock(&cq->nic_lock);
+ COND_READ_ACQUIRE(cq->requires_lock, &cq->nic_lock);
dlist_for_each_safe(&cq->poll_nics, pnic, tmp, list) {
rc = _gnix_nic_progress(pnic->nic);
@@ -266,7 +271,7 @@ static int __gnix_cq_progress(struct gnix_fid_cq *cq)
}
}
- rwlock_unlock(&cq->nic_lock);
+ COND_RW_RELEASE(cq->requires_lock, &cq->nic_lock);
if (unlikely(cq->domain->control_progress != FI_PROGRESS_AUTO)) {
if (cq->domain->cm_nic != NULL) {
@@ -291,7 +296,7 @@ ssize_t _gnix_cq_add_event(struct gnix_fid_cq *cq, void *op_context,
struct gnix_cq_entry *event;
struct slist_entry *item;
- fastlock_acquire(&cq->lock);
+ COND_ACQUIRE(cq->requires_lock, &cq->lock);
item = _gnix_queue_get_free(cq->events);
if (!item) {
@@ -308,12 +313,12 @@ ssize_t _gnix_cq_add_event(struct gnix_fid_cq *cq, void *op_context,
event->src_addr = src_addr;
_gnix_queue_enqueue(cq->events, &event->item);
- GNIX_INFO(FI_LOG_CQ, "Added event: %lx\n", op_context);
+ GNIX_DEBUG(FI_LOG_CQ, "Added event: %lx\n", op_context);
if (cq->wait)
_gnix_signal_wait_obj(cq->wait);
- fastlock_release(&cq->lock);
+ COND_RELEASE(cq->requires_lock, &cq->lock);
return FI_SUCCESS;
}
@@ -332,7 +337,7 @@ ssize_t _gnix_cq_add_error(struct gnix_fid_cq *cq, void *op_context,
GNIX_INFO(FI_LOG_CQ, "creating error event entry\n");
- fastlock_acquire(&cq->lock);
+ COND_ACQUIRE(cq->requires_lock, &cq->lock);
item = _gnix_queue_get_free(cq->errors);
if (!item) {
@@ -359,7 +364,7 @@ ssize_t _gnix_cq_add_error(struct gnix_fid_cq *cq, void *op_context,
_gnix_queue_enqueue(cq->errors, &event->item);
err:
- fastlock_release(&cq->lock);
+ COND_RELEASE(cq->requires_lock, &cq->lock);
return ret;
}
@@ -368,7 +373,7 @@ int _gnix_cq_poll_nic_add(struct gnix_fid_cq *cq, struct gnix_nic *nic)
{
struct gnix_cq_poll_nic *pnic, *tmp;
- rwlock_wrlock(&cq->nic_lock);
+ COND_WRITE_ACQUIRE(cq->requires_lock, &cq->nic_lock);
dlist_for_each_safe(&cq->poll_nics, pnic, tmp, list) {
if (pnic->nic == nic) {
@@ -381,7 +386,7 @@ int _gnix_cq_poll_nic_add(struct gnix_fid_cq *cq, struct gnix_nic *nic)
pnic = malloc(sizeof(struct gnix_cq_poll_nic));
if (!pnic) {
GNIX_WARN(FI_LOG_CQ, "Failed to add NIC to CQ poll list.\n");
- rwlock_unlock(&cq->nic_lock);
+ COND_RW_RELEASE(cq->requires_lock, &cq->nic_lock);
return -FI_ENOMEM;
}
@@ -391,7 +396,7 @@ int _gnix_cq_poll_nic_add(struct gnix_fid_cq *cq, struct gnix_nic *nic)
dlist_init(&pnic->list);
dlist_insert_tail(&pnic->list, &cq->poll_nics);
- rwlock_unlock(&cq->nic_lock);
+ COND_RW_RELEASE(cq->requires_lock, &cq->nic_lock);
GNIX_INFO(FI_LOG_CQ, "Added NIC(%p) to CQ(%p) poll list\n",
nic, cq);
@@ -403,7 +408,7 @@ int _gnix_cq_poll_nic_rem(struct gnix_fid_cq *cq, struct gnix_nic *nic)
{
struct gnix_cq_poll_nic *pnic, *tmp;
- rwlock_wrlock(&cq->nic_lock);
+ COND_WRITE_ACQUIRE(cq->requires_lock, &cq->nic_lock);
dlist_for_each_safe(&cq->poll_nics, pnic, tmp, list) {
if (pnic->nic == nic) {
@@ -419,7 +424,7 @@ int _gnix_cq_poll_nic_rem(struct gnix_fid_cq *cq, struct gnix_nic *nic)
}
}
- rwlock_unlock(&cq->nic_lock);
+ COND_RW_RELEASE(cq->requires_lock, &cq->nic_lock);
GNIX_WARN(FI_LOG_CQ, "NIC not found on CQ poll list.\n");
return -FI_EINVAL;
@@ -500,7 +505,7 @@ DIRECT_FN STATIC ssize_t gnix_cq_readfrom(struct fid_cq *cq, void *buf,
if (_gnix_queue_peek(cq_priv->errors))
return -FI_EAVAIL;
- fastlock_acquire(&cq_priv->lock);
+ COND_ACQUIRE(cq_priv->requires_lock, &cq_priv->lock);
while (_gnix_queue_peek(cq_priv->events) && count--) {
temp = _gnix_queue_dequeue(cq_priv->events);
@@ -513,12 +518,12 @@ DIRECT_FN STATIC ssize_t gnix_cq_readfrom(struct fid_cq *cq, void *buf,
_gnix_queue_enqueue_free(cq_priv->events, &event->item);
- buf += cq_priv->entry_size;
+ buf = (void *) ((uint8_t *) buf + cq_priv->entry_size);
read_count++;
}
- fastlock_release(&cq_priv->lock);
+ COND_RELEASE(cq_priv->requires_lock, &cq_priv->lock);
return read_count ?: -FI_EAGAIN;
}
@@ -582,7 +587,7 @@ DIRECT_FN STATIC ssize_t gnix_cq_readerr(struct fid_cq *cq,
cq_priv = container_of(cq, struct gnix_fid_cq, cq_fid);
- fastlock_acquire(&cq_priv->lock);
+ COND_ACQUIRE(cq_priv->requires_lock, &cq_priv->lock);
entry = _gnix_queue_dequeue(cq_priv->errors);
if (!entry) {
@@ -599,7 +604,7 @@ DIRECT_FN STATIC ssize_t gnix_cq_readerr(struct fid_cq *cq,
read_count++;
err:
- fastlock_release(&cq_priv->lock);
+ COND_RELEASE(cq_priv->requires_lock, &cq_priv->lock);
return read_count;
}
@@ -684,6 +689,9 @@ DIRECT_FN int gnix_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
goto err2;
}
+ cq_priv->requires_lock = (domain_priv->thread_model !=
+ FI_THREAD_COMPLETION);
+
cq_priv->domain = domain_priv;
cq_priv->attr = *attr;
diff --git a/prov/gni/src/gnix_datagram.c b/prov/gni/src/gnix_datagram.c
index 048ee9a..d414918 100644
--- a/prov/gni/src/gnix_datagram.c
+++ b/prov/gni/src/gnix_datagram.c
@@ -1,7 +1,8 @@
/*
* Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- * Copyright (c) 2015-16 Los Alamos National Security, LLC.
- * All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC.
+ * All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -49,11 +50,12 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
+#include <rdma/providers/fi_prov.h>
+
#include "gnix.h"
#include "gnix_datagram.h"
#include "gnix_util.h"
@@ -309,7 +311,7 @@ int _gnix_dgram_wc_post(struct gnix_datagram *d)
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
status = GNI_EpPostDataWId(d->gni_ep,
d->dgram_in_buf,
GNI_DATAGRAM_MAXSIZE,
@@ -322,9 +324,9 @@ int _gnix_dgram_wc_post(struct gnix_datagram *d)
/*
* datagram is active now, listening
*/
- d->state = GNIX_DGRAM_STATE_LISTENING;
+ d->state = GNIX_DGRAM_STATE_ACTIVE;
}
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
return ret;
}
@@ -352,7 +354,7 @@ int _gnix_dgram_bnd_post(struct gnix_datagram *d)
goto err;
}
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
if (d->pre_post_clbk_fn != NULL) {
ret = d->pre_post_clbk_fn(d, &post);
if (ret != FI_SUCCESS)
@@ -385,7 +387,7 @@ int _gnix_dgram_bnd_post(struct gnix_datagram *d)
}
}
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
if (post) {
if ((status != GNI_RC_SUCCESS) &&
@@ -401,7 +403,7 @@ int _gnix_dgram_bnd_post(struct gnix_datagram *d)
/*
* datagram is active now, connecting
*/
- d->state = GNIX_DGRAM_STATE_CONNECTING;
+ d->state = GNIX_DGRAM_STATE_ACTIVE;
} else {
ret = -FI_EBUSY;
}
@@ -462,52 +464,44 @@ int _gnix_dgram_poll(struct gnix_dgram_hndl *hndl,
dg_ptr = (struct gnix_datagram *)datagram_id;
assert(dg_ptr != NULL);
- assert((dg_ptr->state == GNIX_DGRAM_STATE_CONNECTING) ||
- (dg_ptr->state = GNIX_DGRAM_STATE_LISTENING));
-
/*
* do need to take lock here
*/
- fastlock_acquire(&nic->lock);
-
- if (dg_ptr->pre_test_clbk_fn != NULL) {
- ret = dg_ptr->pre_test_clbk_fn(dg_ptr);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "pre_test_clbk returned %d\n",
- ret);
- }
- }
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
status = GNI_EpPostDataTestById(dg_ptr->gni_ep,
datagram_id,
&post_state,
&responding_remote_addr,
&responding_remote_id);
- if (status != GNI_RC_SUCCESS) {
+ if ((status != GNI_RC_SUCCESS) &&
+ (status !=GNI_RC_NO_MATCH)) {
GNIX_WARN(FI_LOG_EP_CTRL,
"GNI_EpPostDataTestById: %s\n",
gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
goto err;
+ } else {
+ if ((status == GNI_RC_SUCCESS) &&
+ (dg_ptr->state != GNIX_DGRAM_STATE_ACTIVE)) {
+ GNIX_DEBUG(FI_LOG_EP_CTRL,
+ "GNI_EpPostDataTestById ",
+ "returned success but dgram not active\n");
+ }
}
- if (dg_ptr->post_test_clbk_fn != NULL) {
- responding_addr.device_addr =
- responding_remote_addr;
- responding_addr.cdm_id =
- responding_remote_id;
- ret = dg_ptr->post_test_clbk_fn(dg_ptr,
- responding_addr,
- post_state);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "post_test_clbk returned %d\n",
- ret);
- }
+ COND_RELEASE(nic->requires_lock, &nic->lock);
+
+ /*
+ * no match is okay, it means another thread
+ * won the race to get this datagram
+ */
+
+ if (status == GNI_RC_NO_MATCH) {
+ ret = FI_SUCCESS;
+ goto err;
}
- fastlock_release(&nic->lock);
/*
* pass COMPLETED and error post state cases to
@@ -560,6 +554,7 @@ int _gnix_dgram_hndl_alloc(const struct gnix_fid_fabric *fabric,
struct gnix_dgram_hndl *the_hndl = NULL;
struct gnix_nic *nic;
gni_return_t status;
+ uint32_t num_corespec_cpus = 0;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
@@ -656,10 +651,21 @@ int _gnix_dgram_hndl_alloc(const struct gnix_fid_fabric *fabric,
* though...
*/
- ret = _gnix_job_disable_affinity_apply();
+ ret = _gnix_get_num_corespec_cpus(&num_corespec_cpus);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "failed to get num corespec cpus\n");
+ }
+
+ if (num_corespec_cpus > 0) {
+ ret = _gnix_job_disable_affinity_apply();
+ } else {
+ ret = _gnix_job_enable_unassigned_cpus();
+ }
if (ret != 0)
GNIX_WARN(FI_LOG_EP_CTRL,
- "_gnix_job_disable call returned %d\n", ret);
+ "disable_affinity/unassigned_cpus call returned %d\n",
+ ret);
ret = pthread_create(&the_hndl->progress_thread,
NULL,
diff --git a/prov/gni/src/gnix_dom.c b/prov/gni/src/gnix_dom.c
index 41d2241..4bc6840 100644
--- a/prov/gni/src/gnix_dom.c
+++ b/prov/gni/src/gnix_dom.c
@@ -42,9 +42,16 @@
#include "gnix.h"
#include "gnix_nic.h"
#include "gnix_util.h"
+#include "gnix_xpmem.h"
gni_cq_mode_t gnix_def_gni_cq_modes = GNI_CQ_PHYS_PAGES;
+static char *__gnix_mr_type_to_str[GNIX_MR_MAX_TYPE] = {
+ [GNIX_MR_TYPE_INTERNAL] = "internal",
+ [GNIX_MR_TYPE_UDREG] = "udreg",
+ [GNIX_MR_TYPE_NONE] = "none",
+};
+
/*******************************************************************************
* Forward declaration for ops structures.
******************************************************************************/
@@ -62,18 +69,9 @@ static void __domain_destruct(void *obj)
GNIX_TRACE(FI_LOG_DOMAIN, "\n");
- /* if the domain isn't being destructed by close, we need to check the
- * cache again. This isn't a likely case. Destroy must succeed since we
- * are in the destruct path */
- if (domain->mr_cache) {
- ret = _gnix_mr_cache_destroy(domain->mr_cache);
- if (ret != FI_SUCCESS)
- GNIX_WARN(FI_LOG_DOMAIN, "failed to destroy mr cache "
- "during domain destruct, dom=%p",
- domain);
- assert(ret == FI_SUCCESS);
- }
-
+ ret = _gnix_close_cache(domain);
+ if (ret != FI_SUCCESS)
+ GNIX_FATAL(FI_LOG_MR, "failed to close memory registration cache\n");
/*
* Drop a reference to each NIC used by this domain.
*/
@@ -253,6 +251,33 @@ static const uint32_t default_tx_cq_size = 2048;
static const uint32_t default_max_retransmits = 5;
static const int32_t default_err_inject_count; /* static var is zeroed */
+static int __gnix_string_to_mr_type(const char *name)
+{
+ int i;
+ for (i = 0; i < GNIX_MR_MAX_TYPE; i++)
+ if (strncmp(name, __gnix_mr_type_to_str[i],
+ strlen(__gnix_mr_type_to_str[i])) == 0)
+ return i;
+
+ return -1;
+}
+
+static int
+__gnix_dom_ops_flush_cache(struct fid *fid)
+{
+ struct gnix_fid_domain *domain;
+
+ GNIX_TRACE(FI_LOG_DOMAIN, "\n");
+
+ domain = container_of(fid, struct gnix_fid_domain, domain_fid.fid);
+ if (domain->domain_fid.fid.fclass != FI_CLASS_DOMAIN) {
+ GNIX_WARN(FI_LOG_DOMAIN, ("Invalid domain\n"));
+ return -FI_EINVAL;
+ }
+
+ return _gnix_flush_registration_cache(domain);
+}
+
static int
__gnix_dom_ops_get_val(struct fid *fid, dom_ops_val_t t, void *val)
{
@@ -314,6 +339,28 @@ __gnix_dom_ops_get_val(struct fid *fid, dom_ops_val_t t, void *val)
case GNI_MR_CACHE_LAZY_DEREG:
*(int32_t *)val = domain->mr_cache_attr.lazy_deregistration;
break;
+ case GNI_MR_CACHE:
+ *(char **) val = __gnix_mr_type_to_str[domain->mr_cache_type];
+ break;
+ case GNI_MR_UDREG_REG_LIMIT:
+ *(int32_t *)val = domain->udreg_reg_limit;
+ break;
+ case GNI_MR_HARD_REG_LIMIT:
+ *(int32_t *)val = domain->mr_cache_attr.hard_reg_limit;
+ break;
+ case GNI_MR_SOFT_REG_LIMIT:
+ *(int32_t *)val = domain->mr_cache_attr.soft_reg_limit;
+ break;
+ case GNI_MR_HARD_STALE_REG_LIMIT:
+ *(int32_t *)val = domain->mr_cache_attr.hard_stale_limit;
+ break;
+ case GNI_XPMEM_ENABLE:
+ *(bool *)val = domain->params.xpmem_enabled;
+#if !HAVE_XPMEM
+ GNIX_WARN(FI_LOG_DOMAIN,
+ "GNI provider XPMEM support not configured\n");
+#endif
+ break;
default:
GNIX_WARN(FI_LOG_DOMAIN, ("Invalid dom_ops_val\n"));
return -FI_EINVAL;
@@ -328,6 +375,7 @@ __gnix_dom_ops_set_val(struct fid *fid, dom_ops_val_t t, void *val)
gni_return_t grc = GNI_RC_SUCCESS;
struct gnix_nic *nic;
struct gnix_fid_domain *domain;
+ int ret, type;
GNIX_TRACE(FI_LOG_DOMAIN, "\n");
@@ -380,7 +428,7 @@ __gnix_dom_ops_set_val(struct fid *fid, dom_ops_val_t t, void *val)
domain->params.max_retransmits = *(uint32_t *)val;
dlist_for_each(&domain->nic_list, nic, dom_nic_list)
{
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
grc = GNI_SmsgSetMaxRetrans(nic->gni_nic_hndl,
*(uint16_t *)val);
fastlock_release(&nic->lock);
@@ -400,6 +448,45 @@ __gnix_dom_ops_set_val(struct fid *fid, dom_ops_val_t t, void *val)
case GNI_MR_CACHE_LAZY_DEREG:
domain->mr_cache_attr.lazy_deregistration = *(int32_t *)val;
break;
+ case GNI_MR_CACHE:
+ if (val != NULL) {
+ GNIX_DEBUG(FI_LOG_DOMAIN, "user provided value=%s\n",
+ *(char **) val);
+
+ type = __gnix_string_to_mr_type(*(const char **) val);
+ if (type < 0 || type >= GNIX_MR_MAX_TYPE)
+ return -FI_EINVAL;
+
+ GNIX_DEBUG(FI_LOG_DOMAIN, "setting domain mr type to %s\n",
+ __gnix_mr_type_to_str[type]);
+
+ ret = _gnix_open_cache(domain, type);
+ if (ret != FI_SUCCESS)
+ return -FI_EINVAL;
+ }
+ break;
+ case GNI_MR_HARD_REG_LIMIT:
+ domain->mr_cache_attr.hard_reg_limit = *(int32_t *) val;
+ break;
+ case GNI_MR_SOFT_REG_LIMIT:
+ domain->mr_cache_attr.soft_reg_limit = *(int32_t *) val;
+ break;
+ case GNI_MR_HARD_STALE_REG_LIMIT:
+ domain->mr_cache_attr.hard_stale_limit = *(int32_t *) val;
+ break;
+ case GNI_MR_UDREG_REG_LIMIT:
+ if (*(int32_t *) val < 0)
+ return -FI_EINVAL;
+ domain->udreg_reg_limit = *(int32_t *) val;
+ break;
+ case GNI_XPMEM_ENABLE:
+#if HAVE_XPMEM
+ domain->params.xpmem_enabled = *(bool *)val;
+#else
+ GNIX_WARN(FI_LOG_DOMAIN,
+ "GNI provider XPMEM support not configured\n");
+#endif
+ break;
default:
GNIX_WARN(FI_LOG_DOMAIN, ("Invalid dom_ops_val\n"));
return -FI_EINVAL;
@@ -411,7 +498,8 @@ __gnix_dom_ops_set_val(struct fid *fid, dom_ops_val_t t, void *val)
static struct fi_gni_ops_domain gnix_ops_domain = {
.set_val = __gnix_dom_ops_set_val,
- .get_val = __gnix_dom_ops_get_val
+ .get_val = __gnix_dom_ops_get_val,
+ .flush_cache = __gnix_dom_ops_flush_cache,
};
DIRECT_FN int gnix_domain_bind(struct fid_domain *domain, struct fid *fid,
@@ -486,9 +574,12 @@ DIRECT_FN int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info,
domain->mr_cache_attr.reg_context = (void *) domain;
domain->mr_cache_attr.dereg_context = NULL;
domain->mr_cache_attr.destruct_context = NULL;
+ domain->mr_cache_attr.notifier = &fabric_priv->mr_notifier;
domain->mr_cache = NULL;
fastlock_init(&domain->mr_cache_lock);
+ domain->udreg_reg_limit = 4096;
+
dlist_init(&domain->nic_list);
dlist_init(&domain->list);
@@ -514,6 +605,11 @@ DIRECT_FN int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info,
domain->params.mbox_msg_maxsize = default_mbox_msg_maxsize;
domain->params.max_retransmits = default_max_retransmits;
domain->params.err_inject_count = default_err_inject_count;
+#if HAVE_XPMEM
+ domain->params.xpmem_enabled = true;
+#else
+ domain->params.xpmem_enabled = false;
+#endif
domain->gni_tx_cq_size = default_tx_cq_size;
domain->gni_rx_cq_size = default_rx_cq_size;
@@ -528,8 +624,13 @@ DIRECT_FN int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info,
domain->control_progress = info->domain_attr->control_progress;
domain->data_progress = info->domain_attr->data_progress;
+ domain->thread_model = info->domain_attr->threading;
+ domain->mr_is_init = 0;
+
fastlock_init(&domain->cm_nic_lock);
+ _gnix_open_cache(domain, GNIX_DEFAULT_CACHE_TYPE);
+
*dom = &domain->domain_fid;
return FI_SUCCESS;
diff --git a/prov/gni/src/gnix_ep.c b/prov/gni/src/gnix_ep.c
index 39cd162..e43c730 100644
--- a/prov/gni/src/gnix_ep.c
+++ b/prov/gni/src/gnix_ep.c
@@ -46,36 +46,124 @@
#include "gnix_ep.h"
#include "gnix_hashtable.h"
#include "gnix_vc.h"
+#include "gnix_vector.h"
#include "gnix_msg.h"
#include "gnix_rma.h"
#include "gnix_atomic.h"
#include "gnix_cntr.h"
+#include "gnix_xpmem.h"
/*******************************************************************************
* gnix_fab_req freelist functions
*
- * These are wrappers around the gnix_s_freelist
+ * These are wrappers around the gnix_freelist
*
******************************************************************************/
#define GNIX_FAB_REQ_FL_MIN_SIZE 100
#define GNIX_FAB_REQ_FL_REFILL_SIZE 10
+void _gnix_ep_htd_pool_init(struct gnix_fid_ep *ep)
+{
+ int ret, i;
+ struct fid_mr *auto_mr = NULL;
+ uint8_t *htd_bufs;
+ struct gnix_htd_buf *htd_buf_list;
+
+ assert(ep);
+
+ htd_bufs = malloc(GNIX_HTD_POOL_SIZE * GNIX_HTD_BUF_SZ);
+ if (htd_bufs == NULL) {
+ GNIX_FATAL(FI_LOG_EP_DATA, "\n");
+ }
+
+ htd_buf_list = malloc(GNIX_HTD_POOL_SIZE * sizeof(struct gnix_htd_buf));
+ if (htd_buf_list == NULL) {
+ GNIX_FATAL(FI_LOG_EP_DATA, "\n");
+ }
+
+ ep->htd_pool.buf_ptr = (void *) htd_bufs;
+ ep->htd_pool.sl_ptr = (void *) htd_buf_list;
+
+ ret = gnix_mr_reg(&ep->domain->domain_fid.fid, htd_bufs,
+ GNIX_HTD_BUF_SZ * GNIX_HTD_POOL_SIZE,
+ FI_READ | FI_WRITE, 0, 0, 0, &auto_mr, NULL);
+
+ if (unlikely(ret != FI_SUCCESS)) {
+ GNIX_DEBUG(FI_LOG_EP_DATA, "gnix_mr_req returned: %s\n",
+ fi_strerror(-ret));
+ } else {
+ ep->htd_pool.md = container_of(auto_mr, struct gnix_fid_mem_desc, mr_fid);
+ }
+
+ slist_init(&ep->htd_pool.sl);
+
+ for (i = 0; i < GNIX_HTD_POOL_SIZE; i++) {
+ GNIX_DEBUG(FI_LOG_EP_DATA, "htd_bufs + (%d * GNIX_HTD_BUF_SZ) = %p\n",
+ i, htd_bufs + (i * GNIX_HTD_BUF_SZ));
+ htd_buf_list[i].buf = htd_bufs + (i * GNIX_HTD_BUF_SZ);
+ slist_insert_tail(&htd_buf_list[i].e, &ep->htd_pool.sl);
+ }
+
+ fastlock_init(&ep->htd_pool.lock);
+
+ ep->htd_pool.enabled = true;
+}
+
+void _gnix_ep_htd_pool_fini(struct gnix_fid_ep *ep)
+{
+ int ret;
+
+ assert(ep);
+
+ if (ep->htd_pool.enabled == false)
+ return;
+
+ ret = fi_close(&ep->htd_pool.md->mr_fid.fid);
+
+ if (ret != FI_SUCCESS) {
+ GNIX_DEBUG(FI_LOG_EP_DATA, "fi_close returned: %s\n", fi_strerror(-ret));
+ }
+
+ if (ep->htd_pool.buf_ptr != NULL) {
+ free(ep->htd_pool.buf_ptr);
+ ep->htd_pool.buf_ptr = NULL;
+ }
+
+ if (ep->htd_pool.sl_ptr != NULL) {
+ free(ep->htd_pool.sl_ptr);
+ ep->htd_pool.sl_ptr = NULL;
+ }
+
+ ep->htd_pool.enabled = false;
+}
+
static int __fr_freelist_init(struct gnix_fid_ep *ep)
{
+ int ret;
+
assert(ep);
- return _gnix_sfl_init_ts(sizeof(struct gnix_fab_req),
- offsetof(struct gnix_fab_req, slist),
- GNIX_FAB_REQ_FL_MIN_SIZE,
- GNIX_FAB_REQ_FL_REFILL_SIZE,
- 0, 0, &ep->fr_freelist);
+ ret = _gnix_fl_init_ts(sizeof(struct gnix_fab_req),
+ offsetof(struct gnix_fab_req, dlist),
+ GNIX_FAB_REQ_FL_MIN_SIZE,
+ GNIX_FAB_REQ_FL_REFILL_SIZE,
+ 0, 0, &ep->fr_freelist);
+
+ if (ret != FI_SUCCESS) {
+ GNIX_DEBUG(FI_LOG_EP_DATA, "_gnix_fl_init_ts returned: %s\n",
+ fi_strerror(-ret));
+ return ret;
+ }
+
+ return ret;
}
static void __fr_freelist_destroy(struct gnix_fid_ep *ep)
{
assert(ep);
- _gnix_sfl_destroy(&ep->fr_freelist);
+
+ _gnix_fl_destroy(&ep->fr_freelist);
}
/*******************************************************************************
@@ -117,44 +205,41 @@ static inline ssize_t __ep_recvv(struct fid_ep *ep, const struct iovec *iov,
{
struct gnix_fid_ep *ep_priv;
- if (!ep || !iov || count != 1) {
+ if (!ep || !iov || count > GNIX_MAX_MSG_IOV_LIMIT) {
return -FI_EINVAL;
}
ep_priv = container_of(ep, struct gnix_fid_ep, ep_fid);
assert(GNIX_EP_RDM_DGM_MSG(ep_priv->type));
- return _gnix_recv(ep_priv, (uint64_t)iov[0].iov_base, iov[0].iov_len,
- desc ? desc[0] : NULL, src_addr, context,
- ep_priv->op_flags | flags, tag, ignore);
+ if (count <= 1) {
+ return _gnix_recv(ep_priv, (uint64_t)iov[0].iov_base,
+ iov[0].iov_len, desc ? desc[0] : NULL,
+ src_addr, context,
+ ep_priv->op_flags | flags, tag, ignore);
+ }
+
+ return _gnix_recvv(ep_priv, iov, desc, count, src_addr,
+ context, ep_priv->op_flags | flags, ignore, tag);
}
static inline ssize_t __ep_recvmsg(struct fid_ep *ep, const struct fi_msg *msg,
uint64_t flags, uint64_t tag,
uint64_t ignore)
{
- struct gnix_fid_ep *ep_priv;
- uint64_t recv_addr, recv_len;
+ struct iovec iov;
+
+ iov.iov_base = NULL;
+ iov.iov_len = 0;
- if (!ep || !msg) {
+ if (!msg) {
return -FI_EINVAL;
}
/* msg_iov can be undefined when using FI_PEEK, etc. */
- if (msg->msg_iov) {
- recv_addr = (uint64_t)msg->msg_iov[0].iov_base;
- recv_len = (uint64_t)msg->msg_iov[0].iov_len;
- } else {
- recv_addr = 0;
- recv_len = 0;
- }
-
- ep_priv = container_of(ep, struct gnix_fid_ep, ep_fid);
- assert(GNIX_EP_RDM_DGM_MSG(ep_priv->type));
-
- return _gnix_recv(ep_priv, recv_addr, recv_len,
- msg->desc ? msg->desc[0] : NULL,
- msg->addr, msg->context, flags, tag, ignore);
+ return __ep_recvv(ep, msg->msg_iov ? msg->msg_iov : &iov, msg->desc,
+ msg->iov_count, msg->addr, msg->context, flags, tag,
+ ignore);
}
static inline ssize_t __ep_send(struct fid_ep *ep, const void *buf, size_t len,
@@ -180,16 +265,22 @@ static inline ssize_t __ep_sendv(struct fid_ep *ep, const struct iovec *iov,
{
struct gnix_fid_ep *gnix_ep;
- if (!ep || !iov || count != 1) {
+ if (!ep || !iov || !count || count > GNIX_MAX_MSG_IOV_LIMIT) {
return -FI_EINVAL;
}
gnix_ep = container_of(ep, struct gnix_fid_ep, ep_fid);
assert(GNIX_EP_RDM_DGM_MSG(gnix_ep->type));
- return _gnix_send(gnix_ep, (uint64_t)iov[0].iov_base, iov[0].iov_len,
- desc ? desc[0] : NULL, dest_addr, context,
- gnix_ep->op_flags | flags, 0, tag);
+ if (count == 1) {
+ return _gnix_send(gnix_ep, (uint64_t)iov[0].iov_base,
+ iov[0].iov_len, desc ? desc[0] : NULL,
+ dest_addr, context, gnix_ep->op_flags | flags,
+ 0, tag);
+ }
+
+ return _gnix_sendv(gnix_ep, iov, desc, count, dest_addr, context,
+ gnix_ep->op_flags | flags, tag);
}
static inline ssize_t __ep_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
@@ -197,10 +288,16 @@ static inline ssize_t __ep_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
{
struct gnix_fid_ep *gnix_ep;
- if (!ep || !msg || !msg->msg_iov || msg->iov_count != 1) {
+ if (!ep || !msg || !msg->msg_iov || !msg->iov_count) {
return -FI_EINVAL;
}
+ /* Must check the iov count here, can't send msg->data to sendv */
+ if (msg->iov_count > 1) {
+ return __ep_sendv(ep, msg->msg_iov, msg->desc, msg->iov_count,
+ msg->addr, msg->context, flags, tag);
+ }
+
gnix_ep = container_of(ep, struct gnix_fid_ep, ep_fid);
assert(GNIX_EP_RDM_DGM_MSG(gnix_ep->type));
@@ -211,7 +308,7 @@ static inline ssize_t __ep_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
}
static inline ssize_t __ep_inject(struct fid_ep *ep, const void *buf,
- size_t len, fi_addr_t dest_addr,
+ size_t len, uint64_t data, fi_addr_t dest_addr,
uint64_t flags, uint64_t tag)
{
struct gnix_fid_ep *gnix_ep;
@@ -228,7 +325,7 @@ static inline ssize_t __ep_inject(struct fid_ep *ep, const void *buf,
GNIX_SUPPRESS_COMPLETION | flags);
return _gnix_send(gnix_ep, (uint64_t)buf, len, NULL, dest_addr,
- NULL, inject_flags, 0, tag);
+ NULL, inject_flags, data, tag);
}
static inline ssize_t __ep_senddata(struct fid_ep *ep, const void *buf,
@@ -252,6 +349,143 @@ static inline ssize_t __ep_senddata(struct fid_ep *ep, const void *buf,
context, sd_flags, data, tag);
}
+static void __gnix_vc_destroy_ht_entry(void *val)
+{
+ struct gnix_vc *vc = (struct gnix_vc *) val;
+
+ _gnix_vc_destroy(vc);
+}
+
+/*******************************************************************************
+ * EP vc initialization helper
+ ******************************************************************************/
+
+static inline int __gnix_ep_init_vc(struct gnix_fid_ep *ep_priv)
+{
+ int ret;
+ enum fi_av_type type;
+ gnix_hashtable_attr_t gnix_ht_attr;
+ gnix_vec_attr_t gnix_vec_attr;
+
+ type = ep_priv->av->type;
+
+ if (type == FI_AV_TABLE) {
+ /* Use array to store EP VCs when using FI_AV_TABLE. */
+ ep_priv->vc_table = calloc(1, sizeof(struct gnix_vector));
+ if(ep_priv->vc_table == NULL)
+ return -FI_ENOMEM;
+
+ gnix_vec_attr.vec_initial_size =
+ ep_priv->domain->params.ct_init_size;
+ /* TODO: ep_priv->domain->params.ct_max_size; */
+ gnix_vec_attr.vec_maximum_size = 1024*1024;
+ gnix_vec_attr.vec_increase_step = ep_priv->domain->params.ct_step;
+ gnix_vec_attr.vec_increase_type = GNIX_VEC_INCREASE_MULT;
+ gnix_vec_attr.vec_internal_locking = GNIX_VEC_UNLOCKED;
+
+ ret = _gnix_vec_init(ep_priv->vc_table, &gnix_vec_attr);
+ GNIX_DEBUG(FI_LOG_EP_CTRL,
+ "ep_priv->vc_table = %p, ep_priv->vc_table->vector = %p\n",
+ ep_priv->vc_table, ep_priv->vc_table->vector);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "_gnix_vec_init returned %s\n",
+ fi_strerror(ret));
+ goto err;
+ }
+ } else {
+ /* Use hash table to store EP VCs when using FI_AV_MAP. */
+ ep_priv->vc_ht = calloc(1, sizeof(struct gnix_hashtable));
+ if (ep_priv->vc_ht == NULL)
+ return -FI_ENOMEM;
+
+ gnix_ht_attr.ht_initial_size =
+ ep_priv->domain->params.ct_init_size;
+ gnix_ht_attr.ht_maximum_size =
+ ep_priv->domain->params.ct_max_size;
+ gnix_ht_attr.ht_increase_step = ep_priv->domain->params.ct_step;
+ gnix_ht_attr.ht_increase_type = GNIX_HT_INCREASE_MULT;
+ gnix_ht_attr.ht_collision_thresh = 500;
+ gnix_ht_attr.ht_hash_seed = 0xdeadbeefbeefdead;
+ gnix_ht_attr.ht_internal_locking = 0;
+ gnix_ht_attr.destructor = __gnix_vc_destroy_ht_entry;
+
+ ret = _gnix_ht_init(ep_priv->vc_ht, &gnix_ht_attr);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gnix_ht_init returned %s\n",
+ fi_strerror(-ret));
+
+ goto err;
+ }
+ }
+
+ dlist_init(&ep_priv->unmapped_vcs);
+
+ return FI_SUCCESS;
+
+err:
+ if (type == FI_AV_TABLE) {
+ free(ep_priv->vc_table);
+ ep_priv->vc_table = NULL;
+ } else {
+ free(ep_priv->vc_ht);
+ ep_priv->vc_ht = NULL;
+ }
+
+ return ret;
+}
+
+static inline int __gnix_ep_fini_vc(struct gnix_fid_ep *ep)
+{
+ int ret;
+ GNIX_VECTOR_ITERATOR(ep->vc_table, iter);
+ struct gnix_vc *vc;
+
+ /* Free unmapped VCs. */
+ dlist_for_each(&ep->unmapped_vcs, vc, list) {
+ _gnix_vc_destroy(vc);
+ }
+
+ if (!ep->av) {
+ /* No AV bound, no mapped VCs clean up. */
+ return FI_SUCCESS;
+ }
+
+ if (ep->av->type == FI_AV_TABLE) {
+ /* Destroy all VCs */
+ while ((vc = (struct gnix_vc *)
+ _gnix_vec_iterator_next(&iter))) {
+ _gnix_vec_remove_at(ep->vc_table,
+ GNIX_VECTOR_ITERATOR_IDX(iter));
+ _gnix_vc_destroy(vc);
+ }
+
+ /* Destroy vector storage */
+ ret = _gnix_vec_close(ep->vc_table);
+ if (ret == FI_SUCCESS) {
+ free(ep->vc_table);
+ ep->vc_table = NULL;
+ } else {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gnix_vec_close returned %s\n",
+ fi_strerror(-ret));
+ }
+ } else {
+ /* Destroy VC storage, it automatically tears down VCs */
+ ret = _gnix_ht_destroy(ep->vc_ht);
+ if (ret == FI_SUCCESS) {
+ free(ep->vc_ht);
+ ep->vc_ht = NULL;
+ } else {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gnix_ht_destroy returned %s\n",
+ fi_strerror(-ret));
+ }
+ }
+
+ return FI_SUCCESS;
+}
+
/*******************************************************************************
* EP messaging API function implementations.
******************************************************************************/
@@ -305,7 +539,7 @@ DIRECT_FN STATIC ssize_t gnix_ep_sendmsg(struct fid_ep *ep,
DIRECT_FN STATIC ssize_t gnix_ep_msg_inject(struct fid_ep *ep, const void *buf,
size_t len, fi_addr_t dest_addr)
{
- return __ep_inject(ep, buf, len, dest_addr, 0, 0);
+ return __ep_inject(ep, buf, len, 0, dest_addr, 0, 0);
}
DIRECT_FN STATIC ssize_t gnix_ep_senddata(struct fid_ep *ep, const void *buf,
@@ -375,7 +609,7 @@ gnix_ep_readv(struct fid_ep *ep, const struct iovec *iov, void **desc,
struct gnix_fid_ep *gnix_ep;
uint64_t flags;
- if (!ep || !iov || !desc || count != 1) {
+ if (!ep || !iov || !desc || count > GNIX_MAX_RMA_IOV_LIMIT) {
return -FI_EINVAL;
}
@@ -442,7 +676,7 @@ gnix_ep_writev(struct fid_ep *ep, const struct iovec *iov, void **desc,
struct gnix_fid_ep *gnix_ep;
uint64_t flags;
- if (!ep || !iov || !desc || count != 1) {
+ if (!ep || !iov || !desc || count > GNIX_MAX_RMA_IOV_LIMIT) {
return -FI_EINVAL;
}
@@ -461,8 +695,9 @@ DIRECT_FN STATIC ssize_t gnix_ep_writemsg(struct fid_ep *ep, const struct fi_msg
{
struct gnix_fid_ep *gnix_ep;
- if (!ep || !msg || !msg->msg_iov || !msg->rma_iov || !msg->desc ||
- msg->iov_count != 1 || msg->rma_iov_count != 1 ||
+ if (!ep || !msg || !msg->msg_iov || !msg->rma_iov ||
+ msg->iov_count != 1 ||
+ msg->rma_iov_count > GNIX_MAX_RMA_IOV_LIMIT ||
msg->rma_iov[0].len > msg->msg_iov[0].iov_len) {
return -FI_EINVAL;
}
@@ -474,7 +709,7 @@ DIRECT_FN STATIC ssize_t gnix_ep_writemsg(struct fid_ep *ep, const struct fi_msg
return _gnix_rma(gnix_ep, GNIX_FAB_RQ_RDMA_WRITE,
(uint64_t)msg->msg_iov[0].iov_base,
- msg->msg_iov[0].iov_len, msg->desc[0],
+ msg->msg_iov[0].iov_len, msg->desc ? msg->desc[0] : NULL,
msg->addr, msg->rma_iov[0].addr, msg->rma_iov[0].key,
msg->context, flags, msg->data);
}
@@ -571,7 +806,7 @@ DIRECT_FN STATIC ssize_t gnix_ep_trecvv(struct fid_ep *ep,
void *context)
{
return __ep_recvv(ep, iov, desc, count, src_addr, context,
- FI_TAGGED, tag, ignore);
+ FI_TAGGED, tag, ignore);
}
DIRECT_FN STATIC ssize_t gnix_ep_trecvmsg(struct fid_ep *ep,
@@ -632,7 +867,7 @@ DIRECT_FN STATIC ssize_t gnix_ep_tsendv(struct fid_ep *ep,
uint64_t tag, void *context)
{
return __ep_sendv(ep, iov, desc, count, dest_addr, context,
- FI_TAGGED, tag);
+ FI_TAGGED, tag);
}
DIRECT_FN STATIC ssize_t gnix_ep_tsendmsg(struct fid_ep *ep,
@@ -659,7 +894,7 @@ DIRECT_FN STATIC ssize_t gnix_ep_tinject(struct fid_ep *ep, const void *buf,
size_t len, fi_addr_t dest_addr,
uint64_t tag)
{
- return __ep_inject(ep, buf, len, dest_addr, FI_TAGGED, tag);
+ return __ep_inject(ep, buf, len, 0, dest_addr, FI_TAGGED, tag);
}
DIRECT_FN STATIC ssize_t gnix_ep_tsenddata(struct fid_ep *ep, const void *buf,
@@ -689,7 +924,8 @@ DIRECT_FN STATIC ssize_t gnix_ep_tinjectdata(struct fid_ep *ep, const void *buf,
size_t len, uint64_t data,
fi_addr_t dest_addr, uint64_t tag)
{
- return -FI_ENOSYS;
+ return __ep_inject(ep, buf, len, data, dest_addr,
+ FI_TAGGED | FI_REMOTE_CQ_DATA, tag);
}
@@ -733,6 +969,54 @@ DIRECT_FN STATIC int gnix_ep_cmp_atomic_valid(struct fid_ep *ep,
0 : -FI_ENOENT;
}
+size_t
+__gnix_fabric_ops_native_amo(struct fid_ep *ep, const void *buf, size_t count,
+ void *desc, void *result, void *result_desc,
+ fi_addr_t dest_addr,
+ uint64_t addr, uint64_t key,
+ enum fi_datatype datatype,
+ int req_type,
+ void *context)
+{
+ struct gnix_fid_ep *gnix_ep;
+ struct fi_msg_atomic msg;
+ struct fi_ioc msg_iov;
+ struct fi_rma_ioc rma_iov;
+ struct fi_ioc result_iov;
+ uint64_t flags;
+
+ if (!ep)
+ return -FI_EINVAL;
+ if ((req_type < 0) || (req_type > GNIX_FAB_RQ_MAX_TYPES) ||
+ (req_type >= GNIX_FAB_RQ_END_NON_NATIVE &&
+ req_type < GNIX_FAB_RQ_START_NATIVE))
+ return -FI_EINVAL;
+
+ gnix_ep = container_of(ep, struct gnix_fid_ep, ep_fid);
+ assert(GNIX_EP_RDM_DGM_MSG(gnix_ep->type));
+
+ msg_iov.addr = (void *)buf;
+ msg_iov.count = count;
+ rma_iov.addr = addr;
+ rma_iov.count = 1;
+ rma_iov.key = key;
+ msg.msg_iov = &msg_iov;
+ msg.desc = &desc;
+ msg.iov_count = 1;
+ msg.addr = dest_addr;
+ msg.rma_iov = &rma_iov;
+ msg.datatype = datatype;
+ msg.op = FI_ATOMIC_OP_LAST; /* not FI_ATOMIC_OP */
+ msg.context = context;
+ result_iov.addr = result;
+ result_iov.count = 1;
+
+ flags = gnix_ep->op_flags | GNIX_ATOMIC_WRITE_FLAGS_DEF;
+
+ return _gnix_atomic(gnix_ep, req_type, &msg, NULL,
+ NULL, 0, &result_iov, &result_desc, 1, flags);
+}
+
DIRECT_FN STATIC ssize_t
gnix_ep_atomic_write(struct fid_ep *ep, const void *buf, size_t count,
void *desc, fi_addr_t dest_addr, uint64_t addr,
@@ -780,7 +1064,7 @@ gnix_ep_atomic_writev(struct fid_ep *ep, const struct fi_ioc *iov, void **desc,
uint64_t key, enum fi_datatype datatype, enum fi_op op,
void *context)
{
- if (!iov || count != 1) {
+ if (!iov || count > 1) {
return -FI_EINVAL;
}
@@ -906,7 +1190,7 @@ gnix_ep_atomic_readwritev(struct fid_ep *ep, const struct fi_ioc *iov,
enum fi_datatype datatype, enum fi_op op,
void *context)
{
- if (!iov || count != 1 || !resultv)
+ if (!iov || count > 1 || !resultv)
return -FI_EINVAL;
return gnix_ep_atomic_readwrite(ep, iov[0].addr, iov[0].count,
@@ -1007,7 +1291,7 @@ DIRECT_FN STATIC ssize_t gnix_ep_atomic_compwritev(struct fid_ep *ep,
enum fi_op op,
void *context)
{
- if (!iov || count != 1 || !resultv || !comparev)
+ if (!iov || count > 1 || !resultv || !comparev)
return -FI_EINVAL;
return gnix_ep_atomic_compwrite(ep, iov[0].addr, iov[0].count,
@@ -1070,6 +1354,14 @@ DIRECT_FN STATIC int gnix_ep_control(fid_t fid, int command, void *arg)
*/
case FI_ENABLE:
if (GNIX_EP_RDM_DGM(ep->type)) {
+ if ((ep->send_cq && ep->tx_enabled)) {
+ ret = -FI_EOPBADSTATE;
+ goto err;
+ }
+ if ((ep->recv_cq && ep->rx_enabled)) {
+ ret = -FI_EOPBADSTATE;
+ goto err;
+ }
ret = _gnix_vc_cm_init(ep->cm_nic);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
@@ -1089,10 +1381,12 @@ DIRECT_FN STATIC int gnix_ep_control(fid_t fid, int command, void *arg)
if (ep->recv_cq)
ep->rx_enabled = true;
}
+
+ _gnix_ep_htd_pool_init(ep);
break;
- case FI_GETFIDFLAG:
- case FI_SETFIDFLAG:
+ case FI_GETOPSFLAG:
+ case FI_SETOPSFLAG:
case FI_ALIAS:
default:
return -FI_ENOSYS;
@@ -1128,7 +1422,7 @@ static int __destruct_tag_storages(struct gnix_fid_ep *ep)
static void __ep_destruct(void *obj)
{
- int __attribute__((unused)) ret;
+ int ret;
struct gnix_fid_domain *domain;
struct gnix_nic *nic;
struct gnix_fid_av *av;
@@ -1138,29 +1432,23 @@ static void __ep_destruct(void *obj)
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
- /*
- * clean up any vc hash table or vector,
- * remove entry from addr_to_ep ht.
- * any outstanding GNI internal requests on
- * the VC's will be completed prior to
- * destroying the VC entries in the ht.
- */
-
- if (GNIX_EP_RDM_DGM(ep->type)) {
-
+ if (ep->av) {
+ /* Remove EP from CM NIC lookup list. */
key_ptr = (gnix_ht_key_t *)&ep->my_name.gnix_addr;
ret = _gnix_ht_remove(ep->cm_nic->addr_to_ep_ht,
*key_ptr);
- if (ep->vc_ht != NULL) {
- ret = _gnix_ht_destroy(ep->vc_ht);
- if (ret == FI_SUCCESS) {
- free(ep->vc_ht);
- ep->vc_ht = NULL;
- } else {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "_gnix_ht_destroy returned %s\n",
- fi_strerror(-ret));
- }
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gnix_ht_remove returned %s\n",
+ fi_strerror(-ret));
+ }
+
+ /* Destroy EP VC storage. */
+ ret = __gnix_ep_fini_vc(ep);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gnix_ht_remove returned %s\n",
+ fi_strerror(-ret));
}
}
@@ -1184,19 +1472,37 @@ static void __ep_destruct(void *obj)
_gnix_ref_put(ep->recv_cntr);
}
+ if (ep->write_cntr) {
+ _gnix_cntr_poll_nic_rem(ep->write_cntr, ep->nic);
+ _gnix_ref_put(ep->write_cntr);
+ }
+
if (ep->read_cntr) {
_gnix_cntr_poll_nic_rem(ep->read_cntr, ep->nic);
_gnix_ref_put(ep->read_cntr);
}
- if (ep->write_cntr) {
- _gnix_cntr_poll_nic_rem(ep->write_cntr, ep->nic);
- _gnix_ref_put(ep->write_cntr);
+ if (ep->rwrite_cntr) {
+ _gnix_cntr_poll_nic_rem(ep->rwrite_cntr, ep->nic);
+ _gnix_ref_put(ep->rwrite_cntr);
+ }
+
+ if (ep->rread_cntr) {
+ _gnix_cntr_poll_nic_rem(ep->rread_cntr, ep->nic);
+ _gnix_ref_put(ep->rread_cntr);
}
if (ep->stx_ctx)
_gnix_ref_put(ep->stx_ctx);
+ if (ep->xpmem_hndl) {
+ ret = _gnix_xpmem_handle_destroy(ep->xpmem_hndl);
+ if (ret != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gnix_xpmem_handle_destroy returned %s\n",
+ fi_strerror(-ret));
+ }
+
domain = ep->domain;
assert(domain != NULL);
_gnix_ref_put(domain);
@@ -1228,6 +1534,7 @@ static void __ep_destruct(void *obj)
*/
__fr_freelist_destroy(ep);
+ _gnix_ep_htd_pool_fini(ep);
free(ep);
}
@@ -1253,7 +1560,7 @@ static int gnix_ep_close(fid_t fid)
DIRECT_FN STATIC int gnix_ep_bind(fid_t fid, struct fid *bfid, uint64_t flags)
{
- int ret = FI_SUCCESS;
+ int ret;
struct gnix_fid_ep *ep;
struct gnix_fid_av *av;
struct gnix_fid_cq *cq;
@@ -1263,9 +1570,9 @@ DIRECT_FN STATIC int gnix_ep_bind(fid_t fid, struct fid *bfid, uint64_t flags)
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
ep = container_of(fid, struct gnix_fid_ep, ep_fid.fid);
-
- if (!bfid)
- return -FI_EINVAL;
+ ret = ofi_ep_bind_valid(&gnix_prov, bfid, flags);
+ if (ret)
+ return ret;
switch (bfid->fclass) {
case FI_CLASS_EQ:
@@ -1316,6 +1623,7 @@ DIRECT_FN STATIC int gnix_ep_bind(fid_t fid, struct fid *bfid, uint64_t flags)
break;
}
ep->av = av;
+ __gnix_ep_init_vc(ep);
_gnix_ref_get(ep->av);
break;
case FI_CLASS_CNTR:
@@ -1353,6 +1661,20 @@ DIRECT_FN STATIC int gnix_ep_bind(fid_t fid, struct fid *bfid, uint64_t flags)
_gnix_ref_get(cntr);
}
+ if (flags & FI_WRITE) {
+ /* don't allow rebinding */
+ if (ep->write_cntr) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "cannot rebind write counter (%p)\n",
+ cntr);
+ ret = -FI_EINVAL;
+ break;
+ }
+ ep->write_cntr = cntr;
+ _gnix_cntr_poll_nic_add(cntr, ep->nic);
+ _gnix_ref_get(cntr);
+ }
+
if (flags & FI_READ) {
/* don't allow rebinding */
if (ep->read_cntr) {
@@ -1367,32 +1689,34 @@ DIRECT_FN STATIC int gnix_ep_bind(fid_t fid, struct fid *bfid, uint64_t flags)
_gnix_ref_get(cntr);
}
- if (flags & FI_WRITE) {
+ if (flags & FI_REMOTE_WRITE) {
/* don't allow rebinding */
- if (ep->write_cntr) {
+ if (ep->rwrite_cntr) {
GNIX_WARN(FI_LOG_EP_CTRL,
- "cannot rebind write counter (%p)\n",
+ "cannot rebind rwrite counter (%p)\n",
cntr);
ret = -FI_EINVAL;
break;
}
- ep->write_cntr = cntr;
+ ep->rwrite_cntr = cntr;
_gnix_cntr_poll_nic_add(cntr, ep->nic);
_gnix_ref_get(cntr);
}
- /* TODO: don't support this option right now,
- never should have gotten here since gni provider
- doesn't claim cap FI_RMA_EVENT. This
- option could be supported via Aries atomics
- or using SMSG cntrl messages */
-
- if ((flags & FI_REMOTE_WRITE) || (flags & FI_REMOTE_READ)) {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "unsupported counter flags (%p)\n",
- cntr);
- ret = -FI_ENOSYS;
+ if (flags & FI_REMOTE_READ) {
+ /* don't allow rebinding */
+ if (ep->rread_cntr) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "cannot rebind rread counter (%p)\n",
+ cntr);
+ ret = -FI_EINVAL;
+ break;
+ }
+ ep->rread_cntr = cntr;
+ _gnix_cntr_poll_nic_add(cntr, ep->nic);
+ _gnix_ref_get(cntr);
}
+
break;
case FI_CLASS_STX_CTX:
@@ -1420,11 +1744,35 @@ DIRECT_FN int gnix_pep_bind(fid_t fid, fid_t *bfid, uint64_t flags)
return -FI_ENOSYS;
}
-static void __gnix_vc_destroy_ht_entry(void *val)
+static int __gnix_ep_cm_nic_prep(struct gnix_fid_domain *domain,
+ struct fi_info *info,
+ uint32_t *cdm_id)
{
- struct gnix_vc *vc = (struct gnix_vc *) val;
+ int ret = FI_SUCCESS;
+ uint32_t name_type = GNIX_EPN_TYPE_UNBOUND;
+ struct gnix_ep_name *name;
+
+ if (info->src_addr &&
+ info->src_addrlen == sizeof(struct gnix_ep_name)) {
+ name = (struct gnix_ep_name *)info->src_addr;
+ if (name->name_type == GNIX_EPN_TYPE_BOUND) {
+ /* EP name includes user specified service/port */
+ *cdm_id = name->gnix_addr.cdm_id;
+ name_type = name->name_type;
+ }
+ }
- _gnix_vc_destroy(vc);
+ if (name_type == GNIX_EPN_TYPE_UNBOUND) {
+ ret = _gnix_cm_nic_create_cdm_id(domain, cdm_id);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "gnix_cm_nic_create_cdm_id returned %s\n",
+ fi_strerror(-ret));
+ return ret;
+ }
+ }
+
+ return ret;
}
/*
@@ -1436,6 +1784,7 @@ static int __gnix_ep_bound_prep(struct gnix_fid_domain *domain,
struct gnix_fid_ep *ep)
{
int ret = FI_SUCCESS;
+ uint32_t cdm_id;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
@@ -1443,8 +1792,14 @@ static int __gnix_ep_bound_prep(struct gnix_fid_domain *domain,
assert(info != NULL);
assert(ep != NULL);
+ ret = __gnix_ep_cm_nic_prep(domain, info, &cdm_id);
+ if (ret != FI_SUCCESS) {
+ return ret;
+ }
+
ret = _gnix_cm_nic_alloc(domain,
info,
+ cdm_id,
&ep->cm_nic);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
@@ -1462,16 +1817,16 @@ static int __gnix_ep_bound_prep(struct gnix_fid_domain *domain,
static void gnix_ep_caps(struct gnix_fid_ep *ep_priv, uint64_t caps)
{
- if (fi_recv_allowed(caps & ~FI_TAGGED))
+ if (ofi_recv_allowed(caps & ~FI_TAGGED))
ep_priv->ep_ops.msg_recv_allowed = 1;
- if (fi_send_allowed(caps & ~FI_TAGGED))
+ if (ofi_send_allowed(caps & ~FI_TAGGED))
ep_priv->ep_ops.msg_send_allowed = 1;
- if (fi_recv_allowed(caps & ~FI_MSG))
+ if (ofi_recv_allowed(caps & ~FI_MSG))
ep_priv->ep_ops.tagged_recv_allowed = 1;
- if (fi_send_allowed(caps & ~FI_MSG))
+ if (ofi_send_allowed(caps & ~FI_MSG))
ep_priv->ep_ops.tagged_send_allowed = 1;
}
@@ -1518,10 +1873,11 @@ DIRECT_FN int gnix_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
int ret = FI_SUCCESS;
- uint32_t cdm_id, seed;
+ int err_ret;
+ uint32_t cdm_id;
struct gnix_fid_domain *domain_priv;
struct gnix_fid_ep *ep_priv;
- gnix_hashtable_attr_t gnix_ht_attr;
+ struct gnix_ep_name *name;
gnix_ht_key_t *key_ptr;
bool free_list_inited = false;
@@ -1543,6 +1899,9 @@ DIRECT_FN int gnix_ep_open(struct fid_domain *domain, struct fi_info *info,
if (!ep_priv)
return -FI_ENOMEM;
+
+ ep_priv->requires_lock = (domain_priv->thread_model !=
+ FI_THREAD_COMPLETION);
ret = __init_tag_storages(ep_priv, GNIX_TAG_LIST);
if (ret) {
goto err;
@@ -1558,12 +1917,10 @@ DIRECT_FN int gnix_ep_open(struct fid_domain *domain, struct fi_info *info,
_gnix_ref_init(&ep_priv->ref_cnt, 1, __ep_destruct);
- fastlock_init(&ep_priv->recv_comp_lock);
fastlock_init(&ep_priv->recv_queue_lock);
fastlock_init(&ep_priv->tagged_queue_lock);
- slist_init(&ep_priv->pending_recv_comp_queue);
- ep_priv->caps = info->caps & GNIX_EP_RDM_CAPS;
+ ep_priv->caps = info->caps & GNIX_EP_RDM_CAPS_FULL;
if (info->tx_attr)
ep_priv->op_flags = info->tx_attr->op_flags;
@@ -1590,8 +1947,22 @@ DIRECT_FN int gnix_ep_open(struct fid_domain *domain, struct fi_info *info,
ep_priv->ep_fid.cm = &gnix_cm_ops;
+ /*
+ * try out XPMEM
+ */
+
+ ret = _gnix_xpmem_handle_create(domain_priv,
+ &ep_priv->xpmem_hndl);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "_gnix_xpmem_handl_create returned %s\n",
+ fi_strerror(-ret));
+ }
+
if (GNIX_EP_RDM_DGM(ep_priv->type)) {
- if (info->src_addr != NULL) {
+ name = (struct gnix_ep_name *)info->src_addr;
+ if ((name != NULL) &&
+ (name->name_type == GNIX_EPN_TYPE_BOUND)) {
+
ret = __gnix_ep_bound_prep(domain_priv,
info,
ep_priv);
@@ -1611,8 +1982,16 @@ DIRECT_FN int gnix_ep_open(struct fid_domain *domain, struct fi_info *info,
* to reduce demand on Aries hw resources.
*/
if (domain_priv->cm_nic == NULL) {
+ __gnix_ep_cm_nic_prep(domain_priv, info,
+ &cdm_id);
+ if (ret != FI_SUCCESS) {
+ fastlock_release(
+ &domain_priv->cm_nic_lock);
+ goto err;
+ }
ret = _gnix_cm_nic_alloc(domain_priv,
info,
+ cdm_id,
&domain_priv->cm_nic);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
@@ -1637,17 +2016,7 @@ DIRECT_FN int gnix_ep_open(struct fid_domain *domain, struct fi_info *info,
ep_priv->my_name.cm_nic_cdm_id =
ep_priv->cm_nic->my_name.gnix_addr.cdm_id;
- ret = _gnix_cm_nic_get_cdm_seed_set(domain_priv, 1,
- &seed);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "gnix_cdm_nic_get_cdm_seed_set returned %s\n",
- fi_strerror(-ret));
- goto err;
- }
- ret = _gnix_cm_nic_create_cdm_id(domain_priv,
- seed,
- &cdm_id);
+ ret = _gnix_cm_nic_create_cdm_id(domain_priv, &cdm_id);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"gnix_cm_nic_create_cdm_id returned %s\n",
@@ -1667,32 +2036,11 @@ DIRECT_FN int gnix_ep_open(struct fid_domain *domain, struct fi_info *info,
ret);
goto err;
}
-
- gnix_ht_attr.ht_initial_size = domain_priv->params.ct_init_size;
- gnix_ht_attr.ht_maximum_size = domain_priv->params.ct_max_size;
- gnix_ht_attr.ht_increase_step = domain_priv->params.ct_step;
- gnix_ht_attr.ht_increase_type = GNIX_HT_INCREASE_MULT;
- gnix_ht_attr.ht_collision_thresh = 500;
- gnix_ht_attr.ht_hash_seed = 0xdeadbeefbeefdead;
- gnix_ht_attr.ht_internal_locking = 0;
- gnix_ht_attr.destructor = __gnix_vc_destroy_ht_entry;
-
- ep_priv->vc_ht = calloc(1, sizeof(struct gnix_hashtable));
- if (ep_priv->vc_ht == NULL)
- goto err;
- ret = _gnix_ht_init(ep_priv->vc_ht, &gnix_ht_attr);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "gnix_ht_init call returned %d\n",
- ret);
- goto err;
- }
- fastlock_init(&ep_priv->vc_ht_lock);
-
} else {
ep_priv->cm_nic = NULL;
ep_priv->vc = NULL;
}
+ fastlock_init(&ep_priv->vc_lock);
ep_priv->progress_fn = NULL;
ep_priv->rx_progress_fn = NULL;
@@ -1721,23 +2069,42 @@ DIRECT_FN int gnix_ep_open(struct fid_domain *domain, struct fi_info *info,
return ret;
err:
- __destruct_tag_storages(ep_priv);
+ if (ep_priv->xpmem_hndl) {
+ err_ret = _gnix_xpmem_handle_destroy(ep_priv->xpmem_hndl);
+ if (err_ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gnix_xpmem_handle_destroy returned %s\n",
+ fi_strerror(-err_ret));
+ }
+ }
+
+ err_ret = __destruct_tag_storages(ep_priv);
+ if (err_ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "__destruct_tag_stroages returned %s\n",
+ fi_strerror(-err_ret));
+ }
if (free_list_inited == true)
__fr_freelist_destroy(ep_priv);
- if (ep_priv->vc_ht != NULL) {
- _gnix_ht_destroy(ep_priv->vc_ht); /* may not be initialized but
- okay */
- free(ep_priv->vc_ht);
- ep_priv->vc_ht = NULL;
+ if (ep_priv->cm_nic != NULL) {
+ err_ret = _gnix_cm_nic_free(ep_priv->cm_nic);
+ if (err_ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gnix_cm_nic_free returned %s\n",
+ fi_strerror(-err_ret));
+ }
}
- if (ep_priv->cm_nic != NULL)
- ret = _gnix_cm_nic_free(ep_priv->cm_nic);
-
- if (ep_priv->nic != NULL)
- ret = _gnix_nic_free(ep_priv->nic);
+ if (ep_priv->nic != NULL) {
+ err_ret = _gnix_nic_free(ep_priv->nic);
+ if (err_ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gnix_nic_free returned %s\n",
+ fi_strerror(-err_ret));
+ }
+ }
free(ep_priv);
return ret;
@@ -1751,11 +2118,11 @@ DIRECT_FN int gnix_passive_ep_open(struct fid_fabric *fabric,
return -FI_ENOSYS;
}
-static int __match_context(struct slist_entry *item, const void *arg)
+static int __match_context(struct dlist_entry *item, const void *arg)
{
struct gnix_fab_req *req;
- req = container_of(item, struct gnix_fab_req, slist);
+ req = container_of(item, struct gnix_fab_req, dlist);
return req->user_context == arg;
}
@@ -1765,25 +2132,56 @@ static inline struct gnix_fab_req *__find_tx_req(
void *context)
{
struct gnix_fab_req *req = NULL;
- struct slist_entry *entry;
+ struct dlist_entry *entry;
struct gnix_vc *vc;
- GNIX_HASHTABLE_ITERATOR(ep->vc_ht, iter);
GNIX_DEBUG(FI_LOG_EP_CTRL, "searching VCs for the correct context to"
" cancel, context=%p", context);
- fastlock_acquire(&ep->vc_ht_lock);
- while ((vc = _gnix_ht_iterator_next(&iter))) {
- fastlock_acquire(&vc->tx_queue_lock);
- entry = slist_remove_first_match(&vc->tx_queue,
- __match_context, context);
- fastlock_release(&vc->tx_queue_lock);
- if (entry) {
- req = container_of(entry, struct gnix_fab_req, slist);
- break;
+ COND_ACQUIRE(ep->requires_lock, &ep->vc_lock);
+
+ if (ep->av->type == FI_AV_TABLE) {
+ GNIX_VECTOR_ITERATOR(ep->vc_table, iter);
+
+ while ((vc = (struct gnix_vc *)
+ _gnix_vec_iterator_next(&iter))) {
+ COND_ACQUIRE(vc->ep->requires_lock,
+ &vc->tx_queue_lock);
+ entry = dlist_remove_first_match(&vc->tx_queue,
+ __match_context,
+ context);
+ COND_RELEASE(vc->ep->requires_lock,
+ &vc->tx_queue_lock);
+
+ if (entry) {
+ req = container_of(entry,
+ struct gnix_fab_req,
+ dlist);
+ break;
+ }
+ }
+ } else {
+ GNIX_HASHTABLE_ITERATOR(ep->vc_ht, iter);
+
+ while ((vc = _gnix_ht_iterator_next(&iter))) {
+ COND_ACQUIRE(vc->ep->requires_lock,
+ &vc->tx_queue_lock);
+ entry = dlist_remove_first_match(&vc->tx_queue,
+ __match_context,
+ context);
+ COND_RELEASE(vc->ep->requires_lock,
+ &vc->tx_queue_lock);
+
+ if (entry) {
+ req = container_of(entry,
+ struct gnix_fab_req,
+ dlist);
+ break;
+ }
}
}
- fastlock_release(&ep->vc_ht_lock);
+
+ COND_RELEASE(ep->requires_lock, &ep->vc_lock);
return req;
}
@@ -1794,17 +2192,17 @@ static inline struct gnix_fab_req *__find_rx_req(
{
struct gnix_fab_req *req = NULL;
- fastlock_acquire(&ep->recv_queue_lock);
+ COND_ACQUIRE(ep->requires_lock, &ep->recv_queue_lock);
req = _gnix_remove_req_by_context(&ep->posted_recv_queue, context);
- fastlock_release(&ep->recv_queue_lock);
+ COND_RELEASE(ep->requires_lock, &ep->recv_queue_lock);
if (req)
return req;
- fastlock_acquire(&ep->tagged_queue_lock);
+ COND_ACQUIRE(ep->requires_lock, &ep->tagged_queue_lock);
req = _gnix_remove_req_by_context(&ep->tagged_posted_recv_queue,
context);
- fastlock_release(&ep->tagged_queue_lock);
+ COND_RELEASE(ep->requires_lock, &ep->tagged_queue_lock);
return req;
}
@@ -1855,11 +2253,11 @@ DIRECT_FN STATIC ssize_t gnix_ep_cancel(fid_t fid, void *context)
if (!(req->type == GNIX_FAB_RQ_RDMA_READ ||
req->type == GNIX_FAB_RQ_RDMA_WRITE)) {
if (!is_send) {
- addr = (void *) req->msg.recv_addr;
- len = req->msg.recv_len;
+ addr = (void *) req->msg.recv_info[0].recv_addr;
+ len = req->msg.cum_recv_len;
} else {
- addr = (void *) req->msg.send_addr;
- len = req->msg.send_len;
+ addr = (void *) req->msg.send_info[0].send_addr;
+ len = req->msg.cum_send_len;
}
tag = req->msg.tag;
} else {
@@ -1912,7 +2310,7 @@ ssize_t gnix_cancel(fid_t fid, void *context)
}
static int
-__gnix_ep_ops_get_val(struct fid *fid, dom_ops_val_t t, void *val)
+__gnix_ep_ops_get_val(struct fid *fid, ep_ops_val_t t, void *val)
{
struct gnix_fid_ep *ep;
@@ -1941,7 +2339,7 @@ __gnix_ep_ops_get_val(struct fid *fid, dom_ops_val_t t, void *val)
}
static int
-__gnix_ep_ops_set_val(struct fid *fid, dom_ops_val_t t, void *val)
+__gnix_ep_ops_set_val(struct fid *fid, ep_ops_val_t t, void *val)
{
struct gnix_fid_ep *ep;
int v;
@@ -1992,9 +2390,10 @@ __gnix_ep_ops_set_val(struct fid *fid, dom_ops_val_t t, void *val)
return FI_SUCCESS;
}
-static struct fi_gni_ops_domain gnix_ops_ep = {
+static struct fi_gni_ops_ep gnix_ops_ep = {
.set_val = __gnix_ep_ops_set_val,
- .get_val = __gnix_ep_ops_get_val
+ .get_val = __gnix_ep_ops_get_val,
+ .native_amo = __gnix_fabric_ops_native_amo
};
static int
@@ -2062,8 +2461,18 @@ DIRECT_FN STATIC int gnix_ep_setopt(fid_t fid, int level, int optname,
DIRECT_FN STATIC ssize_t gnix_ep_rx_size_left(struct fid_ep *ep)
{
- if (!ep)
+ if (!ep) {
return -FI_EINVAL;
+ }
+
+ struct gnix_fid_ep *ep_priv = container_of(ep,
+ struct gnix_fid_ep,
+ ep_fid);
+
+ /* A little arbitrary... */
+ if (ep_priv->htd_pool.enabled == false) {
+ return -FI_EOPBADSTATE;
+ }
switch (ep->fid.fclass) {
case FI_CLASS_EP:
@@ -2076,14 +2485,24 @@ DIRECT_FN STATIC ssize_t gnix_ep_rx_size_left(struct fid_ep *ep)
return -FI_EINVAL;
}
- /* We can queue RXs indefinitely, return an arbitrary low water mark. */
- return 64;
+ /* We can queue RXs indefinitely, so just return the default size. */
+ return GNIX_RX_SIZE_DEFAULT;
}
DIRECT_FN STATIC ssize_t gnix_ep_tx_size_left(struct fid_ep *ep)
{
- if (!ep)
+ if (!ep) {
return -FI_EINVAL;
+ }
+
+ struct gnix_fid_ep *ep_priv = container_of(ep,
+ struct gnix_fid_ep,
+ ep_fid);
+
+ /* A little arbitrary... */
+ if (ep_priv->htd_pool.enabled == false) {
+ return -FI_EOPBADSTATE;
+ }
switch (ep->fid.fclass) {
case FI_CLASS_EP:
@@ -2095,8 +2514,8 @@ DIRECT_FN STATIC ssize_t gnix_ep_tx_size_left(struct fid_ep *ep)
return -FI_EINVAL;
}
- /* We can queue TXs indefinitely, return an arbitrary low water mark. */
- return 64;
+ /* We can queue TXs indefinitely, so just return the default size. */
+ return GNIX_TX_SIZE_DEFAULT;
}
__attribute__((unused))
diff --git a/prov/gni/src/gnix_eq.c b/prov/gni/src/gnix_eq.c
index 7024618..0fdd9b6 100644
--- a/prov/gni/src/gnix_eq.c
+++ b/prov/gni/src/gnix_eq.c
@@ -155,14 +155,10 @@ err:
return NULL;
}
-/* Temporarily mark as unused to avoid build warnings. */
-static ssize_t gnix_eq_write_error(struct fid_eq*, fid_t, void*, uint64_t, int,
- int, void*, size_t) __attribute__((unused));
-
-static ssize_t gnix_eq_write_error(struct fid_eq *eq, fid_t fid,
- void *context, uint64_t index, int err,
- int prov_errno, void *err_data,
- size_t err_size)
+ssize_t _gnix_eq_write_error(struct fid_eq *eq, fid_t fid,
+ void *context, uint64_t index, int err,
+ int prov_errno, void *err_data,
+ size_t err_size)
{
struct fi_eq_err_entry *error;
struct gnix_eq_entry *event;
@@ -329,24 +325,22 @@ DIRECT_FN STATIC int gnix_eq_close(struct fid *fid)
return FI_SUCCESS;
}
-DIRECT_FN STATIC ssize_t gnix_eq_sread(struct fid_eq *eq, uint32_t *event,
- void *buf, size_t len, int timeout,
- uint64_t flags)
-{
- return -FI_ENOSYS;
-}
-
DIRECT_FN STATIC ssize_t gnix_eq_read(struct fid_eq *eq, uint32_t *event,
void *buf, size_t len, uint64_t flags)
{
struct gnix_fid_eq *eq_priv;
struct gnix_eq_entry *entry;
struct slist_entry *item;
+ ssize_t read_size;
- ssize_t read_size = len;
+ if (!eq || !event || (len && !buf))
+ return -FI_EINVAL;
eq_priv = container_of(eq, struct gnix_fid_eq, eq_fid);
+ if (_gnix_queue_peek(eq_priv->errors))
+ return -FI_EAVAIL;
+
fastlock_acquire(&eq_priv->lock);
item = _gnix_queue_peek(eq_priv->events);
@@ -358,13 +352,14 @@ DIRECT_FN STATIC ssize_t gnix_eq_read(struct fid_eq *eq, uint32_t *event,
entry = container_of(item, struct gnix_eq_entry, item);
- if (read_size < entry->len) {
+ if (len < entry->len) {
read_size = -FI_ETOOSMALL;
goto err;
}
*event = entry->type;
+ read_size = entry->len;
memcpy(buf, entry->the_entry, read_size);
if (!(flags & FI_PEEK)) {
@@ -382,6 +377,38 @@ err:
return read_size;
}
+DIRECT_FN STATIC ssize_t gnix_eq_sread(struct fid_eq *eq, uint32_t *event,
+ void *buf, size_t len, int timeout,
+ uint64_t flags)
+{
+ double start_ms = 0.0, end_ms = 0.0;
+ int ret;
+
+ ret = gnix_eq_read(eq, event, buf, len, flags);
+ if (ret != -FI_EAGAIN)
+ return ret;
+
+ if (timeout > 0)
+ start_ms = fi_gettime_ms();
+
+ do {
+ usleep(1000);
+
+ ret = gnix_eq_read(eq, event, buf, len, flags);
+ if (ret != -FI_EAGAIN)
+ return ret;
+
+ if (timeout > 0) {
+ end_ms = fi_gettime_ms();
+ timeout -= (end_ms - start_ms);
+ timeout = timeout < 0 ? 0 : timeout;
+ start_ms = end_ms;
+ }
+ } while (timeout);
+
+ return -FI_EAGAIN;
+}
+
DIRECT_FN STATIC int gnix_eq_control(struct fid *eq, int command, void *arg)
{
/* disabled until new trywait interface is implemented
diff --git a/prov/gni/src/gnix_fabric.c b/prov/gni/src/gnix_fabric.c
index 310ba54..b677e59 100644
--- a/prov/gni/src/gnix_fabric.c
+++ b/prov/gni/src/gnix_fabric.c
@@ -59,7 +59,7 @@
#include "gnix_util.h"
#include "gnix_nameserver.h"
#include "gnix_wait.h"
-#include "gnix_av.h"
+#include "gnix_xpmem.h"
const char gnix_fab_name[] = "gni";
const char gnix_dom_name[] = "/sys/class/gni/kgni0";
@@ -101,7 +101,12 @@ static void __fabric_destruct(void *obj)
{
struct gnix_fid_fabric *fab = (struct gnix_fid_fabric *) obj;
- _gnix_alps_cleanup();
+ /*
+ * close the MR notifier
+ */
+ (void) _gnix_notifier_close(&fab->mr_notifier);
+
+ _gnix_app_cleanup();
free(fab);
}
@@ -137,6 +142,7 @@ static int gnix_fabric_open(struct fi_fabric_attr *attr,
struct fid_fabric **fabric,
void *context)
{
+ int ret;
struct gnix_fid_fabric *fab;
if (strcmp(attr->name, gnix_fab_name)) {
@@ -162,6 +168,17 @@ static int gnix_fabric_open(struct fi_fabric_attr *attr,
_gnix_ref_init(&fab->ref_cnt, 1, __fabric_destruct);
dlist_init(&fab->domain_list);
+ ret = _gnix_notifier_init(&fab->mr_notifier);
+ if (ret != FI_SUCCESS) {
+ return ret;
+ }
+
+ // TODO: open dynamically as needed
+ ret = _gnix_notifier_open(&fab->mr_notifier);
+ if (ret != FI_SUCCESS && ret != -FI_EBUSY) {
+ return ret;
+ }
+
*fabric = &fab->fab_fid;
return FI_SUCCESS;
@@ -172,15 +189,39 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service,
struct fi_info **info)
{
int ret = 0;
+ uint32_t pe = -1;
+ uint32_t cpu_id = -1;
int ep_type_unspec = 1;
uint64_t mode = GNIX_FAB_MODES;
- uint64_t caps = GNIX_EP_RDM_CAPS;
struct fi_info *gnix_info = NULL;
struct gnix_ep_name *dest_addr = NULL;
struct gnix_ep_name *src_addr = NULL;
struct gnix_ep_name *addr = NULL;
- char *str_src_addr = NULL;
- char *str_dest_addr = NULL;
+ gni_return_t status;
+
+ /*
+ * do an early check of hints if an app is trying to use
+ * an addressing format we don't handle
+ */
+
+ if (hints) {
+ switch (hints->addr_format) {
+ case FI_FORMAT_UNSPEC:
+ case FI_ADDR_GNI:
+ break;
+ default:
+ GNIX_INFO(FI_LOG_FABRIC,
+ "hints->addr_format=%d, supported=%d,%d.\n",
+ hints->addr_format, FI_FORMAT_UNSPEC, FI_ADDR_GNI);
+ ret = -FI_EADDRNOTAVAIL;
+ goto err;
+ }
+ }
+
+ addr = malloc(sizeof(*addr));
+ if (!addr) {
+ goto err;
+ }
/*
* the code below for resolving a node/service to what
@@ -188,10 +229,6 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service,
* but put a place holder in place
*/
if (node) {
- addr = malloc(sizeof(*addr));
- if (!addr) {
- goto err;
- }
/* resolve node/service to gnix_ep_name */
ret = gnix_resolve_name(node, service, flags, addr);
@@ -210,6 +247,30 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service,
if (hints && hints->src_addr)
src_addr = hints->src_addr;
}
+ } else {
+ /*
+ * okay per the man page, just fill in the src_addr
+ */
+ src_addr = addr;
+
+ status = GNI_CdmGetNicAddress(0, &pe, &cpu_id);
+ if(status != GNI_RC_SUCCESS) {
+ GNIX_WARN(FI_LOG_FABRIC,
+ "Unable to get NIC address.");
+ ret = gnixu_to_fi_errno(status);
+ goto err;
+ }
+
+ if (pe == -1) {
+ GNIX_WARN(FI_LOG_FABRIC,
+ "Unable to acquire valid address for local Aries\n");
+ ret = -FI_EADDRNOTAVAIL;
+ goto err;
+ }
+
+ src_addr->gnix_addr.device_addr = pe;
+ src_addr->gnix_addr.cdm_id = 0; /* just set this to zero */
+ src_addr->name_type = GNIX_EPN_TYPE_UNBOUND;
}
if (src_addr)
@@ -252,46 +313,27 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service,
gnix_info->domain_attr->cq_data_size = sizeof(uint64_t);
gnix_info->domain_attr->mr_mode = FI_MR_BASIC;
gnix_info->domain_attr->resource_mgmt = FI_RM_ENABLED;
+ gnix_info->domain_attr->mr_key_size = sizeof(uint64_t),
gnix_info->next = NULL;
gnix_info->addr_format = FI_ADDR_GNI;
- gnix_info->src_addrlen = 0;
- gnix_info->dest_addrlen = 0;
- gnix_info->src_addr = NULL;
- gnix_info->dest_addr = NULL;
- if (src_addr) {
- str_src_addr = malloc(GNIX_AV_MAX_STR_ADDR_LEN);
- if (!str_src_addr) {
- goto err;
- }
- gnix_info->src_addrlen = GNIX_AV_MAX_STR_ADDR_LEN;
- gnix_av_straddr(NULL, (void *) src_addr, (char *) str_src_addr,
- &gnix_info->src_addrlen);
- gnix_info->src_addr = str_src_addr;
- }
- if (dest_addr) {
- str_dest_addr = malloc(GNIX_AV_MAX_STR_ADDR_LEN);
- if (!str_dest_addr) {
- goto err;
- }
- gnix_info->dest_addrlen = GNIX_AV_MAX_STR_ADDR_LEN;
- gnix_av_straddr(NULL, (void *) addr, (char *) str_dest_addr,
- &gnix_info->dest_addrlen);
- gnix_info->dest_addr = str_dest_addr;
- }
+ gnix_info->src_addrlen = sizeof(struct gnix_ep_name);
+ gnix_info->dest_addrlen = sizeof(struct gnix_ep_name);
+ gnix_info->src_addr = src_addr;
+ gnix_info->dest_addr = dest_addr;
/* prov_name gets filled in by fi_getinfo from the gnix_prov struct */
/* let's consider gni copyrighted :) */
gnix_info->tx_attr->msg_order = FI_ORDER_SAS;
gnix_info->tx_attr->comp_order = FI_ORDER_NONE;
gnix_info->tx_attr->size = GNIX_TX_SIZE_DEFAULT;
- gnix_info->tx_attr->iov_limit = 1;
+ gnix_info->tx_attr->iov_limit = GNIX_MAX_MSG_IOV_LIMIT;
gnix_info->tx_attr->inject_size = GNIX_INJECT_SIZE;
- gnix_info->tx_attr->rma_iov_limit = 1;
+ gnix_info->tx_attr->rma_iov_limit = GNIX_MAX_RMA_IOV_LIMIT;
gnix_info->rx_attr->msg_order = FI_ORDER_SAS;
gnix_info->rx_attr->comp_order = FI_ORDER_NONE;
gnix_info->rx_attr->size = GNIX_RX_SIZE_DEFAULT;
- gnix_info->rx_attr->iov_limit = 1;
+ gnix_info->rx_attr->iov_limit = GNIX_MAX_MSG_IOV_LIMIT;
if (hints) {
if (hints->ep_attr) {
@@ -344,12 +386,20 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service,
mode = hints->mode & ~GNIX_FAB_MODES_CLEAR;
}
- if ((hints->caps & GNIX_EP_RDM_CAPS) != hints->caps) {
- goto err;
- }
+ if (!hints->caps) {
+ /* Return all supported capabilities. */
+ gnix_info->caps = GNIX_EP_RDM_CAPS_FULL;
+ } else {
+ /* The provider must support all requested
+ * capabilities. */
+ if ((hints->caps & GNIX_EP_RDM_CAPS_FULL) !=
+ hints->caps) {
+ goto err;
+ }
- if (hints->caps) {
- caps = hints->caps & GNIX_EP_RDM_CAPS;
+ /* The provider may silently enable secondary
+ * capabilities that do not introduce any overhead. */
+ gnix_info->caps = hints->caps | GNIX_EP_RDM_SEC_CAPS;
}
if (hints->tx_attr) {
@@ -408,32 +458,24 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service,
goto err;
}
+ switch (hints->domain_attr->threading) {
+ case FI_THREAD_COMPLETION:
+ gnix_info->domain_attr->threading =
+ hints->domain_attr->threading;
+ break;
+ default:
+ break;
+ }
+
ret = fi_check_domain_attr(&gnix_prov,
gnix_info->domain_attr,
- hints->domain_attr);
+ hints->domain_attr,
+ FI_MATCH_EXACT);
if (ret)
goto err;
}
}
- /*
- * Set the values based on hints
- */
-
- switch (gnix_info->ep_attr->type) {
- case FI_EP_RDM:
- case FI_EP_DGRAM:
- gnix_info->caps = caps | GNIX_EP_RDM_SEC_CAPS;
- break;
- case FI_EP_MSG:
- gnix_info->caps = caps | GNIX_EP_MSG_SEC_CAPS;
- break;
- default:
- GNIX_ERR(FI_LOG_FABRIC, "unknown ep type %d",
- gnix_info->ep_attr->type);
- break;
- }
-
gnix_info->mode = mode;
gnix_info->fabric_attr->name = strdup(gnix_fab_name);
gnix_info->tx_attr->caps = gnix_info->caps;
@@ -466,18 +508,6 @@ err:
free(gnix_info);
}
- if (addr) {
- free(addr);
- }
-
- if (str_src_addr) {
- free(str_src_addr);
- }
-
- if (str_dest_addr) {
- free(str_dest_addr);
- }
-
/*
* for the getinfo method, we need to return -FI_ENODATA otherwise
* the fi_getinfo call will make an early exit without querying
@@ -534,19 +564,24 @@ GNI_INI
GNIX_INFO(FI_LOG_FABRIC, "gnix_max_nics_per_ptag: %u\n",
gnix_max_nics_per_ptag);
} else {
- GNIX_INFO(FI_LOG_FABRIC, "_gnix_nics_per_rank failed: %d\n", rc);
+ GNIX_WARN(FI_LOG_FABRIC, "_gnix_nics_per_rank failed: %d\n",
+ rc);
}
if (getenv("GNIX_MAX_NICS") != NULL)
gnix_max_nics_per_ptag = atoi(getenv("GNIX_MAX_NICS"));
+ if (getenv("GNIX_DISABLE_XPMEM") != NULL)
+ gnix_xpmem_disabled = true;
+
/*
- * if for some reason we can't even allocate a single nic, bail.
+ * well if we didn't get 1 nic, that means we must really be doing
+ * FMA sharing.
*/
if (gnix_max_nics_per_ptag == 0) {
- GNIX_WARN(FI_LOG_FABRIC, "Insufficient network resources\n");
- provider = NULL;
+ gnix_max_nics_per_ptag = 1;
+ GNIX_WARN(FI_LOG_FABRIC, "Using inter-procss FMA sharing\n");
}
return (provider);
diff --git a/prov/gni/src/gnix_freelist.c b/prov/gni/src/gnix_freelist.c
index 5a39a76..9445847 100644
--- a/prov/gni/src/gnix_freelist.c
+++ b/prov/gni/src/gnix_freelist.c
@@ -43,7 +43,7 @@
/*
* NOTES:
- * - thread safe if initialized with _nix_sfl_init_ts
+ * - thread safe if initialized with _nix_fl_init_ts
* - Does not shrink
* - Cannot be used for data structures with alignment requirements
* - Refill size increases by growth_factor each time growth is needed
@@ -57,7 +57,7 @@
* it just has to be at least as big as an slist_entry.
*/
-static int __gnix_sfl_refill(struct gnix_s_freelist *fl, int n)
+static int __gnix_fl_refill(struct gnix_freelist *fl, int n)
{ int i, ret = FI_SUCCESS;
unsigned char *elems;
@@ -82,16 +82,17 @@ static int __gnix_sfl_refill(struct gnix_s_freelist *fl, int n)
elems += fl->elem_size + fl->offset;
for (i = 0; i < n; i++) {
- slist_insert_tail((struct slist_entry *) elems, &fl->freelist);
+ dlist_init((struct dlist_entry *) elems);
+ dlist_insert_tail((struct dlist_entry *) elems, &fl->freelist);
elems += fl->elem_size;
}
err:
return ret;
}
-int _gnix_sfl_init(int elem_size, int offset, int init_size,
+int _gnix_fl_init(int elem_size, int offset, int init_size,
int refill_size, int growth_factor,
- int max_refill_size, struct gnix_s_freelist *fl)
+ int max_refill_size, struct gnix_freelist *fl)
{
assert(elem_size > 0);
assert(offset >= 0);
@@ -100,35 +101,34 @@ int _gnix_sfl_init(int elem_size, int offset, int init_size,
assert(growth_factor >= 0);
assert(max_refill_size >= 0);
- int fill_size = init_size != 0 ? init_size : GNIX_SFL_INIT_SIZE;
+ int fill_size = init_size != 0 ? init_size : GNIX_FL_INIT_SIZE;
fl->refill_size = (refill_size != 0 ?
refill_size :
- GNIX_SFL_INIT_REFILL_SIZE);
+ GNIX_FL_INIT_REFILL_SIZE);
fl->growth_factor = (growth_factor != 0 ?
growth_factor :
- GNIX_SFL_GROWTH_FACTOR);
+ GNIX_FL_GROWTH_FACTOR);
fl->max_refill_size = (max_refill_size != 0 ?
max_refill_size :
fill_size);
fl->elem_size = elem_size;
fl->offset = offset;
- assert(slist_empty(&fl->freelist)); /* maybe should be a warning? */
- slist_init(&fl->freelist);
+ dlist_init(&fl->freelist);
assert(slist_empty(&fl->chunks)); /* maybe should be a warning? */
slist_init(&fl->chunks);
- return __gnix_sfl_refill(fl, fill_size);
+ return __gnix_fl_refill(fl, fill_size);
}
-int _gnix_sfl_init_ts(int elem_size, int offset, int init_size,
+int _gnix_fl_init_ts(int elem_size, int offset, int init_size,
int refill_size, int growth_factor,
- int max_refill_size, struct gnix_s_freelist *fl)
+ int max_refill_size, struct gnix_freelist *fl)
{
int ret;
- ret = _gnix_sfl_init(elem_size,
+ ret = _gnix_fl_init(elem_size,
offset,
init_size,
refill_size,
@@ -144,7 +144,7 @@ int _gnix_sfl_init_ts(int elem_size, int offset, int init_size,
}
-void _gnix_sfl_destroy(struct gnix_s_freelist *fl)
+void _gnix_fl_destroy(struct gnix_freelist *fl)
{
assert(fl);
@@ -160,19 +160,18 @@ void _gnix_sfl_destroy(struct gnix_s_freelist *fl)
fastlock_destroy(&fl->lock);
}
-int _gnix_sfe_alloc(struct slist_entry **e, struct gnix_s_freelist *fl)
+int _gnix_fl_alloc(struct dlist_entry **e, struct gnix_freelist *fl)
{
int ret = FI_SUCCESS;
+ struct dlist_entry *de;
assert(fl);
if (fl->ts)
fastlock_acquire(&fl->lock);
- struct slist_entry *se = slist_remove_head(&fl->freelist);
-
- if (!se) {
- ret = __gnix_sfl_refill(fl, fl->refill_size);
+ if (dlist_empty(&fl->freelist)) {
+ ret = __gnix_fl_refill(fl, fl->refill_size);
if (ret != FI_SUCCESS)
goto err;
if (fl->refill_size < fl->max_refill_size) {
@@ -182,22 +181,25 @@ int _gnix_sfe_alloc(struct slist_entry **e, struct gnix_s_freelist *fl)
fl->max_refill_size :
ns);
}
- se = slist_remove_head(&fl->freelist);
- if (!se) {
+
+ if (dlist_empty(&fl->freelist)) {
/* Can't happen unless multithreaded */
ret = -FI_EAGAIN;
goto err;
}
}
- *e = se;
+ de = fl->freelist.next;
+ dlist_remove_init(de);
+
+ *e = de;
err:
if (fl->ts)
fastlock_release(&fl->lock);
return ret;
}
-void _gnix_sfe_free(struct slist_entry *e, struct gnix_s_freelist *fl)
+void _gnix_fl_free(struct dlist_entry *e, struct gnix_freelist *fl)
{
assert(e);
assert(fl);
@@ -206,7 +208,8 @@ void _gnix_sfe_free(struct slist_entry *e, struct gnix_s_freelist *fl)
if (fl->ts)
fastlock_acquire(&fl->lock);
- slist_insert_tail(e, &fl->freelist);
+ dlist_init(e);
+ dlist_insert_head(e, &fl->freelist);
if (fl->ts)
fastlock_release(&fl->lock);
}
diff --git a/prov/gni/src/gnix_hashtable.c b/prov/gni/src/gnix_hashtable.c
index 22582c2..d334a6f 100644
--- a/prov/gni/src/gnix_hashtable.c
+++ b/prov/gni/src/gnix_hashtable.c
@@ -126,22 +126,34 @@ static inline gnix_ht_key_t __gnix_hash_func(
ht->ht_attr.ht_hash_seed) % ht->ht_size;
}
+static inline gnix_ht_entry_t *__gnix_ht_lookup_entry_collision(
+ struct dlist_entry *head,
+ gnix_ht_key_t key,
+ uint64_t *collision_count)
+{
+ gnix_ht_entry_t *ht_entry;
+
+ dlist_for_each(head, ht_entry, entry) {
+ READ_PREFETCH(ht_entry->entry.next);
+ if (ht_entry->key == key)
+ return ht_entry;
+
+ *collision_count += 1;
+ }
+
+ return NULL;
+}
+
static inline gnix_ht_entry_t *__gnix_ht_lookup_entry(
struct dlist_entry *head,
- gnix_ht_key_t key,
- uint64_t *collision_count)
+ gnix_ht_key_t key)
{
gnix_ht_entry_t *ht_entry;
- if (dlist_empty(head))
- return NULL;
-
dlist_for_each(head, ht_entry, entry) {
+ READ_PREFETCH(ht_entry->entry.next);
if (ht_entry->key == key)
return ht_entry;
-
- if (collision_count)
- *collision_count += 1;
}
return NULL;
@@ -151,12 +163,9 @@ static inline void *__gnix_ht_lookup_key(
struct dlist_entry *head,
gnix_ht_key_t key)
{
- gnix_ht_entry_t *ht_entry = __gnix_ht_lookup_entry(head, key, NULL);
-
- if (!ht_entry)
- return NULL;
+ gnix_ht_entry_t *ht_entry = __gnix_ht_lookup_entry(head, key);
- return ht_entry->value;
+ return ((ht_entry != NULL) ? ht_entry->value : NULL);
}
static inline int __gnix_ht_destroy_list(
@@ -186,7 +195,7 @@ static inline int __gnix_ht_insert_list(
{
gnix_ht_entry_t *found;
- found = __gnix_ht_lookup_entry(head, ht_entry->key, collisions);
+ found = __gnix_ht_lookup_entry_collision(head, ht_entry->key, collisions);
if (!found) {
dlist_insert_tail(&ht_entry->entry, head);
} else {
@@ -202,7 +211,7 @@ static inline int __gnix_ht_remove_list(
{
gnix_ht_entry_t *ht_entry;
- ht_entry = __gnix_ht_lookup_entry(head, key, NULL);
+ ht_entry = __gnix_ht_lookup_entry(head, key);
if (!ht_entry) {
return -FI_ENOENT;
}
@@ -218,6 +227,7 @@ static inline void __gnix_ht_rehash_list(
gnix_ht_entry_t *ht_entry, *tmp;
gnix_ht_key_t bucket;
struct dlist_entry *ht_lh;
+ uint64_t trash; // No collision information is recorded
if (dlist_empty(head))
return;
@@ -228,7 +238,7 @@ static inline void __gnix_ht_rehash_list(
dlist_remove(&ht_entry->entry);
- __gnix_ht_insert_list(ht_lh, ht_entry, NULL);
+ __gnix_ht_insert_list(ht_lh, ht_entry, &trash);
}
}
@@ -715,9 +725,6 @@ int _gnix_ht_remove(gnix_hashtable_t *ht, gnix_ht_key_t key)
void *_gnix_ht_lookup(gnix_hashtable_t *ht, gnix_ht_key_t key)
{
- if (ht->ht_state != GNIX_HT_STATE_READY)
- return NULL;
-
return ht->ht_ops->lookup(ht, key);
}
diff --git a/prov/gni/src/gnix_init.c b/prov/gni/src/gnix_init.c
index 2fe79c8..9f29aa7 100644
--- a/prov/gni/src/gnix_init.c
+++ b/prov/gni/src/gnix_init.c
@@ -55,12 +55,49 @@ __thread uint32_t gnix_debug_tid = ~(uint32_t) 0;
atomic_t gnix_debug_next_tid;
#endif
+uint8_t precomputed_crc_results[256];
+
+static inline uint8_t __gni_crc_bits(uint8_t data)
+{
+ uint8_t lcrc = 0;
+
+ if(data & 1)
+ lcrc ^= 0x5e;
+ if(data & 2)
+ lcrc ^= 0xbc;
+ if(data & 4)
+ lcrc ^= 0x61;
+ if(data & 8)
+ lcrc ^= 0xc2;
+ if(data & 0x10)
+ lcrc ^= 0x9d;
+ if(data & 0x20)
+ lcrc ^= 0x23;
+ if(data & 0x40)
+ lcrc ^= 0x46;
+ if(data & 0x80)
+ lcrc ^= 0x8c;
+
+ return lcrc;
+}
+
+void __setup_precomputed_crcs(void)
+{
+ int i;
+
+ for (i = 0; i < 256; i++)
+ {
+ precomputed_crc_results[i] = __gni_crc_bits(i);
+ }
+}
+
/**
* Initialization function for performing global setup
*/
__attribute__((constructor))
void gnix_init(void)
{
+ __setup_precomputed_crcs();
atomic_initialize(&gnix_id_counter, 0);
atomic_initialize(&file_id_counter, 0);
#ifndef NDEBUG
diff --git a/prov/gni/src/gnix_mbox_allocator.c b/prov/gni/src/gnix_mbox_allocator.c
index 2f94b28..2a3d99c 100644
--- a/prov/gni/src/gnix_mbox_allocator.c
+++ b/prov/gni/src/gnix_mbox_allocator.c
@@ -1,5 +1,6 @@
/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC.
+ * All rights reserved.
* Copyright (c) 2015 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -34,6 +35,7 @@
#include <stdio.h>
#include <sys/statfs.h>
#include <sys/mman.h>
+#include <mntent.h>
#include "gnix_mbox_allocator.h"
#include "gnix_nic.h"
@@ -51,74 +53,46 @@
* @return -FI_EINVAL if an invalid parameter was given
* @return -FI_EIO if an error occurred while opening the /proc/mounts
* file.
- * @return -FI_ENOMEM if an error occurred while allocating space for the text
- * from the file.
*/
static int __find_huge_page(size_t page_size, char **directory)
{
- int bytes_read = -FI_EINVAL;
+ int rc = -FI_EINVAL;
struct statfs pg_size;
- size_t size = 1024;
- char error_buf[256];
- char *line, *field;
- char *saveptr;
- char *error;
+ struct mntent *mntent;
FILE *fd;
if (!directory || !page_size) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Invalid page_size or directory provided.\n");
- bytes_read = -FI_EINVAL;
- goto err_fopen;
+ return -FI_EINVAL;
}
- *directory = NULL;
-
- fd = fopen("/proc/mounts", "r");
- if (!fd) {
- error = strerror_r(errno, error_buf, sizeof(error_buf));
- GNIX_WARN(FI_LOG_EP_CTRL, "Error opening /proc/mounts: %s\n",
- error);
- bytes_read = -FI_EIO;
- goto err_fopen;
+ fd = setmntent ("/proc/mounts", "r");
+ if (fd == NULL) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "Unable to open /proc/mounts - %s.\n",
+ strerror(errno));
+ return -FI_EIO;
}
- line = malloc(size);
- if (!line) {
+ while ((mntent = getmntent(fd)) != NULL) {
- error = strerror_r(errno, error_buf, sizeof(error_buf));
- GNIX_WARN(FI_LOG_EP_CTRL, "Error allocating line: %s\n",
- error);
- bytes_read = -FI_ENOMEM;
- goto err_malloc;
- }
-
- /*
- * Look for entries with substring hugetlbfs in /proc/mounts.
- */
- while (getline(&line, &size, fd) != -1) {
- if (strstr(line, "hugetlbfs")) {
- field = strtok_r(line, " ", &saveptr);
- field = strtok_r(NULL, " ", &saveptr);
-
- /*
- * Check if page is the correct size.
- */
- if (statfs(field, &pg_size) == 0) {
- if (pg_size.f_bsize == page_size) {
- *directory = strdup(field);
- bytes_read = FI_SUCCESS;
- break;
- }
+ if (strcmp (mntent->mnt_type, "hugetlbfs") != 0) {
+ continue;
+ }
+
+ if (statfs(mntent->mnt_dir, &pg_size) == 0) {
+ if (pg_size.f_bsize == page_size) {
+ *directory = strdup(mntent->mnt_dir);
+ rc = FI_SUCCESS;
+ break;
}
}
}
- free(line);
-err_malloc:
- fclose(fd);
-err_fopen:
- return bytes_read;
+ endmntent(fd);
+
+ return rc;
}
/**
@@ -155,9 +129,10 @@ static int __generate_file_name(size_t page_size, char **filename)
}
ret = __find_huge_page(page_size, &huge_page);
- if (ret < 0) {
+ if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
- "Find huge page returned negative condition.\n");
+ "Find huge page returned error %s\n",
+ fi_strerror(-ret));
goto err_invalid;
}
@@ -363,13 +338,13 @@ static int __create_slab(struct gnix_mbox_alloc_handle *handle)
goto err_alloc_bitmap;
}
- fastlock_acquire(&handle->nic_handle->lock);
+ COND_ACQUIRE(handle->nic_handle->requires_lock, &handle->nic_handle->lock);
status = GNI_MemRegister(handle->nic_handle->gni_nic_hndl,
(uint64_t) slab->base, total_size,
handle->cq_handle,
GNI_MEM_READWRITE, -1,
&slab->memory_handle);
- fastlock_release(&handle->nic_handle->lock);
+ COND_RELEASE(handle->nic_handle->requires_lock, &handle->nic_handle->lock);
if (status != GNI_RC_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL, "GNI_MemRegister failed: %s\n",
gni_err_str[status]);
@@ -425,10 +400,10 @@ static int __destroy_slab(struct gnix_mbox_alloc_handle *handle,
_gnix_free_bitmap(slab->used);
free(slab->used);
- fastlock_acquire(&handle->nic_handle->lock);
+ COND_ACQUIRE(handle->nic_handle->requires_lock, &handle->nic_handle->lock);
GNI_MemDeregister(handle->nic_handle->gni_nic_hndl,
&slab->memory_handle);
- fastlock_release(&handle->nic_handle->lock);
+ COND_RELEASE(handle->nic_handle->requires_lock, &handle->nic_handle->lock);
munmap(slab->base, total_size);
diff --git a/prov/gni/src/gnix_mr.c b/prov/gni/src/gnix_mr.c
index 48503bf..4fb177b 100644
--- a/prov/gni/src/gnix_mr.c
+++ b/prov/gni/src/gnix_mr.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
*
*
@@ -201,8 +201,9 @@ DIRECT_FN int gnix_mr_reg(struct fid *fid, const void *buf, size_t len,
if (unlikely(dlist_empty(&domain->nic_list))) {
rc = gnix_nic_alloc(domain, NULL, &nic);
if (rc) {
- GNIX_WARN(FI_LOG_MR, "could not allocate nic to do mr_reg,"
- " ret=%i\n", rc);
+ GNIX_INFO(FI_LOG_MR,
+ "could not allocate nic to do mr_reg,"
+ " ret=%i\n", rc);
goto err;
}
}
@@ -213,16 +214,18 @@ DIRECT_FN int gnix_mr_reg(struct fid *fid, const void *buf, size_t len,
/* call cache register op to retrieve the right entry */
fastlock_acquire(&domain->mr_cache_lock);
- if (unlikely(!domain->mr_cache)) {
- rc = _gnix_mr_cache_init(&domain->mr_cache,
- &domain->mr_cache_attr);
+ if (unlikely(!domain->mr_ops))
+ _gnix_open_cache(domain, GNIX_DEFAULT_CACHE_TYPE);
+
+ if (unlikely(!domain->mr_ops->is_init(domain))) {
+ rc = domain->mr_ops->init(domain);
if (rc != FI_SUCCESS) {
fastlock_release(&domain->mr_cache_lock);
goto err;
}
}
- rc = _gnix_mr_cache_register(domain->mr_cache,
+ rc = domain->mr_ops->reg_mr(domain,
(uint64_t) reg_addr, reg_len, &fi_reg_context,
(void **) &mr);
fastlock_release(&domain->mr_cache_lock);
@@ -252,7 +255,7 @@ err:
}
/**
- * Closes and deallocates a libfabric memory registration
+ * Closes and deallocates a libfabric memory registration in the internal cache
*
* @param[in] fid libfabric memory registration fid
*
@@ -281,7 +284,7 @@ static int fi_gnix_mr_close(fid_t fid)
/* call cache deregister op */
fastlock_acquire(&domain->mr_cache_lock);
- ret = _gnix_mr_cache_deregister(mr->domain->mr_cache, mr);
+ ret = domain->mr_ops->dereg_mr(domain, mr);
fastlock_release(&domain->mr_cache_lock);
/* check retcode */
@@ -290,47 +293,38 @@ static int fi_gnix_mr_close(fid_t fid)
_gnix_ref_put(domain);
_gnix_ref_put(nic);
} else {
- GNIX_WARN(FI_LOG_MR, "failed to deregister memory, "
- "ret=%i\n", ret);
+ GNIX_INFO(FI_LOG_MR, "failed to deregister memory, "
+ "ret=%i\n", ret);
}
return ret;
}
-static void *__gnix_register_region(
- void *handle,
+static inline void *__gnix_generic_register(
+ struct gnix_fid_domain *domain,
+ struct gnix_fid_mem_desc *md,
void *address,
size_t length,
- struct _gnix_fi_reg_context *fi_reg_context,
- void *context)
+ gni_cq_handle_t dst_cq_hndl,
+ int flags,
+ int vmdh_index)
{
- struct gnix_fid_mem_desc *md = (struct gnix_fid_mem_desc *) handle;
- struct gnix_fid_domain *domain = context;
struct gnix_nic *nic;
- gni_cq_handle_t dst_cq_hndl = NULL;
- int flags = 0;
- int vmdh_index = -1;
gni_return_t grc = GNI_RC_SUCCESS;
- /* If network would be able to write to this buffer, use read-write */
- if (fi_reg_context->access & (FI_RECV | FI_READ | FI_REMOTE_WRITE))
- flags |= GNI_MEM_READWRITE;
- else
- flags |= GNI_MEM_READ_ONLY;
-
dlist_for_each(&domain->nic_list, nic, dom_nic_list)
{
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
grc = GNI_MemRegister(nic->gni_nic_hndl, (uint64_t) address,
- length, dst_cq_hndl, flags,
- vmdh_index, &md->mem_hndl);
- fastlock_release(&nic->lock);
+ length, dst_cq_hndl, flags,
+ vmdh_index, &md->mem_hndl);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
if (grc == GNI_RC_SUCCESS)
break;
}
if (unlikely(grc != GNI_RC_SUCCESS)) {
- GNIX_WARN(FI_LOG_MR, "failed to register memory with uGNI, "
+ GNIX_INFO(FI_LOG_MR, "failed to register memory with uGNI, "
"ret=%s\n", gni_err_str[grc]);
return NULL;
}
@@ -346,6 +340,29 @@ static void *__gnix_register_region(
return md;
}
+static void *__gnix_register_region(
+ void *handle,
+ void *address,
+ size_t length,
+ struct _gnix_fi_reg_context *fi_reg_context,
+ void *context)
+{
+ struct gnix_fid_mem_desc *md = (struct gnix_fid_mem_desc *) handle;
+ struct gnix_fid_domain *domain = context;
+ gni_cq_handle_t dst_cq_hndl = NULL;
+ int flags = 0;
+ int vmdh_index = -1;
+
+ /* If network would be able to write to this buffer, use read-write */
+ if (fi_reg_context->access & (FI_RECV | FI_READ | FI_REMOTE_WRITE))
+ flags |= GNI_MEM_READWRITE;
+ else
+ flags |= GNI_MEM_READ_ONLY;
+
+ return __gnix_generic_register(domain, md, address, length, dst_cq_hndl,
+ flags, vmdh_index);
+}
+
static int __gnix_deregister_region(
void *handle,
void *context)
@@ -358,9 +375,9 @@ static int __gnix_deregister_region(
domain = mr->domain;
nic = mr->nic;
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
ret = GNI_MemDeregister(nic->gni_nic_hndl, &mr->mem_hndl);
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
if (ret == GNI_RC_SUCCESS) {
/* release reference to domain */
_gnix_ref_put(domain);
@@ -368,8 +385,8 @@ static int __gnix_deregister_region(
/* release reference to nic */
_gnix_ref_put(nic);
} else {
- GNIX_WARN(FI_LOG_MR, "failed to deregister memory"
- " region, entry=%p ret=%i\n", handle, ret);
+ GNIX_INFO(FI_LOG_MR, "failed to deregister memory"
+ " region, entry=%p ret=%i\n", handle, ret);
}
return ret;
@@ -398,13 +415,366 @@ static int __gnix_destruct_registration(void *context)
return GNI_RC_SUCCESS;
}
+
+#ifdef HAVE_UDREG
+void *__udreg_register(void *addr, uint64_t length, void *context)
+{
+ struct gnix_fid_mem_desc *md;
+ struct gnix_fid_domain *domain;
+
+ domain = (struct gnix_fid_domain *) context;
+
+ /* Allocate an udreg info block for this registration. */
+ md = calloc(1, sizeof(*md));
+ if (!md) {
+ GNIX_WARN(FI_LOG_MR, "failed to allocate memory for registration\n");
+ return NULL;
+ }
+
+ return __gnix_generic_register(domain, md, addr, length, NULL,
+ GNI_MEM_READWRITE, -1);
+}
+
+
+uint32_t __udreg_deregister(void *registration, void *context)
+{
+ gni_return_t grc;
+
+ grc = __gnix_deregister_region(registration, NULL);
+
+ return (grc == GNI_RC_SUCCESS) ? 0 : 1;
+}
+
+
+/* Called via dreg when a cache is destroyed. */
+void __udreg_cache_destructor(void *context)
+{
+ /* Nothing needed here. */
+}
+
+static int __udreg_init(struct gnix_fid_domain *domain)
+{
+ udreg_return_t urc;
+
+ udreg_cache_attr_t udreg_cache_attr = {
+ .cache_name = {"gnix_app_cache"},
+ .max_entries = domain->udreg_reg_limit,
+ .modes = UDREG_CC_MODE_USE_LARGE_PAGES,
+ .debug_mode = 0,
+ .debug_rank = 0,
+ .reg_context = (void *) domain,
+ .dreg_context = (void *) domain,
+ .destructor_context = (void *) domain,
+ .device_reg_func = __udreg_register,
+ .device_dereg_func = __udreg_deregister,
+ .destructor_callback = __udreg_cache_destructor,
+ };
+
+ if (domain->mr_cache_attr.lazy_deregistration)
+ udreg_cache_attr.modes |= UDREG_CC_MODE_USE_LAZY_DEREG;
+
+ /*
+ * Create a udreg cache for application memory registrations.
+ */
+ urc = UDREG_CacheCreate(&udreg_cache_attr);
+ if (urc != UDREG_RC_SUCCESS) {
+ GNIX_FATAL(FI_LOG_MR,
+ "Could not initialize udreg application cache, urc=%d\n",
+ urc);
+ }
+
+ urc = UDREG_CacheAccess(udreg_cache_attr.cache_name, &domain->udreg_cache);
+ if (urc != UDREG_RC_SUCCESS) {
+ GNIX_FATAL(FI_LOG_MR,
+ "Could not access udreg application cache, urc=%d",
+ urc);
+ }
+
+ domain->mr_is_init = 1;
+
+ return FI_SUCCESS;
+}
+
+static int __udreg_is_init(struct gnix_fid_domain *domain)
+{
+ return domain->udreg_cache != NULL;
+}
+
+static int __udreg_reg_mr(
+ struct gnix_fid_domain *domain,
+ uint64_t address,
+ uint64_t length,
+ struct _gnix_fi_reg_context *fi_reg_context,
+ void **handle) {
+
+ udreg_return_t urc;
+ udreg_entry_t *udreg_entry;
+ struct gnix_fid_mem_desc *md;
+
+ urc = UDREG_Register(domain->udreg_cache, (void *) address, length, &udreg_entry);
+ if (unlikely(urc != UDREG_RC_SUCCESS))
+ return -FI_EIO;
+
+ md = udreg_entry->device_data;
+ md->entry = udreg_entry;
+
+ *handle = md;
+
+ return FI_SUCCESS;
+}
+
+static int __udreg_dereg_mr(struct gnix_fid_domain *domain,
+ struct gnix_fid_mem_desc *md)
+{
+ udreg_return_t urc;
+
+ urc = UDREG_Unregister(domain->udreg_cache,
+ (udreg_entry_t *) md->entry);
+ if (urc != UDREG_RC_SUCCESS) {
+ GNIX_WARN(FI_LOG_MR, "UDREG_Unregister() returned %d\n", urc);
+ return -FI_ENOENT;
+ }
+
+ return urc;
+}
+
+static int __udreg_close(struct gnix_fid_domain *domain)
+{
+ udreg_return_t ret;
+
+ if (domain->udreg_cache) {
+ ret = UDREG_CacheRelease(domain->udreg_cache);
+ if (unlikely(ret != UDREG_RC_SUCCESS))
+ GNIX_FATAL(FI_LOG_DOMAIN, "failed to release from "
+ "mr cache during domain destruct, dom=%p rc=%d\n",
+ domain, ret);
+
+ ret = UDREG_CacheDestroy(domain->udreg_cache);
+ if (unlikely(ret != UDREG_RC_SUCCESS))
+ GNIX_FATAL(FI_LOG_DOMAIN, "failed to destroy mr "
+ "cache during domain destruct, dom=%p rc=%d\n",
+ domain, ret);
+ }
+
+ return FI_SUCCESS;
+}
+#else
+static int __udreg_init(struct gnix_fid_domain *domain)
+{
+ return -FI_ENOSYS;
+}
+
+static int __udreg_is_init(struct gnix_fid_domain *domain)
+{
+ return FI_SUCCESS;
+}
+
+static int __udreg_reg_mr(struct gnix_fid_domain *domain,
+ uint64_t address,
+ uint64_t length,
+ struct _gnix_fi_reg_context *fi_reg_context,
+ void **handle) {
+
+ return -FI_ENOSYS;
+}
+
+static int __udreg_dereg_mr(struct gnix_fid_domain *domain,
+ struct gnix_fid_mem_desc *md)
+{
+ return -FI_ENOSYS;
+}
+
+static int __udreg_close(struct gnix_fid_domain *domain)
+{
+ return FI_SUCCESS;
+}
+#endif
+
+struct gnix_mr_ops udreg_mr_ops = {
+ .init = __udreg_init,
+ .is_init = __udreg_is_init,
+ .reg_mr = __udreg_reg_mr,
+ .dereg_mr = __udreg_dereg_mr,
+ .destroy_cache = __udreg_close,
+ .flush_cache = NULL, // UDREG doesn't support cache flush
+};
+
+static int __cache_init(struct gnix_fid_domain *domain) {
+ int ret;
+
+ ret = _gnix_mr_cache_init(&domain->mr_cache,
+ &domain->mr_cache_attr);
+ if (ret == FI_SUCCESS)
+ domain->mr_is_init = 1;
+
+ return ret;
+}
+
+static int __cache_is_init(struct gnix_fid_domain *domain) {
+ return domain->mr_cache != NULL;
+}
+
+static int __cache_reg_mr(
+ struct gnix_fid_domain *domain,
+ uint64_t address,
+ uint64_t length,
+ struct _gnix_fi_reg_context *fi_reg_context,
+ void **handle) {
+
+ return _gnix_mr_cache_register(domain->mr_cache, address, length,
+ fi_reg_context, handle);
+}
+
+static int __cache_dereg_mr(struct gnix_fid_domain *domain,
+ struct gnix_fid_mem_desc *md)
+{
+ return _gnix_mr_cache_deregister(domain->mr_cache, md);
+}
+
+static int __cache_close(struct gnix_fid_domain *domain)
+{
+ int ret;
+
+ if (domain->mr_cache) {
+ ret = _gnix_mr_cache_destroy(domain->mr_cache);
+ if (ret != FI_SUCCESS)
+ GNIX_FATAL(FI_LOG_DOMAIN, "failed to destroy mr cache "
+ "during domain destruct, dom=%p ret=%d\n",
+ domain, ret);
+ }
+
+ return FI_SUCCESS;
+}
+
+static int __cache_flush(struct gnix_fid_domain *domain)
+{
+ int ret;
+
+ fastlock_acquire(&domain->mr_cache_lock);
+ ret = _gnix_mr_cache_flush(domain->mr_cache);
+ fastlock_release(&domain->mr_cache_lock);
+
+ return ret;
+}
+
+struct gnix_mr_ops cache_mr_ops = {
+ .init = __cache_init,
+ .is_init = __cache_is_init,
+ .reg_mr = __cache_reg_mr,
+ .dereg_mr = __cache_dereg_mr,
+ .destroy_cache = __cache_close,
+ .flush_cache = __cache_flush,
+};
+
+
+static int __basic_mr_init(struct gnix_fid_domain *domain) {
+ domain->mr_is_init = 1;
+ return FI_SUCCESS;
+}
+
+static int __basic_mr_is_init(struct gnix_fid_domain *domain) {
+ return domain->mr_is_init;
+}
+
+static int __basic_mr_reg_mr(
+ struct gnix_fid_domain *domain,
+ uint64_t address,
+ uint64_t length,
+ struct _gnix_fi_reg_context *fi_reg_context,
+ void **handle) {
+
+ struct gnix_fid_mem_desc *md, *ret;
+
+ md = calloc(1, sizeof(*md));
+ if (!md) {
+ GNIX_WARN(FI_LOG_MR, "failed to allocate memory");
+ return -FI_ENOMEM;
+ }
+ ret = __gnix_register_region((void *) md, (void *) address, length,
+ fi_reg_context, (void *) domain);
+ if (!ret) {
+ GNIX_WARN(FI_LOG_MR, "failed to register memory");
+ free(md);
+ return -FI_ENOSPC;
+ }
+
+ *handle = (void *) md;
+
+ return FI_SUCCESS;
+}
+
+static int __basic_mr_dereg_mr(struct gnix_fid_domain *domain,
+ struct gnix_fid_mem_desc *md)
+{
+ int ret;
+
+ ret = __gnix_deregister_region((void *) md, NULL);
+ if (ret == FI_SUCCESS)
+ free((void *) md);
+
+ return ret;
+}
+
+struct gnix_mr_ops basic_mr_ops = {
+ .init = __basic_mr_init,
+ .is_init = __basic_mr_is_init,
+ .reg_mr = __basic_mr_reg_mr,
+ .dereg_mr = __basic_mr_dereg_mr,
+ .flush_cache = NULL, // unsupported since there is no caching here
+};
+
+
+int _gnix_open_cache(struct gnix_fid_domain *domain, int type)
+{
+ if (type < 0 || type >= GNIX_MR_MAX_TYPE)
+ return -FI_EINVAL;
+
+ if (domain->mr_ops && domain->mr_ops->is_init(domain))
+ return -FI_EBUSY;
+
+ switch(type) {
+ case GNIX_MR_TYPE_UDREG:
+ domain->mr_ops = &udreg_mr_ops;
+ break;
+ case GNIX_MR_TYPE_NONE:
+ domain->mr_ops = &basic_mr_ops;
+ break;
+ default:
+ domain->mr_ops = &cache_mr_ops;
+ break;
+ }
+
+ domain->mr_cache_type = type;
+ return FI_SUCCESS;
+}
+
+
+int _gnix_flush_registration_cache(struct gnix_fid_domain *domain)
+{
+ if (domain->mr_ops && domain->mr_ops->flush_cache)
+ return domain->mr_ops->flush_cache(domain);
+
+ return FI_SUCCESS; // if no flush was present, silently pass
+}
+
+int _gnix_close_cache(struct gnix_fid_domain *domain)
+{
+ /* if the domain isn't being destructed by close, we need to check the
+ * cache again. This isn't a likely case. Destroy must succeed since we
+ * are in the destruct path */
+ if (domain->mr_ops && domain->mr_ops->destroy_cache)
+ return domain->mr_ops->destroy_cache(domain);
+
+ return FI_SUCCESS;
+}
+
gnix_mr_cache_attr_t _gnix_default_mr_cache_attr = {
.soft_reg_limit = 4096,
.hard_reg_limit = -1,
.hard_stale_limit = 128,
.lazy_deregistration = 1,
- .reg_callback = __gnix_register_region,
- .dereg_callback = __gnix_deregister_region,
- .destruct_callback = __gnix_destruct_registration,
- .elem_size = sizeof(struct gnix_fid_mem_desc),
+ .reg_callback = __gnix_register_region,
+ .dereg_callback = __gnix_deregister_region,
+ .destruct_callback = __gnix_destruct_registration,
+ .elem_size = sizeof(struct gnix_fid_mem_desc),
};
diff --git a/prov/gni/src/gnix_mr_cache.c b/prov/gni/src/gnix_mr_cache.c
index d0c0ceb..14a2725 100644
--- a/prov/gni/src/gnix_mr_cache.c
+++ b/prov/gni/src/gnix_mr_cache.c
@@ -31,17 +31,24 @@
*/
#include <gnix_mr_cache.h>
+#include <gnix_mr_notifier.h>
#include <gnix.h>
-typedef enum cache_entry_flags {
- GNIX_CE_RETIRED = 1 << 0,
-} cache_entry_flags_e;
-
-typedef enum cache_entry_state {
- GNIX_CES_DEAD = 0,
- GNIX_CES_INUSE,
- GNIX_CES_STALE,
-} cache_entry_state_e;
+typedef unsigned long long int cache_entry_state_t;
+/* These are used for entry state and should be unique */
+#define GNIX_CES_INUSE (1ULL << 8) /* in use */
+#define GNIX_CES_STALE (2ULL << 8) /* cached for possible reuse */
+#define GNIX_CES_STATE_MASK (0xFULL << 8)
+
+typedef unsigned long long int cache_entry_flag_t;
+/* One or more of these can be combined with the above */
+#define GNIX_CE_RETIRED (1ULL << 61) /* in use, but not to be reused */
+#define GNIX_CE_MERGED (1ULL << 62) /* merged entry, i.e., not
+ * an original request from
+ * fi_mr_reg */
+#define GNIX_CE_UNMAPPED (1ULL << 63) /* at least 1 page of the
+ * entry has been unmapped
+ * by the OS */
/**
* @brief structure for containing the fields relevant to the memory cache key
@@ -67,24 +74,18 @@ typedef struct gnix_mr_cache_key {
* @var lru_entry lru list entry
* @var siblings list of sibling entries
* @var children list of subsumed child entries
- * @var flags cache entry flags @see cache_entry_flags_e
*/
typedef struct gnix_mr_cache_entry {
- cache_entry_state_e state;
+ cache_entry_state_t state;
gnix_mr_cache_key_t key;
atomic_t ref_cnt;
struct dlist_entry lru_entry;
struct dlist_entry siblings;
struct dlist_entry children;
- cache_entry_flags_e flags;
uint64_t mr[0];
} gnix_mr_cache_entry_t;
/* forward declarations */
-int _gnix_mr_cache_init(
- gnix_mr_cache_t **cache,
- gnix_mr_cache_attr_t *attr);
-
int _gnix_mr_cache_register(
gnix_mr_cache_t *cache,
uint64_t address,
@@ -104,6 +105,9 @@ static inline int __mr_cache_entry_get(
gnix_mr_cache_t *cache,
gnix_mr_cache_entry_t *entry);
+static inline int __mr_cache_entry_destroy(gnix_mr_cache_t *cache,
+ gnix_mr_cache_entry_t *entry);
+
static int __mr_cache_create_registration(
gnix_mr_cache_t *cache,
uint64_t address,
@@ -123,6 +127,66 @@ gnix_mr_cache_attr_t __default_mr_cache_attr = {
.lazy_deregistration = 1,
};
+/* Functions for using and manipulating cache entry state */
+static inline cache_entry_state_t __entry_get_state(gnix_mr_cache_entry_t *e)
+{
+ return e->state & GNIX_CES_STATE_MASK;
+}
+
+static inline void __entry_set_state(gnix_mr_cache_entry_t *e,
+ cache_entry_state_t s)
+{
+ e->state = (e->state & ~GNIX_CES_STATE_MASK) |
+ (s & GNIX_CES_STATE_MASK);
+}
+
+static inline void __entry_reset_state(gnix_mr_cache_entry_t *e)
+{
+ e->state = 0ULL;
+}
+
+static inline bool __entry_is_flag(gnix_mr_cache_entry_t *e,
+ cache_entry_flag_t f)
+{
+ return (e->state & f) != 0;
+}
+
+static inline void __entry_set_flag(gnix_mr_cache_entry_t *e,
+ cache_entry_flag_t f)
+{
+ e->state = e->state | f;
+}
+
+static inline bool __entry_is_retired(gnix_mr_cache_entry_t *e)
+{
+ return __entry_is_flag(e, GNIX_CE_RETIRED);
+}
+
+static inline bool __entry_is_merged(gnix_mr_cache_entry_t *e)
+{
+ return __entry_is_flag(e, GNIX_CE_MERGED);
+}
+
+static inline bool __entry_is_unmapped(gnix_mr_cache_entry_t *e)
+{
+ return __entry_is_flag(e, GNIX_CE_UNMAPPED);
+}
+
+static inline void __entry_set_retired(gnix_mr_cache_entry_t *e)
+{
+ __entry_set_flag(e, GNIX_CE_RETIRED);
+}
+
+static inline void __entry_set_merged(gnix_mr_cache_entry_t *e)
+{
+ __entry_set_flag(e, GNIX_CE_MERGED);
+}
+
+static inline void __entry_set_unmapped(gnix_mr_cache_entry_t *e)
+{
+ __entry_set_flag(e, GNIX_CE_UNMAPPED);
+}
+
/**
* Key comparison function for finding overlapping gnix memory
* registration cache entries
@@ -130,7 +194,7 @@ gnix_mr_cache_attr_t __default_mr_cache_attr = {
* @param[in] x key to be inserted or found
* @param[in] y key to be compared against
*
- * @return -1 if it should be positioned at the left, 0 if the same,
+ * @return -1 if it should be positioned at the left, 0 if it overlaps,
* 1 otherwise
*/
static int __find_overlapping_addr(
@@ -259,7 +323,11 @@ static inline void __attach_retired_entries_to_registration(
__mr_cache_entry_put(cache, entry);
}
}
- assert(dlist_empty(retired_entries));
+
+ if (!dlist_empty(retired_entries)) {
+ GNIX_FATAL(FI_LOG_MR, "retired_entries not empty\n");
+ }
+
__mr_cache_entry_get(cache, parent);
}
@@ -274,18 +342,19 @@ static inline void __remove_sibling_entries_from_tree(
dlist_for_each(list, entry, siblings)
{
- GNIX_INFO(FI_LOG_MR,
- "removing key from tree, key=%llx:%llx\n",
- entry->key.address,
- entry->key.length);
+ GNIX_DEBUG(FI_LOG_MR,
+ "removing key from tree, key=%llx:%llx\n",
+ entry->key.address, entry->key.length);
iter = rbtFind(tree, &entry->key);
- assert(iter);
+ if (unlikely(!iter)) {
+ GNIX_FATAL(FI_LOG_MR, "key not found\n");
+ }
rc = rbtErase(tree, iter);
- if (unlikely(rc != RBT_STATUS_OK))
- GNIX_INFO(FI_LOG_MR,
- "could not remove entry from tree\n");
- assert(rc == RBT_STATUS_OK);
+ if (unlikely(rc != RBT_STATUS_OK)) {
+ GNIX_FATAL(FI_LOG_MR,
+ "could not remove entry from tree\n");
+ }
}
}
@@ -337,6 +406,250 @@ static inline int __mr_cache_lru_dequeue(
}
/**
+ * Removes a particular registration cache entry from the lru cache.
+ *
+ * @param[in] cache a memory registration cache
+ * @param[in] entry a memory registration cache entry
+ *
+ * @return FI_SUCCESS, on success
+ * @return -FI_ENOENT, on empty LRU
+ */
+static inline int __mr_cache_lru_remove(
+ gnix_mr_cache_t *cache,
+ gnix_mr_cache_entry_t *entry)
+{
+ /* Could do some error checking to see if in cache */
+
+ dlist_remove(&entry->lru_entry);
+
+ return FI_SUCCESS;
+}
+
+/**
+ * Remove entries that have been unmapped as indicated by the notifer
+ *
+ * @param[in] cache a memory registration cache
+ *
+ * @return nothing
+ */
+static bool __notifier_warned = false;
+static void
+__clear_notifier_events(gnix_mr_cache_t *cache)
+{
+ int ret;
+ gnix_mr_cache_entry_t *entry;
+ RbtIterator iter;
+ uint64_t cookie;
+
+ if (!cache->attr.notifier) {
+ return;
+ }
+
+ if (!cache->attr.lazy_deregistration) {
+ return;
+ }
+
+ ret = _gnix_notifier_get_event(cache->attr.notifier,
+ &cookie, sizeof(cookie));
+ while (ret > 0) {
+ if (ret == sizeof(cookie)) {
+ entry = (gnix_mr_cache_entry_t *) cookie;
+ switch (__entry_get_state(entry)) {
+ case GNIX_CES_INUSE:
+ /* First, warn that this might be a
+ * problem.*/
+ if ((__notifier_warned == false) &&
+ !__entry_is_merged(entry)) {
+ GNIX_WARN(FI_LOG_MR,
+ "Registered memory region"
+ " includes unmapped pages."
+ " Have you freed memory"
+ " without closing the memory"
+ " region?\n");
+ __notifier_warned = true;
+ }
+
+ GNIX_DEBUG(FI_LOG_MR,
+ "Marking unmapped entry (%p)"
+ " as retired %llx:%llx\n", entry,
+ entry->key.address,
+ entry->key.length);
+
+ __entry_set_unmapped(entry);
+
+ if (__entry_is_retired(entry)) {
+ /* Nothing to do */
+ break;
+ }
+
+ /* Retire this entry (remove from
+ * inuse tree) */
+
+ __entry_set_retired(entry);
+ iter = rbtFind(cache->inuse.rb_tree,
+ &entry->key);
+ if (likely(iter != NULL)) {
+ ret = rbtErase(cache->inuse.rb_tree,
+ iter);
+ if (ret != RBT_STATUS_OK) {
+ GNIX_FATAL(FI_LOG_MR,
+ "Unmapped entry"
+ " could not be"
+ " removed from"
+ " in usetree.\n");
+ }
+ } else {
+ /* The only way we should get
+ * here is if we're in the
+ * middle of retiring this
+ * entry. Not sure if this
+ * is worth a separate
+ * warning from the one
+ * above. */
+ }
+
+ break;
+ case GNIX_CES_STALE:
+ __entry_set_unmapped(entry);
+ iter = rbtFind(cache->stale.rb_tree,
+ &entry->key);
+ if (!iter) {
+ break;
+ }
+
+ ret = rbtErase(cache->stale.rb_tree, iter);
+ if (ret != RBT_STATUS_OK) {
+ GNIX_FATAL(FI_LOG_MR,
+ "Unmapped entry could"
+ " not be removed "
+ " from stale tree.\n");
+ }
+
+ GNIX_DEBUG(FI_LOG_MR, "Removed unmapped entry"
+ " (%p) from stale tree %llx:%llx\n",
+ entry, entry->key.address,
+ entry->key.length);
+
+ if (__mr_cache_lru_remove(cache, entry)
+ == FI_SUCCESS) {
+ GNIX_DEBUG(FI_LOG_MR, "Removed"
+ " unmapped entry (%p)"
+ " from lru list %llx:%llx\n",
+ entry, entry->key.address,
+ entry->key.length);
+
+ atomic_dec(&cache->stale.elements);
+
+ } else {
+ GNIX_WARN(FI_LOG_MR, "Failed to remove"
+ " unmapped entry"
+ " from lru list (%p) %p\n",
+ entry, iter);
+ }
+
+ __mr_cache_entry_destroy(cache, entry);
+
+ break;
+ default:
+ GNIX_FATAL(FI_LOG_MR,
+ "Unmapped entry (%p) in incorrect"
+ " state: %d\n",
+ entry, entry->state);
+ }
+
+ } else {
+ /* Should we do something else here? */
+ GNIX_FATAL(FI_LOG_MR,
+ "_gnix_notifier_get_event returned incomplete event\n");
+ }
+ ret = _gnix_notifier_get_event(cache->attr.notifier,
+ &cookie, sizeof(cookie));
+ }
+ if (ret != -FI_EAGAIN) {
+ /* Should we do something else here? */
+ GNIX_WARN(FI_LOG_MR,
+ "_gnix_notifier_get_event returned error: %s\n",
+ fi_strerror(-ret));
+ }
+
+ return;
+}
+
+/**
+ * Start monitoring a memory region
+ *
+ * @param[in] cache a memory registration cache
+ * @param[in] entry a memory registration entry
+ *
+ * @return return code from _gnix_notifier_monitor
+ */
+static int
+__notifier_monitor(gnix_mr_cache_t *cache,
+ gnix_mr_cache_entry_t *entry)
+{
+
+ if (!cache->attr.lazy_deregistration) {
+ return FI_SUCCESS;
+ }
+
+ if (cache->attr.notifier == NULL) {
+ return FI_SUCCESS;
+ }
+
+ GNIX_DEBUG(FI_LOG_MR, "monitoring entry=%p %llx:%llx\n", entry,
+ entry->key.address, entry->key.length);
+
+ return _gnix_notifier_monitor(cache->attr.notifier,
+ (void *) entry->key.address,
+ entry->key.length,
+ (uint64_t) entry);
+}
+
+/**
+ * Stop monitoring a memory region
+ *
+ * @param[in] cache a memory registration cache
+ * @param[in] entry a memory registration entry
+ *
+ * @return nothing
+ */
+static void
+__notifier_unmonitor(gnix_mr_cache_t *cache,
+ gnix_mr_cache_entry_t *entry)
+{
+ int rc;
+
+ if (!cache->attr.lazy_deregistration) {
+ return;
+ }
+
+ if (cache->attr.notifier == NULL) {
+ return;
+ }
+
+ __clear_notifier_events(cache);
+
+ if (!__entry_is_unmapped(entry)) {
+ GNIX_DEBUG(FI_LOG_MR, "unmonitoring entry=%p (state=%d)\n",
+ entry, entry->state);
+ rc = _gnix_notifier_unmonitor(cache->attr.notifier,
+ (uint64_t) entry);
+ if (rc != FI_SUCCESS) {
+ /* This probably is okay (ESRCH), because the
+ * memory could have been unmapped in the
+ * interim, so clear the notifier events
+ * again */
+ GNIX_DEBUG(FI_LOG_MR,
+ "failed to unmonitor memory (entry=%p),"
+ " so clear notifier events again\n",
+ entry, fi_strerror(-rc));
+
+ __clear_notifier_events(cache);
+ }
+ }
+}
+
+/**
* Destroys the memory registration cache entry and deregisters the memory
* region with uGNI
*
@@ -352,15 +665,21 @@ static inline int __mr_cache_entry_destroy(gnix_mr_cache_t *cache,
rc = cache->attr.dereg_callback(entry->mr,
cache->attr.destruct_context);
if (!rc) {
- entry->state = GNIX_CES_DEAD;
+ /* Should we bother with this check? If we don't, the
+ * only difference it that __clear_notifier_events
+ * will be called one additional time. */
+ if (!__entry_is_unmapped(entry)) {
+ __notifier_unmonitor(cache, entry);
+ }
+ __entry_reset_state(entry);
rc = cache->attr.destruct_callback(cache->attr.dereg_context);
if (!rc)
free(entry);
} else {
- GNIX_WARN(FI_LOG_MR, "failed to deregister memory"
- " region with callback, "
- "cache_entry=%p ret=%i\n", entry, rc);
+ GNIX_INFO(FI_LOG_MR, "failed to deregister memory"
+ " region with callback, "
+ "cache_entry=%p ret=%i\n", entry, rc);
}
return rc;
@@ -373,6 +692,14 @@ static inline int __insert_entry_into_stale(
RbtStatus rc;
int ret = 0;
+ if (__entry_is_unmapped(entry)) {
+ GNIX_DEBUG(FI_LOG_MR, "entry (%p) unmapped, not inserting"
+ " into stale %llx:%llx", entry,
+ entry->key.address, entry->key.length);
+ /* Should we return some other value? */
+ return ret;
+ }
+
rc = rbtInsert(cache->stale.rb_tree,
&entry->key,
entry);
@@ -387,14 +714,23 @@ static inline int __insert_entry_into_stale(
ret = __mr_cache_entry_destroy(cache, entry);
} else {
- GNIX_INFO(FI_LOG_MR,
- "inserted key=%llx:%llx into stale\n",
- entry->key.address,
- entry->key.length);
+ GNIX_DEBUG(FI_LOG_MR,
+ "inserted key=%llx:%llx into stale\n",
+ entry->key.address, entry->key.length);
__mr_cache_lru_enqueue(cache, entry);
atomic_inc(&cache->stale.elements);
- entry->state = GNIX_CES_STALE;
+ switch (__entry_get_state(entry)) {
+ case GNIX_CES_INUSE:
+ __entry_set_state(entry, GNIX_CES_STALE);
+ break;
+ default:
+ GNIX_FATAL(FI_LOG_MR,
+ "stale entry (%p) %llx:%llx in bad"
+ " state (%d)\n", entry,
+ entry->key.address, entry->key.length,
+ entry->state);
+ }
}
return ret;
@@ -413,7 +749,11 @@ static inline void __resolve_stale_entry_collision(
RbtIterator iter = found;
int add_new_entry = 1, cmp;
- GNIX_INFO(FI_LOG_MR, "resolving collisions\n");
+ GNIX_TRACE(FI_LOG_MR, "\n");
+
+ GNIX_DEBUG(FI_LOG_MR,
+ "resolving collisions with entry (%p) %llx:%llx\n",
+ entry, entry->key.address, entry->key.length);
while (iter) {
rbtKeyValue(cache->stale.rb_tree, iter, (void **) &c_key,
@@ -425,9 +765,10 @@ static inline void __resolve_stale_entry_collision(
if (__can_subsume(&entry->key, c_key) ||
(entry->key.length > c_key->length)) {
- GNIX_INFO(FI_LOG_MR,
- "adding stale entry to destroy list, key=%llx:%llx",
- c_key->address, c_key->length);
+ GNIX_DEBUG(FI_LOG_MR,
+ "adding stale entry (%p) to destroy list,"
+ " key=%llx:%llx\n", c_entry,
+ c_key->address, c_key->length);
dlist_insert_tail(&c_entry->siblings, &to_destroy);
} else {
add_new_entry = 0;
@@ -441,34 +782,50 @@ static inline void __resolve_stale_entry_collision(
*/
dlist_for_each_safe(&to_destroy, c_entry, tmp, siblings)
{
- GNIX_INFO(FI_LOG_MR, "removing key from tree, key=%llx:%ll\n",
- c_entry->key.address, c_entry->key.length);
+ GNIX_DEBUG(FI_LOG_MR, "removing key from tree, entry %p"
+ " key=%llx:%llx\n", c_entry,
+ c_entry->key.address, c_entry->key.length);
iter = rbtFind(cache->stale.rb_tree, &c_entry->key);
- assert(iter);
+ if (unlikely(!iter)) {
+ GNIX_FATAL(FI_LOG_MR, "key not found\n");
+ }
rc = rbtErase(cache->stale.rb_tree,
iter);
- assert(rc == RBT_STATUS_OK);
+ if (unlikely(rc != RBT_STATUS_OK)) {
+ GNIX_FATAL(FI_LOG_MR,
+ "could not remove entry from tree\n");
+ }
- dlist_remove(&c_entry->lru_entry);
- dlist_remove(&c_entry->siblings);
+ if (__mr_cache_lru_remove(cache, c_entry) != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_MR, "Failed to remove entry"
+ " from lru list (%p)\n",
+ c_entry);
+ }
atomic_dec(&cache->stale.elements);
+ dlist_remove(&c_entry->siblings);
__mr_cache_entry_destroy(cache, c_entry);
}
- assert(dlist_empty(&to_destroy));
+ if (unlikely(!dlist_empty(&to_destroy))) {
+ GNIX_FATAL(FI_LOG_MR, "to_destroy not empty\n");
+ }
if (add_new_entry) {
ret = __insert_entry_into_stale(cache, entry);
- assert(!ret);
+ if (ret) {
+ GNIX_FATAL(FI_LOG_MR,
+ "Failed to insert subsumed MR "
+ " entry (%p) into stale list\n",
+ entry);
+ }
} else {
/* stale entry is larger than this one
* so lets just toss this entry out
*/
- GNIX_INFO(FI_LOG_MR,
- "larger entry already exists, "
- "to_destroy=%llx:%llx\n",
- entry->key.address,
- entry->key.length);
+ GNIX_DEBUG(FI_LOG_MR,
+ "larger entry already exists, "
+ "to_destroy=%llx:%llx\n",
+ entry->key.address, entry->key.length);
ret = __mr_cache_entry_destroy(cache, entry);
if (ret) {
@@ -492,6 +849,8 @@ static inline int __mr_cache_entry_get(
gnix_mr_cache_t *cache,
gnix_mr_cache_entry_t *entry)
{
+ GNIX_TRACE(FI_LOG_MR, "\n");
+
return atomic_inc(&entry->ref_cnt);
}
@@ -508,11 +867,18 @@ static inline int __mr_cache_entry_put(
gnix_mr_cache_entry_t *entry)
{
RbtIterator iter;
+ int rc;
gni_return_t grc = GNI_RC_SUCCESS;
RbtIterator found;
gnix_mr_cache_entry_t *parent = NULL;
struct dlist_entry *next;
+ GNIX_TRACE(FI_LOG_MR, "\n");
+
+ if (cache->attr.lazy_deregistration) {
+ __clear_notifier_events(cache);
+ }
+
if (atomic_dec(&entry->ref_cnt) == 0) {
next = entry->siblings.next;
dlist_remove(&entry->children);
@@ -524,7 +890,6 @@ static inline int __mr_cache_entry_put(
if (next != &entry->siblings && dlist_empty(next)) {
parent = container_of(next, gnix_mr_cache_entry_t,
children);
-
grc = __mr_cache_entry_put(cache, parent);
if (unlikely(grc != GNI_RC_SUCCESS)) {
GNIX_ERR(FI_LOG_MR,
@@ -537,13 +902,18 @@ static inline int __mr_cache_entry_put(
atomic_dec(&cache->inuse.elements);
- if (!(entry->flags & GNIX_CE_RETIRED)) {
+ if (!__entry_is_retired(entry)) {
iter = rbtFind(cache->inuse.rb_tree, &entry->key);
if (unlikely(!iter)) {
GNIX_ERR(FI_LOG_MR,
"failed to find entry in the inuse cache\n");
} else {
- rbtErase(cache->inuse.rb_tree, iter);
+ rc = rbtErase(cache->inuse.rb_tree, iter);
+ if (unlikely(rc != RBT_STATUS_OK)) {
+ GNIX_ERR(FI_LOG_MR,
+ "failed to erase lru entry"
+ " from stale tree\n");
+ }
}
}
@@ -551,9 +921,10 @@ static inline int __mr_cache_entry_put(
* isn't retired, put it in the stale cache
*/
if (cache->attr.lazy_deregistration &&
- !(entry->flags & GNIX_CE_RETIRED)) {
- GNIX_INFO(FI_LOG_MR, "moving key %llx:%llx to stale\n",
- entry->key.address, entry->key.length);
+ !(__entry_is_retired(entry))) {
+ GNIX_DEBUG(FI_LOG_MR,
+ "moving key %llx:%llx to stale\n",
+ entry->key.address, entry->key.length);
found = rbtFindLeftmost(cache->stale.rb_tree,
&entry->key, __find_overlapping_addr);
@@ -569,18 +940,16 @@ static inline int __mr_cache_entry_put(
}
} else {
/* if retired or not using lazy registration */
- GNIX_INFO(FI_LOG_MR,
- "destroying entry, key=%llx:%llx\n",
- entry->key.address,
- entry->key.length);
+ GNIX_DEBUG(FI_LOG_MR,
+ "destroying entry, key=%llx:%llx\n",
+ entry->key.address, entry->key.length);
grc = __mr_cache_entry_destroy(cache, entry);
}
if (unlikely(grc != GNI_RC_SUCCESS)) {
- GNIX_WARN(FI_LOG_MR,
- "dereg callback returned '%s'\n",
- gni_err_str[grc]);
+ GNIX_INFO(FI_LOG_MR, "dereg callback returned '%s'\n",
+ gni_err_str[grc]);
}
}
@@ -612,8 +981,8 @@ static inline int __check_mr_cache_attr_sanity(gnix_mr_cache_attr_t *attr)
}
int _gnix_mr_cache_init(
- gnix_mr_cache_t **cache,
- gnix_mr_cache_attr_t *attr)
+ gnix_mr_cache_t **cache,
+ gnix_mr_cache_attr_t *attr)
{
gnix_mr_cache_attr_t *cache_attr = &__default_mr_cache_attr;
gnix_mr_cache_t *cache_p;
@@ -621,9 +990,6 @@ int _gnix_mr_cache_init(
GNIX_TRACE(FI_LOG_MR, "\n");
- if (!cache)
- return -FI_EINVAL;
-
/* if the provider asks us to use their attributes, are they sane? */
if (attr) {
if (__check_mr_cache_attr_sanity(attr) != FI_SUCCESS)
@@ -672,6 +1038,7 @@ int _gnix_mr_cache_init(
cache_p->misses = 0;
cache_p->state = GNIX_MRC_STATE_READY;
+
*cache = cache_p;
return FI_SUCCESS;
@@ -725,13 +1092,12 @@ int _gnix_mr_cache_destroy(gnix_mr_cache_t *cache)
int __mr_cache_flush(gnix_mr_cache_t *cache, int flush_count) {
int rc;
RbtIterator iter;
- gnix_mr_cache_key_t *e_key;
- gnix_mr_cache_entry_t *entry, *e_entry;
+ gnix_mr_cache_entry_t *entry;
int destroyed = 0;
GNIX_TRACE(FI_LOG_MR, "\n");
- GNIX_INFO(FI_LOG_MR, "starting flush on memory registration cache\n");
+ GNIX_DEBUG(FI_LOG_MR, "starting flush on memory registration cache\n");
/* flushes are unnecessary for caches without lazy deregistration */
if (!cache->attr.lazy_deregistration)
@@ -749,30 +1115,18 @@ int __mr_cache_flush(gnix_mr_cache_t *cache, int flush_count) {
break;
}
- GNIX_INFO(FI_LOG_MR, "attempting to flush key %llx:%llx\n",
- entry->key.address, entry->key.length);
+ GNIX_DEBUG(FI_LOG_MR, "attempting to flush key %llx:%llx\n",
+ entry->key.address, entry->key.length);
iter = rbtFind(cache->stale.rb_tree, &entry->key);
if (unlikely(!iter)) {
GNIX_ERR(FI_LOG_MR,
- "lru entries MUST be present in the cache,"
- " could not find key in stale tree\n");
+ "lru entries MUST be present in the cache,"
+ " could not find entry (%p) in stale tree"
+ " %llx:%llx\n",
+ entry, entry->key.address, entry->key.length);
break;
}
- rbtKeyValue(cache->stale.rb_tree, iter, (void **) &e_key,
- (void **) &e_entry);
- if (e_entry != entry) {
- /* If not an exact match, remove the found entry,
- and then put the original entry back on the LRU list */
- GNIX_INFO(FI_LOG_MR,
- "Flushing non-lru entry %llx:%llx\n",
- e_entry->key.address, e_entry->key.length);
- dlist_remove(&e_entry->lru_entry);
- dlist_insert_tail(&entry->lru_entry, &cache->lru_head);
- /* Destroy the actual entry below */
- entry = e_entry;
- }
-
rc = rbtErase(cache->stale.rb_tree, iter);
if (unlikely(rc != RBT_STATUS_OK)) {
GNIX_ERR(FI_LOG_MR,
@@ -785,9 +1139,9 @@ int __mr_cache_flush(gnix_mr_cache_t *cache, int flush_count) {
++destroyed;
}
- GNIX_INFO(FI_LOG_MR, "flushed %i of %i entries from memory "
- "registration cache\n", destroyed,
- atomic_get(&cache->stale.elements));
+ GNIX_DEBUG(FI_LOG_MR, "flushed %i of %i entries from memory "
+ "registration cache\n", destroyed,
+ atomic_get(&cache->stale.elements));
if (destroyed > 0) {
atomic_sub(&cache->stale.elements, destroyed);
@@ -822,6 +1176,10 @@ static int __mr_cache_search_inuse(
uint64_t new_end, found_end;
DLIST_HEAD(retired_entries);
+ if (cache->attr.lazy_deregistration) {
+ __clear_notifier_events(cache);
+ }
+
/* first we need to find an entry that overlaps with this one.
* care should be taken to find the left most entry that overlaps
* with this entry since the entry we are searching for might overlap
@@ -831,31 +1189,31 @@ static int __mr_cache_search_inuse(
iter = rbtFindLeftmost(cache->inuse.rb_tree, (void *) key,
__find_overlapping_addr);
if (!iter) {
- GNIX_INFO(FI_LOG_MR,
- "could not find key in inuse, key=%llx:%llx\n",
- key->address, key->length);
+ GNIX_DEBUG(FI_LOG_MR,
+ "could not find key in inuse, key=%llx:%llx\n",
+ key->address, key->length);
return -FI_ENOENT;
}
rbtKeyValue(cache->inuse.rb_tree, iter, (void **) &found_key,
(void **) &found_entry);
- GNIX_INFO(FI_LOG_MR,
- "found a key that matches the search criteria, "
- "found=%llx:%llx key=%llx:%llx\n",
- found_key->address, found_key->length,
- key->address, key->length);
+ GNIX_DEBUG(FI_LOG_MR,
+ "found a key that matches the search criteria, "
+ "found=%llx:%llx key=%llx:%llx\n",
+ found_key->address, found_key->length,
+ key->address, key->length);
/* if the entry that we've found completely subsumes
* the requested entry, just return a reference to
* that existing registration
*/
if (__can_subsume(found_key, key)) {
- GNIX_INFO(FI_LOG_MR,
- "found an entry that subsumes the request, "
- "existing=%llx:%llx key=%llx:%llx\n",
- found_key->address, found_key->length,
- key->address, key->length);
+ GNIX_DEBUG(FI_LOG_MR,
+ "found an entry that subsumes the request, "
+ "existing=%llx:%llx key=%llx:%llx\n",
+ found_key->address, found_key->length,
+ key->address, key->length);
*entry = found_entry;
__mr_cache_entry_get(cache, found_entry);
@@ -875,10 +1233,10 @@ static int __mr_cache_search_inuse(
cmp = __find_overlapping_addr(found_key, key);
- GNIX_INFO(FI_LOG_MR,
- "candidate: key=%llx:%llx result=%d\n",
- found_key->address,
- found_key->length, cmp);
+ GNIX_DEBUG(FI_LOG_MR,
+ "candidate: key=%llx:%llx result=%d\n",
+ found_key->address,
+ found_key->length, cmp);
if (cmp != 0)
break;
@@ -886,9 +1244,9 @@ static int __mr_cache_search_inuse(
found_end = found_key->address + found_key->length;
/* mark the entry as retired */
- GNIX_INFO(FI_LOG_MR, "retiring entry, key=%llx:%llx\n",
- found_key->address, found_key->length);
- found_entry->flags |= GNIX_CE_RETIRED;
+ GNIX_DEBUG(FI_LOG_MR, "retiring entry, key=%llx:%llx\n",
+ found_key->address, found_key->length);
+ __entry_set_retired(found_entry);
dlist_insert_tail(&found_entry->siblings, &retired_entries);
iter = rbtNext(cache->inuse.rb_tree, iter);
@@ -900,24 +1258,36 @@ static int __mr_cache_search_inuse(
/* remove retired entries from tree */
- GNIX_INFO(FI_LOG_MR, "removing retired entries from inuse tree\n");
+ GNIX_DEBUG(FI_LOG_MR, "removing retired entries from inuse tree\n");
__remove_sibling_entries_from_tree(cache,
&retired_entries, cache->inuse.rb_tree);
/* create new registration */
- GNIX_INFO(FI_LOG_MR,
- "creating a new merged registration, key=%llx:%llx\n",
- new_key.address, new_key.length);
+ GNIX_DEBUG(FI_LOG_MR,
+ "creating a new merged registration, key=%llx:%llx\n",
+ new_key.address, new_key.length);
ret = __mr_cache_create_registration(cache,
new_key.address, new_key.length,
entry, &new_key, fi_reg_context);
if (ret) {
- GNIX_ERR(FI_LOG_MR, "failure in registration, can't really "
- "recover at this point since we've already retired "
- "the existing entries\n");
+ /* If we get here, one of two things have happened.
+ * Either some part of the new merged registration was
+ * unmapped (i.e., freed by user) or the merged
+ * registration failed for some other reason (probably
+ * GNI_RC_ERROR_RESOURCE). The first case is a user
+ * error (which they should have been warned about by
+ * the notifier), and the second case is always
+ * possible. Neither case is a problem. The entries
+ * above have been retired, and here we return the
+ * error */
+ GNIX_DEBUG(FI_LOG_MR,
+ "failed to create merged registration, key=",
+ new_key.address, new_key.length);
return ret;
}
+ __entry_set_merged(*entry);
+
/* move retired entries to the head of the new entry's child list */
if (!dlist_empty(&retired_entries)) {
__attach_retired_entries_to_registration(cache,
@@ -943,8 +1313,12 @@ static int __mr_cache_search_stale(
gnix_mr_cache_key_t *mr_key;
gnix_mr_cache_entry_t *mr_entry, *tmp;
- GNIX_INFO(FI_LOG_MR, "searching for stale entry, key=%llx:%llx\n",
- key->address, key->length);
+ if (cache->attr.lazy_deregistration) {
+ __clear_notifier_events(cache);
+ }
+
+ GNIX_DEBUG(FI_LOG_MR, "searching for stale entry, key=%llx:%llx\n",
+ key->address, key->length);
iter = rbtFindLeftmost(cache->stale.rb_tree, (void *) key,
__find_overlapping_addr);
@@ -954,10 +1328,10 @@ static int __mr_cache_search_stale(
rbtKeyValue(cache->stale.rb_tree, iter, (void **) &mr_key,
(void **) &mr_entry);
- GNIX_INFO(FI_LOG_MR,
- "found a matching entry, found=%llx:%llx key=%llx:%llx\n",
- mr_key->address, mr_key->length,
- key->address, key->length);
+ GNIX_DEBUG(FI_LOG_MR,
+ "found a matching entry, found=%llx:%llx key=%llx:%llx\n",
+ mr_key->address, mr_key->length,
+ key->address, key->length);
/* if the entry that we've found completely subsumes
@@ -976,35 +1350,45 @@ static int __mr_cache_search_stale(
* The old entry (mr_entry) should be destroyed
* now as it is no longer needed.
*/
- GNIX_INFO(FI_LOG_MR,
- "removing entry from stale key=%llx:%llx\n",
- mr_key->address, mr_key->length);
+ GNIX_DEBUG(FI_LOG_MR,
+ "removing entry from stale key=%llx:%llx\n",
+ mr_key->address, mr_key->length);
rc = rbtErase(cache->stale.rb_tree, iter);
if (unlikely(rc != RBT_STATUS_OK)) {
GNIX_ERR(FI_LOG_MR,
"failed to erase entry from stale tree\n");
} else {
- dlist_remove(&mr_entry->lru_entry);
-
- atomic_dec(&cache->stale.elements);
-
+ if (__mr_cache_lru_remove(cache, mr_entry)
+ == FI_SUCCESS) {
+ atomic_dec(&cache->stale.elements);
+ } else {
+ GNIX_WARN(FI_LOG_MR, "Failed to remove"
+ " entry (%p) from lru list\n",
+ mr_entry);
+ }
__mr_cache_entry_destroy(cache, mr_entry);
}
*entry = tmp;
} else {
- GNIX_INFO(FI_LOG_MR,
- "removing entry from stale and migrating to inuse, "
- "key=%llx:%llx\n",
- mr_key->address, mr_key->length);
+ GNIX_DEBUG(FI_LOG_MR,
+ "removing entry (%p) from stale and"
+ " migrating to inuse, key=%llx:%llx\n",
+ mr_entry, mr_key->address, mr_key->length);
rc = rbtErase(cache->stale.rb_tree, iter);
- if (unlikely(rc != RBT_STATUS_OK))
- GNIX_WARN(FI_LOG_MR,
- "failed to erase entry from stale tree\n");
- assert(rc == RBT_STATUS_OK);
+ if (unlikely(rc != RBT_STATUS_OK)) {
+ GNIX_FATAL(FI_LOG_MR,
+ "failed to erase entry (%p) from "
+ " stale tree\n", mr_entry);
+ }
- dlist_remove(&mr_entry->lru_entry);
+ if (__mr_cache_lru_remove(cache, mr_entry)
+ != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_MR, "Failed to remove"
+ " entry (%p) from lru list\n",
+ mr_entry);
+ }
atomic_dec(&cache->stale.elements);
@@ -1014,10 +1398,11 @@ static int __mr_cache_search_stale(
*/
rc = rbtInsert(cache->inuse.rb_tree,
&mr_entry->key, mr_entry);
- if (unlikely(rc != RBT_STATUS_OK))
- GNIX_WARN(FI_LOG_MR,
- "failed to insert entry into inuse tree\n");
- assert(rc == RBT_STATUS_OK);
+ if (unlikely(rc != RBT_STATUS_OK)) {
+ GNIX_FATAL(FI_LOG_MR,
+ "failed to insert entry into"
+ "inuse tree\n");
+ }
atomic_set(&mr_entry->ref_cnt, 1);
atomic_inc(&cache->inuse.elements);
@@ -1028,10 +1413,10 @@ static int __mr_cache_search_stale(
return FI_SUCCESS;
}
- GNIX_INFO(FI_LOG_MR,
- "could not use matching entry, "
- "found=%llx:%llx\n",
- mr_key->address, mr_key->length);
+ GNIX_DEBUG(FI_LOG_MR,
+ "could not use matching entry, "
+ "found=%llx:%llx\n",
+ mr_key->address, mr_key->length);
return -FI_ENOENT;
}
@@ -1062,15 +1447,23 @@ static int __mr_cache_create_registration(
handle = cache->attr.reg_callback(handle, (void *) address, length,
fi_reg_context, cache->attr.reg_context);
if (unlikely(!handle)) {
- free(current_entry);
- GNIX_WARN(FI_LOG_MR, "failed to register memory with callback");
- return -FI_ENOMEM;
+ GNIX_INFO(FI_LOG_MR,
+ "failed to register memory with callback\n");
+ goto err;
}
+ __entry_reset_state(current_entry);
+
/* set up the entry's key */
current_entry->key.address = address;
current_entry->key.length = length;
- current_entry->flags = 0;
+
+ rc = __notifier_monitor(cache, current_entry);
+ if (unlikely(rc != FI_SUCCESS)) {
+ GNIX_INFO(FI_LOG_MR,
+ "failed to monitor memory with notifier\n");
+ goto err_dereg;
+ }
rc = rbtInsert(cache->inuse.rb_tree, ¤t_entry->key,
current_entry);
@@ -1078,22 +1471,11 @@ static int __mr_cache_create_registration(
GNIX_ERR(FI_LOG_MR, "failed to insert registration "
"into cache, ret=%i\n", rc);
- rc = cache->attr.dereg_callback(current_entry->mr,
- cache->attr.dereg_context);
- if (unlikely(rc)) {
- GNIX_WARN(FI_LOG_MR,
- "failed to deregister memory with "
- "callback, ret=%d\n",
- rc);
- }
-
- free(current_entry);
-
- return -FI_ENOMEM;
+ goto err_dereg;
}
- GNIX_INFO(FI_LOG_MR, "inserted key %llx:%llx into inuse\n",
- current_entry->key.address, current_entry->key.length);
+ GNIX_DEBUG(FI_LOG_MR, "inserted key %llx:%llx into inuse\n",
+ current_entry->key.address, current_entry->key.length);
atomic_inc(&cache->inuse.elements);
@@ -1102,6 +1484,18 @@ static int __mr_cache_create_registration(
*entry = current_entry;
return FI_SUCCESS;
+
+err_dereg:
+ rc = cache->attr.dereg_callback(current_entry->mr,
+ cache->attr.dereg_context);
+ if (unlikely(rc)) {
+ GNIX_INFO(FI_LOG_MR,
+ "failed to deregister memory with "
+ "callback, ret=%d\n", rc);
+ }
+err:
+ free(current_entry);
+ return -FI_ENOMEM;
}
/**
@@ -1141,10 +1535,14 @@ int _gnix_mr_cache_register(
/* if we shouldn't introduce any new elements, return -FI_ENOSPC */
if (unlikely(cache->attr.hard_reg_limit > 0 &&
(atomic_get(&cache->inuse.elements) >=
- cache->attr.hard_reg_limit)))
- return -FI_ENOSPC;
+ cache->attr.hard_reg_limit))) {
+ ret = -FI_ENOSPC;
+ goto err;
+ }
if (cache->attr.lazy_deregistration) {
+ __clear_notifier_events(cache);
+
/* if lazy deregistration is in use, we can check the
* stale tree
*/
@@ -1166,22 +1564,25 @@ int _gnix_mr_cache_register(
ret = __mr_cache_create_registration(cache, address, length,
&entry, &key, fi_reg_context);
- if (ret)
- return ret;
+ if (ret) {
+ goto err;
+ }
cache->misses++;
success:
- entry->state = GNIX_CES_INUSE;
+ __entry_set_state(entry, GNIX_CES_INUSE);
*handle = (void *) entry->mr;
return FI_SUCCESS;
+
+err:
+ return ret;
}
/**
* Function to deregister memory in the cache
*
- * @param[in] cache gnix memory registration cache pointer
* @param[in] mr gnix memory registration descriptor pointer
*
* @return FI_SUCCESS on success
@@ -1203,11 +1604,14 @@ int _gnix_mr_cache_deregister(
*/
entry = container_of(handle, gnix_mr_cache_entry_t, mr);
- if (entry->state != GNIX_CES_INUSE)
+ if (__entry_get_state(entry) != GNIX_CES_INUSE) {
+ GNIX_INFO(FI_LOG_MR, "entry (%p) in incorrect state (%d)\n",
+ entry, entry->state);
return -FI_EINVAL;
+ }
- GNIX_INFO(FI_LOG_MR, "entry found, entry=%p refs=%d\n",
- entry, atomic_get(&entry->ref_cnt));
+ GNIX_DEBUG(FI_LOG_MR, "entry found, entry=%p refs=%d\n",
+ entry, atomic_get(&entry->ref_cnt));
grc = __mr_cache_entry_put(cache, entry);
diff --git a/prov/gni/src/gnix_mr_notifier.c b/prov/gni/src/gnix_mr_notifier.c
new file mode 100644
index 0000000..174cc6c
--- /dev/null
+++ b/prov/gni/src/gnix_mr_notifier.c
@@ -0,0 +1,247 @@
+/*
+ * Copyright (c) 2016 Cray Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifdef HAVE_KDREG
+
+#include "gnix_mr_notifier.h"
+
+static inline int
+notifier_verify_stuff(struct gnix_mr_notifier *mrn) {
+ /* Can someone confirm that these values are POSIX so we can
+ * be less pedantic? */
+ if (mrn->fd == STDIN_FILENO ||
+ mrn->fd == STDOUT_FILENO ||
+ mrn->fd == STDERR_FILENO ||
+ mrn->fd < 0) {
+ // Be quiet here
+ return -FI_EBADF;
+ }
+
+ if (mrn->cntr == NULL) {
+ // Be quiet here
+ return -FI_ENODATA;
+ }
+
+ return FI_SUCCESS;
+}
+
+int
+_gnix_notifier_init(struct gnix_mr_notifier *mrn)
+{
+ if (mrn == NULL) {
+ GNIX_INFO(FI_LOG_MR, "mr notifier NULL\n");
+ return -FI_EINVAL;
+ }
+
+ mrn->fd = 0;
+ mrn->cntr = NULL;
+ fastlock_init(&mrn->lock);
+
+ return FI_SUCCESS;
+}
+
+int
+_gnix_notifier_open(struct gnix_mr_notifier *mrn)
+{
+ int ret = FI_SUCCESS;
+ int kdreg_fd, ret_errno;
+ kdreg_get_user_delta_args_t get_user_delta_args;
+
+ if ((mrn->fd != 0) || (mrn->cntr != NULL)) {
+ GNIX_INFO(FI_LOG_MR, "mr notifier already open\n");
+ return -FI_EBUSY;
+ }
+
+ fastlock_acquire(&mrn->lock);
+
+ kdreg_fd = open(KDREG_DEV, O_RDWR | O_NONBLOCK);
+ if (kdreg_fd < 0) {
+ ret_errno = errno;
+ if (ret_errno != FI_EBUSY) {
+ GNIX_INFO(FI_LOG_MR, "kdreg device open failed: %s\n",
+ strerror(ret_errno));
+ }
+ // Not all of these map to fi_errno values
+ ret = -ret_errno;
+ goto err_exit;
+ }
+
+ (void) memset(&get_user_delta_args,0,sizeof(get_user_delta_args));
+ if (ioctl(kdreg_fd, KDREG_IOC_GET_USER_DELTA,
+ &get_user_delta_args) < 0) {
+ ret_errno = errno;
+ GNIX_INFO(FI_LOG_MR, "kdreg get_user_delta failed: %s\n",
+ strerror(ret_errno));
+ close(kdreg_fd);
+ // Not all of these map to fi_errno values
+ ret = -ret_errno;
+ goto err_exit;
+ }
+
+ if (get_user_delta_args.user_delta == NULL) {
+ GNIX_INFO(FI_LOG_MR, "kdreg get_user_delta is NULL\n");
+ ret = -FI_ENODATA;
+ goto err_exit;
+ }
+
+ mrn->fd = kdreg_fd;
+ mrn->cntr = (kdreg_user_delta_t *) get_user_delta_args.user_delta;
+
+err_exit:
+ fastlock_release(&mrn->lock);
+
+ return ret;
+}
+
+int
+_gnix_notifier_close(struct gnix_mr_notifier *mrn)
+{
+ int ret = FI_SUCCESS;
+ int ret_errno;
+
+ ret = notifier_verify_stuff(mrn);
+
+ if (ret == 0) {
+ fastlock_acquire(&mrn->lock);
+
+ if (close(mrn->fd) != 0) {
+ ret_errno = errno;
+ GNIX_INFO(FI_LOG_MR, "error closing kdreg device: %s\n",
+ strerror(ret_errno));
+ // Not all of these map to fi_errno values
+ ret = -ret_errno;
+ goto err_exit;
+ }
+
+ mrn->cntr = NULL;
+ err_exit:
+ fastlock_release(&mrn->lock);
+ }
+
+ return ret;
+}
+
+static inline int
+kdreg_write(struct gnix_mr_notifier *mrn, void *buf, size_t len) {
+ int ret;
+
+ ret = write(mrn->fd, buf, len);
+ if ((ret < 0) || (ret != len)) {
+ // Not all of these map to fi_errno values
+ ret = -errno;
+ GNIX_INFO(FI_LOG_MR, "kdreg_write failed: %s\n",
+ strerror(errno));
+ return ret;
+ }
+
+ return FI_SUCCESS;
+}
+
+int
+_gnix_notifier_monitor(struct gnix_mr_notifier *mrn,
+ void *addr, uint64_t len, uint64_t cookie)
+{
+ int ret;
+ struct registration_monitor rm;
+
+ ret = notifier_verify_stuff(mrn);
+
+ if (ret == 0) {
+ GNIX_DEBUG(FI_LOG_MR, "monitoring %p (len=%lu) cookie=%lu\n",
+ addr, len, cookie);
+
+ memset(&rm, 0, sizeof(rm));
+ rm.type = REGISTRATION_MONITOR;
+ rm.u.mon.addr = (uint64_t) addr;
+ rm.u.mon.len = len;
+ rm.u.mon.user_cookie = cookie;
+
+ ret = kdreg_write(mrn, &rm, sizeof(rm));
+ }
+
+ return ret;
+}
+
+int
+_gnix_notifier_unmonitor(struct gnix_mr_notifier *mrn, uint64_t cookie)
+{
+ int ret;
+ struct registration_monitor rm;
+
+ ret = notifier_verify_stuff(mrn);
+ if (ret == 0) {
+ GNIX_DEBUG(FI_LOG_MR, "unmonitoring cookie=%lu\n", cookie);
+
+ memset(&rm, 0, sizeof(rm));
+
+ rm.type = REGISTRATION_UNMONITOR;
+ rm.u.unmon.user_cookie = cookie;
+
+ ret = kdreg_write(mrn, &rm, sizeof(rm));
+ }
+
+ return ret;
+}
+
+int
+_gnix_notifier_get_event(struct gnix_mr_notifier *mrn, void* buf, size_t len)
+{
+ int ret, ret_errno;
+
+ if ((mrn == NULL) || (buf == NULL) || (len <= 0)) {
+ GNIX_WARN(FI_LOG_MR,
+ "Invalid argument to _gnix_notifier_get_event\n");
+ return -FI_EINVAL;
+ }
+
+ if (*(mrn->cntr) > 0) {
+ GNIX_DEBUG(FI_LOG_MR, "reading kdreg event\n");
+ ret = read(mrn->fd, buf, len);
+ if (ret >= 0) {
+ return ret;
+ } else {
+ ret_errno = errno;
+ if (ret_errno != EAGAIN) {
+ GNIX_WARN(FI_LOG_MR,
+ "kdreg event read failed: %s\n",
+ strerror(ret_errno));
+ }
+ // Not all of these map to fi_errno values
+ return -ret_errno;
+ }
+ } else {
+ GNIX_DEBUG(FI_LOG_MR, "nothing to read from kdreg :(\n");
+ return -FI_EAGAIN;
+ }
+}
+
+#endif /* HAVE_KDREG */
diff --git a/prov/gni/src/gnix_msg.c b/prov/gni/src/gnix_msg.c
index aa576f1..7678b1d 100644
--- a/prov/gni/src/gnix_msg.c
+++ b/prov/gni/src/gnix_msg.c
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -45,6 +45,7 @@
#include "gnix_cntr.h"
#include "gnix_av.h"
#include "gnix_rma.h"
+#include "gnix_atomic.h"
#define INVALID_PEEK_FORMAT(fmt) \
((fmt) == FI_CQ_FORMAT_CONTEXT || (fmt) == FI_CQ_FORMAT_MSG)
@@ -54,6 +55,180 @@
/*******************************************************************************
* helper functions
******************************************************************************/
+/**
+ * This function return the receiver's address given that we are at offset cur_len
+ * within the send_iov.
+ *
+ * @param req the fabric request
+ * @param cur_len the current offset at which we found an unaligned head or tail
+ * within the sender's iov entries.
+ */
+static inline uint8_t *__gnix_msg_iov_unaligned_recv_addr(struct gnix_fab_req *req,
+ size_t cur_len)
+{
+ int i;
+
+ /* Find the recv_address for the given head data */
+ for (i = 0; i < req->msg.recv_iov_cnt; i++) {
+ if ((int64_t) cur_len - (int64_t) req->msg.recv_info[i].recv_len < 0) {
+ return (uint8_t *) (req->msg.recv_info[i].recv_addr + cur_len);
+ } else {
+ cur_len -= req->msg.recv_info[i].recv_len;
+ }
+ }
+
+ return NULL;
+}
+
+static inline void __gnix_msg_send_alignment(struct gnix_fab_req *req)
+{
+ int i;
+
+ /* Ensure all head and tail fields are initialized properly */
+ for (i = 0; i < req->msg.send_iov_cnt; i++) {
+ /* Check head for four byte alignment, if not aligned store
+ * the unaligned bytes (<=3) in head */
+ if (req->msg.send_info[i].send_addr & GNI_READ_ALIGN_MASK) {
+ req->msg.send_info[i].head =
+ *(uint32_t *)(req->msg.send_info[i].send_addr &
+ ~GNI_READ_ALIGN_MASK);
+
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "Sending %d unaligned head bytes (%x)\n",
+ GNI_READ_ALIGN -
+ (req->msg.send_info[i].send_addr &
+ GNI_READ_ALIGN_MASK),
+ req->msg.send_info[i].head);
+ }
+
+ /* Check tail for four byte alignment. */
+ if ((req->msg.send_info[i].send_addr +
+ req->msg.send_info[i].send_len) &
+ GNI_READ_ALIGN_MASK) {
+ req->msg.send_info[i].tail =
+ *(uint32_t *)((req->msg.send_info[i].send_addr +
+ req->msg.send_info[i].send_len) &
+ ~GNI_READ_ALIGN_MASK);
+
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "Sending %d unaligned tail bytes (%x)\n",
+ (req->msg.send_info[i].send_addr +
+ req->msg.send_info[i].send_len) &
+ GNI_READ_ALIGN_MASK,
+ req->msg.send_info[i].tail);
+ }
+ }
+}
+static inline void __gnix_msg_free_rma_txd(struct gnix_fab_req *req,
+ struct gnix_tx_descriptor *txd)
+{
+ gni_ct_get_post_descriptor_t *cur_ct, *nxt_ct;
+
+ if (txd->gni_desc.type & GNI_POST_FMA_GET) {
+ for (cur_ct = txd->gni_desc.next_descr;
+ cur_ct != NULL;
+ cur_ct = nxt_ct) {
+ nxt_ct = cur_ct->next_descr;
+ free(cur_ct);
+ }
+ }
+ _gnix_nic_tx_free(req->gnix_ep->nic, txd);
+}
+
+static inline void __gnix_msg_free_iov_txds(struct gnix_fab_req *req,
+ size_t txd_cnt)
+{
+ int i;
+
+ for (i = 0; i < txd_cnt; i++) {
+ __gnix_msg_free_rma_txd(req, req->iov_txds[i]);
+ }
+}
+
+static void __gnix_msg_unpack_data_into_iov(const struct recv_info_t *dest,
+ size_t dest_cnt,
+ uint64_t src, size_t src_len)
+{
+ static int i;
+ static size_t cum_len;
+ static size_t offset;
+ size_t bwrite;
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "Unpacking data for recvv count (%lu)"
+ " src = %p, dest = %p,"
+ " i = %d, cum_len = %lu, offset = %lu\n",
+ dest_cnt, src, dest, i, cum_len, offset);
+
+ /* Pull out each iov len and base field into the iov */
+ while (i < dest_cnt) {
+ bwrite = dest[i].recv_len - offset < src_len - cum_len ?
+ dest[i].recv_len - offset : src_len - cum_len;
+
+ memcpy((uint8_t *) dest[i].recv_addr + offset,
+ (void *) (src + cum_len), bwrite);
+
+ cum_len += bwrite;
+
+ /* Src exhausted */
+ if (cum_len == src_len) {
+ offset = dest[i].recv_len - bwrite;
+ cum_len = 0;
+
+ /* dest exhausted too, reset the counters */
+ if (!offset) {
+ i = cum_len = 0;
+ }
+
+ return;
+ }
+
+ i++;
+ }
+}
+
+static void __gnix_msg_pack_data_from_iov(uint64_t dest, size_t dest_len,
+ const struct iovec *src,
+ size_t src_cnt)
+{
+ int i;
+ size_t cum_len = 0;
+
+ GNIX_DEBUG(FI_LOG_EP_CTRL, "Packing data for sendv count (%d)\n",
+ src_cnt);
+
+ /* Pull out each iov len and base field into the dest buffer */
+ for (i = 0; i < src_cnt && cum_len < dest_len; i++) {
+ memcpy((void *) (dest + cum_len), src[i].iov_base,
+ src[i].iov_len < dest_len - cum_len ?
+ src[i].iov_len : dest_len - cum_len);
+ cum_len += src[i].iov_len;
+ }
+}
+
+static void __gnix_msg_copy_data_to_recv_addr(struct gnix_fab_req *req,
+ void *data)
+{
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
+ switch(req->type) {
+ case GNIX_FAB_RQ_RECV:
+ memcpy((void *)req->msg.recv_info[0].recv_addr, data,
+ req->msg.cum_send_len);
+ break;
+
+ case GNIX_FAB_RQ_RECVV:
+ case GNIX_FAB_RQ_TRECVV:
+ __gnix_msg_unpack_data_into_iov(req->msg.recv_info,
+ req->msg.recv_iov_cnt,
+ (uint64_t) data,
+ req->msg.cum_send_len);
+ break;
+
+ default:
+ GNIX_FATAL(FI_LOG_EP_DATA, "Invalid request type: %d", req->type);
+ return;
+ }
+}
static struct gnix_fab_req *__gnix_msg_dup_req(struct gnix_fab_req *req)
{
@@ -137,8 +312,9 @@ static int __gnix_msg_recv_err(struct gnix_fid_ep *ep, struct gnix_fab_req *req)
flags |= req->msg.send_flags & FI_TAGGED;
- return __recv_err(ep, req->user_context, flags, req->msg.recv_len,
- (void *)req->msg.recv_addr, req->msg.imm,
+ /* TODO: Check for FI_MULTI_RECV? */
+ return __recv_err(ep, req->user_context, flags, req->msg.cum_recv_len,
+ (void *)req->msg.recv_info[0].recv_addr, req->msg.imm,
req->msg.tag, 0, FI_ECANCELED,
GNI_RC_TRANSACTION_ERROR, NULL);
}
@@ -186,8 +362,17 @@ static inline int __gnix_msg_recv_completion(struct gnix_fid_ep *ep,
flags |= req->msg.recv_flags & (FI_PEEK | FI_CLAIM | FI_DISCARD |
FI_MULTI_RECV);
+ /*
+ * see fi_cq man page
+ */
+ if (req->msg.recv_flags & GNIX_MSG_MULTI_RECV_SUP)
+ flags &= ~FI_MULTI_RECV;
+
return __recv_completion(ep, req, req->user_context, flags,
- req->msg.send_len, (void *)req->msg.recv_addr,
+ MIN(req->msg.cum_send_len, req->msg.cum_recv_len),
+ req->msg.recv_flags & FI_MULTI_RECV ?
+ (void *)req->msg.recv_info[0].recv_addr :
+ NULL,
req->msg.imm, req->msg.tag,
_gnix_vc_peer_fi_addr(req->vc));
}
@@ -229,6 +414,8 @@ static int __gnix_msg_send_completion(struct gnix_fid_ep *ep,
flags |= req->msg.send_flags & FI_TAGGED;
+ GNIX_DEBUG(FI_LOG_EP_DATA, "send_cq = %p\n", ep->send_cq);
+
if ((req->msg.send_flags & FI_COMPLETION) && ep->send_cq) {
rc = _gnix_cq_add_event(ep->send_cq,
req->user_context,
@@ -277,18 +464,23 @@ static int __gnix_rndzv_req_send_fin(void *arg)
}
txd->rndzv_fin_hdr.req_addr = req->msg.rma_id;
+ txd->rndzv_fin_hdr.status = req->msg.status;
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "req_addr = %p\n",
+ (void *) txd->rndzv_fin_hdr.req_addr);
txd->req = req;
+
txd->completer_fn = gnix_ep_smsg_completers[GNIX_SMSG_T_RNDZV_FIN];
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
status = GNI_SmsgSendWTag(req->vc->gni_ep,
&txd->rndzv_fin_hdr, sizeof(txd->rndzv_fin_hdr),
NULL, 0, txd->id, GNIX_SMSG_T_RNDZV_FIN);
if ((status == GNI_RC_SUCCESS) &&
(ep->domain->data_progress == FI_PROGRESS_AUTO))
_gnix_rma_post_irq(req->vc);
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
if (status == GNI_RC_NOT_DONE) {
_gnix_nic_tx_free(nic, txd);
@@ -312,29 +504,129 @@ static void __gnix_msg_copy_unaligned_get_data(struct gnix_fab_req *req)
int head_off, head_len, tail_len;
void *addr;
- head_off = req->msg.send_addr & GNI_READ_ALIGN_MASK;
+ head_off = req->msg.send_info[0].send_addr & GNI_READ_ALIGN_MASK;
head_len = head_off ? GNI_READ_ALIGN - head_off : 0;
- tail_len = (req->msg.send_addr + req->msg.send_len) &
- GNI_READ_ALIGN_MASK;
+ tail_len = (req->msg.send_info[0].send_addr + req->msg.send_info[0].send_len) &
+ GNI_READ_ALIGN_MASK;
if (head_off) {
- addr = (void *)&req->msg.rndzv_head + head_off;
+ addr = (uint8_t *)&req->msg.send_info[0].head + head_off;
GNIX_INFO(FI_LOG_EP_DATA,
- "writing %d bytes to head (%p, 0x%x)\n",
- head_len, req->msg.recv_addr, *(uint32_t *)addr);
- memcpy((void *)req->msg.recv_addr, addr, head_len);
+ "writing %d bytes to head (%p, %hxx)\n",
+ head_len, req->msg.recv_info[0].recv_addr,
+ *(uint32_t *)addr);
+ memcpy((void *)req->msg.recv_info[0].recv_addr, addr, head_len);
}
if (tail_len) {
- addr = (void *)(req->msg.recv_addr +
- req->msg.send_len -
+ addr = (void *)(req->msg.recv_info[0].recv_addr +
+ req->msg.send_info[0].send_len -
tail_len);
GNIX_INFO(FI_LOG_EP_DATA,
- "writing %d bytes to tail (%p, 0x%x)\n",
- tail_len, addr, req->msg.rndzv_tail);
- memcpy((void *)addr, &req->msg.rndzv_tail, tail_len);
+ "writing %d bytes to tail (%p, %hxx)\n",
+ tail_len, addr, req->msg.send_info[0].tail);
+ memcpy((void *)addr, &req->msg.send_info[0].tail, tail_len);
+ }
+}
+
+static inline void __gnix_msg_iov_cpy_unaligned_head_tail_data(struct gnix_fab_req *req)
+{
+ int i, head_off, head_len, tail_len;
+ void *addr, *recv_addr;
+ size_t cur_len = 0;
+
+#if ENABLE_DEBUG
+ for (i = 0; i < req->msg.send_iov_cnt; i++) {
+ GNIX_DEBUG(FI_LOG_EP_DATA, "req->msg.send_info[%d].head = 0x%x\n"
+ "req->msg.send_info[%d].tail = 0x%x\n", i,
+ req->msg.send_info[i].head, i, req->msg.send_info[i].tail);
+ }
+#endif
+
+ /* Copy out the original head/tail data sent across in the control message */
+ for (i = 0; i < req->msg.send_iov_cnt; i++) {
+ head_off = req->msg.send_info[i].send_addr & GNI_READ_ALIGN_MASK;
+ head_len = head_off ? GNI_READ_ALIGN - head_off : 0;
+
+ if (head_off) {
+ recv_addr = __gnix_msg_iov_unaligned_recv_addr(req, cur_len);
+
+ if (recv_addr) {
+ addr = (uint8_t *)&req->msg.send_info[i].head + head_off;
+
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "writing %d bytes to head (%p, 0x%x) for i = %d\n",
+ head_len, recv_addr,
+ *(uint32_t *)addr, i);
+ memcpy(recv_addr, addr, head_len);
+ }
+ }
+
+ tail_len = (req->msg.send_info[i].send_addr + req->msg.send_info[i].send_len) &
+ GNI_READ_ALIGN_MASK;
+
+ if (tail_len) {
+ recv_addr = __gnix_msg_iov_unaligned_recv_addr(req,
+ cur_len +
+ req->msg.send_info[i].send_len - tail_len);
+
+ if (recv_addr) {
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "writing %d bytes to tail (%p, 0x%x)\n",
+ tail_len, recv_addr, req->msg.send_info[i].tail);
+ memcpy((void *)recv_addr, &req->msg.send_info[i].tail, tail_len);
+ }
+ }
+
+ cur_len += req->msg.send_info[i].send_len;
+ }
+
+#if ENABLE_DEBUG
+ for (i = 0; i < req->msg.recv_iov_cnt; i++) {
+ GNIX_DEBUG(FI_LOG_EP_DATA, "req->msg.recv_info[%d].tail_len = %d\n"
+ "req->msg.recv_info[%d].head_len = %d\n",
+ i, req->msg.recv_info[i].tail_len,
+ i, req->msg.recv_info[i].head_len);
+ }
+#endif
+
+ /* Copy out the "middle" head and tail data found when building the iov request */
+ for (i = 0; i < req->msg.recv_iov_cnt; i++) {
+ if (req->msg.recv_info[i].tail_len) {
+ addr = (void *) ((uint8_t *) req->msg.htd_buf + (GNI_READ_ALIGN * i));
+
+ recv_addr = (void *) (req->msg.recv_info[i].recv_addr +
+ req->msg.recv_info[i].recv_len -
+ req->msg.recv_info[i].tail_len);
+
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "writing %d bytes to mid-tail (%p, 0x%x)\n",
+ req->msg.recv_info[i].tail_len,
+ recv_addr, *(uint32_t *)addr);
+
+ memcpy(recv_addr, addr, req->msg.recv_info[i].tail_len);
+ }
+
+ if (req->msg.recv_info[i].head_len) {
+ /* Since we move the remote addr backwards to a four
+ * byte address and read four bytes, ensure that
+ * what we read from the htd_buf is just the actual head
+ * data we are interested in.
+ */
+ addr = (void *) ((uint8_t *) req->msg.htd_buf +
+ (GNI_READ_ALIGN * (i + GNIX_MAX_MSG_IOV_LIMIT)) +
+ GNI_READ_ALIGN - req->msg.recv_info[i].head_len);
+ recv_addr = (void *) req->msg.recv_info[i].recv_addr;
+
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "writing %d bytes to mid-head (%p, 0x%x)\n",
+ req->msg.recv_info[i].head_len,
+ recv_addr, *(uint32_t *)addr);
+
+ memcpy(recv_addr, addr, req->msg.recv_info[i].head_len);
+ }
}
}
@@ -351,14 +643,13 @@ static int __gnix_rndzv_req_complete(void *arg, gni_return_t tx_status)
* unaligned bytes. Bytes are copied from the request to the
* user buffer once both TXDs arrive. */
if (txd->gni_desc.type == GNI_POST_FMA_GET)
- req->msg.rndzv_tail = *(uint32_t *)txd->int_buf;
+ req->msg.send_info[0].tail = *(uint32_t *)txd->int_buf;
/* Remember any failure. Retransmit both TXDs once both are
* complete. */
req->msg.status |= tx_status;
- atomic_dec(&req->msg.outstanding_txds);
- if (atomic_get(&req->msg.outstanding_txds)) {
+ if (atomic_dec(&req->msg.outstanding_txds) == 1) {
_gnix_nic_tx_free(req->gnix_ep->nic, txd);
GNIX_INFO(FI_LOG_EP_DATA,
"Received first RDMA chain TXD, req: %p\n",
@@ -372,18 +663,30 @@ static int __gnix_rndzv_req_complete(void *arg, gni_return_t tx_status)
_gnix_nic_tx_free(req->gnix_ep->nic, txd);
if (tx_status != GNI_RC_SUCCESS) {
- req->tx_failures++;
- if (req->tx_failures <
- req->gnix_ep->domain->params.max_retransmits) {
-
+ if (GNIX_EP_RDM(req->gnix_ep->type) &&
+ _gnix_req_replayable(req)) {
+ req->tx_failures++;
GNIX_INFO(FI_LOG_EP_DATA,
"Requeueing failed request: %p\n", req);
return _gnix_vc_queue_work_req(req);
}
- /* TODO should this be fatal? A request will sit waiting at the
- * peer. */
- return __gnix_msg_recv_err(req->gnix_ep, req);
+ if (!GNIX_EP_DGM(req->gnix_ep->type)) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "Dropping failed request: %p\n", req);
+ ret = __gnix_msg_send_err(req->gnix_ep,
+ req);
+ if (ret != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "__gnix_msg_send_err() failed: %s\n",
+ fi_strerror(-ret));
+ __gnix_msg_send_fr_complete(req, txd);
+ return ret;
+ }
+
+ req->msg.status = tx_status;
+ req->work_fn = __gnix_rndzv_req_send_fin;
+ return _gnix_vc_queue_work_req(req);
}
__gnix_msg_copy_unaligned_get_data(req);
@@ -392,8 +695,8 @@ static int __gnix_rndzv_req_complete(void *arg, gni_return_t tx_status)
if (req->msg.recv_flags & FI_LOCAL_MR) {
GNIX_INFO(FI_LOG_EP_DATA, "freeing auto-reg MR: %p\n",
- req->msg.recv_md);
- fi_close(&req->msg.recv_md->mr_fid.fid);
+ req->msg.recv_md[0]);
+ fi_close(&req->msg.recv_md[0]->mr_fid.fid);
}
req->work_fn = __gnix_rndzv_req_send_fin;
@@ -402,6 +705,174 @@ static int __gnix_rndzv_req_complete(void *arg, gni_return_t tx_status)
return ret;
}
+/*
+ * As the completer fn is called in the nic's progress loop, the remote
+ * side's fabric request will keep track of the state of this rndzv iov
+ * transaction, once the outstanding_txds reaches zero for this fabric
+ * request, we are either ready to generate CQEs and send the fin msg
+ * back to the sender or rebuild and retransmit the iov txds.
+ */
+static int __gnix_rndzv_iov_req_build(void *arg);
+static int __gnix_rndzv_iov_req_complete(void *arg, gni_return_t tx_status)
+{
+ struct gnix_tx_descriptor *txd = (struct gnix_tx_descriptor *)arg;
+ struct gnix_fab_req *req = txd->req;
+ int i, ret = FI_SUCCESS;
+
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "req->msg.outstanding_txds = %d\n",
+ atomic_get(&req->msg.outstanding_txds));
+
+ req->msg.status |= tx_status;
+
+ __gnix_msg_free_rma_txd(req, txd);
+
+ if (atomic_dec(&req->msg.outstanding_txds) == 0) {
+
+ /* All the txd's are complete, we just need our unaligned heads
+ * and tails now
+ */
+ __gnix_msg_iov_cpy_unaligned_head_tail_data(req);
+ GNIX_DEBUG(FI_LOG_EP_DATA, "req->msg.recv_flags == FI_LOCAL_MR "
+ "is %s, req->msg.recv_iov_cnt = %lu\n",
+ req->msg.recv_flags & FI_LOCAL_MR ? "true" : "false",
+ req->msg.recv_iov_cnt);
+
+ if (req->msg.status != FI_SUCCESS) {
+
+ if (GNIX_EP_RDM(req->gnix_ep->type) &&
+ _gnix_req_replayable(req)) {
+ req->tx_failures++;
+ /* Build and re-tx the entire iov request if the
+ * ep type is "reliable datagram" */
+ req->work_fn = __gnix_rndzv_iov_req_build;
+ return _gnix_vc_queue_work_req(req);
+ }
+
+ if (!GNIX_EP_DGM(req->gnix_ep->type)) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "Dropping failed request: %p\n", req);
+ ret = __gnix_msg_send_err(req->gnix_ep,
+ req);
+ if (ret != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "__gnix_msg_send_err() failed: %s\n",
+ fi_strerror(-ret));
+ __gnix_msg_send_fr_complete(req, txd);
+ return ret;
+ }
+ } else {
+ if (req->msg.recv_flags & FI_LOCAL_MR) {
+ for (i = 0; i < req->msg.recv_iov_cnt; i++) {
+ GNIX_INFO(FI_LOG_EP_DATA, "freeing auto"
+ "-reg MR: %p\n",
+ req->msg.recv_md[i]);
+ fi_close(&req->msg.recv_md[i]->mr_fid.fid);
+ }
+ }
+ }
+
+ /* Generate remote CQE and send fin msg back to sender */
+ req->work_fn = __gnix_rndzv_req_send_fin;
+ return _gnix_vc_queue_work_req(req);
+ }
+
+ /*
+ * Successful tx, continue until the txd counter reaches zero
+ * or we can't recover from the error.
+ */
+ return ret;
+}
+
+static int __gnix_rndzv_req_xpmem(struct gnix_fab_req *req)
+{
+ int ret = FI_SUCCESS, send_idx = 0, recv_idx = 0;
+ size_t cpy_len, recv_len;
+ uint64_t recv_ptr = 0UL;
+ struct gnix_xpmem_access_handle *access_hndl;
+
+ recv_len = req->msg.recv_info[0].recv_len;
+ recv_ptr = req->msg.recv_info[0].recv_addr;
+
+ /* Copy data from/to (>=1) iovec entries */
+ while (send_idx < req->msg.send_iov_cnt) {
+ cpy_len = MIN(recv_len, req->msg.send_info[send_idx].send_len);
+
+ /*
+ * look up mapping from other EP
+ */
+ ret = _gnix_xpmem_access_hndl_get(req->gnix_ep->xpmem_hndl,
+ req->vc->peer_apid,
+ req->msg.send_info[send_idx].send_addr,
+ cpy_len,
+ &access_hndl);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA, "_gnix_xpmem_access_hndl_get failed %s\n",
+ fi_strerror(-ret));
+ req->msg.status = GNI_RC_TRANSACTION_ERROR;
+ return ret;
+ }
+
+ /*
+ * pull the data from the other process' address space
+ */
+ ret = _gnix_xpmem_copy(access_hndl,
+ (void *)recv_ptr,
+ (void *)req->msg.send_info[send_idx].send_addr,
+ cpy_len);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA, "_gnix_xpmem_vaddr_copy failed %s\n",
+ fi_strerror(-ret));
+ req->msg.status = GNI_RC_TRANSACTION_ERROR;
+ _gnix_xpmem_access_hndl_put(access_hndl);
+ return ret;
+ }
+
+ ret = _gnix_xpmem_access_hndl_put(access_hndl);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA, "_gnix_xpmem_access_hndl_put failed %s\n",
+ fi_strerror(-ret));
+ }
+
+ /* Update the local and remote addresses */
+ recv_len -= cpy_len;
+
+ /* We have exhausted the current recv (and possibly send)
+ * buffer */
+ if (recv_len == 0) {
+ recv_idx++;
+
+ /* We cannot receive any more. */
+ if (recv_idx == req->msg.recv_iov_cnt)
+ break;
+
+ recv_ptr = req->msg.recv_info[recv_idx].recv_addr;
+ recv_len = req->msg.recv_info[recv_idx].recv_len;
+
+ /* Also exhausted send buffer */
+ if (cpy_len == req->msg.send_info[send_idx].send_len) {
+ send_idx++;
+ } else {
+ req->msg.send_info[send_idx].send_addr += cpy_len;
+ req->msg.send_info[send_idx].send_len -= cpy_len;
+ }
+ } else { /* Just exhausted current send buffer. */
+ send_idx++;
+ recv_ptr += cpy_len;
+ }
+ GNIX_DEBUG(FI_LOG_EP_DATA, "send_idx = %d, recv_idx = %d\n", send_idx, recv_idx);
+ }
+
+ /*
+ * set the req send fin and reschedule req
+ */
+
+ req->msg.status = GNI_RC_SUCCESS; /* hmph */
+ req->work_fn = __gnix_rndzv_req_send_fin;
+ return _gnix_vc_queue_work_req(req);
+}
+
static int __gnix_rndzv_req(void *arg)
{
struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
@@ -416,9 +887,18 @@ static int __gnix_rndzv_req(void *arg)
int head_off, head_len, tail_len;
void *tail_data = NULL;
- if (!req->msg.recv_md) {
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+ /*
+ * TODO: xpmem intercept here
+ */
+
+ if (req->vc->modes & GNIX_VC_MODE_XPMEM)
+ return __gnix_rndzv_req_xpmem(req);
+
+ if (!req->msg.recv_md[0]) {
rc = gnix_mr_reg(&ep->domain->domain_fid.fid,
- (void *)req->msg.recv_addr, req->msg.recv_len,
+ (void *)req->msg.recv_info[0].recv_addr,
+ req->msg.recv_info[0].recv_len,
FI_READ | FI_WRITE, 0, 0, 0, &auto_mr, NULL);
if (rc != FI_SUCCESS) {
GNIX_INFO(FI_LOG_EP_DATA,
@@ -427,9 +907,10 @@ static int __gnix_rndzv_req(void *arg)
return -FI_EAGAIN;
}
req->msg.recv_flags |= FI_LOCAL_MR;
- req->msg.recv_md = container_of(auto_mr,
- struct gnix_fid_mem_desc,
- mr_fid);
+ req->msg.recv_md[0] = container_of(auto_mr,
+ struct gnix_fid_mem_desc,
+ mr_fid);
+ req->msg.recv_info[0].mem_hndl = req->msg.recv_md[0]->mem_hndl;
GNIX_INFO(FI_LOG_EP_DATA, "auto-reg MR: %p\n", auto_mr);
}
@@ -443,25 +924,26 @@ static int __gnix_rndzv_req(void *arg)
txd->completer_fn = __gnix_rndzv_req_complete;
txd->req = req;
+
use_tx_cq_blk = (ep->domain->data_progress == FI_PROGRESS_AUTO) ? 1 : 0;
txd->gni_desc.type = GNI_POST_RDMA_GET;
txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT;
txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE;
- txd->gni_desc.local_mem_hndl = req->msg.recv_md->mem_hndl;
+ txd->gni_desc.local_mem_hndl = req->msg.recv_info[0].mem_hndl;
txd->gni_desc.remote_mem_hndl = req->msg.rma_mdh;
txd->gni_desc.rdma_mode = 0;
txd->gni_desc.src_cq_hndl = (use_tx_cq_blk) ?
nic->tx_cq_blk : nic->tx_cq;
- head_off = req->msg.send_addr & GNI_READ_ALIGN_MASK;
+ head_off = req->msg.send_info[0].send_addr & GNI_READ_ALIGN_MASK;
head_len = head_off ? GNI_READ_ALIGN - head_off : 0;
- tail_len = (req->msg.send_addr + req->msg.send_len) &
+ tail_len = (req->msg.send_info[0].send_addr + req->msg.send_info[0].send_len) &
GNI_READ_ALIGN_MASK;
- txd->gni_desc.local_addr = (uint64_t)req->msg.recv_addr + head_len;
- txd->gni_desc.remote_addr = (uint64_t)req->msg.send_addr + head_len;
- txd->gni_desc.length = req->msg.send_len - head_len - tail_len;
+ txd->gni_desc.local_addr = (uint64_t)req->msg.recv_info[0].recv_addr + head_len;
+ txd->gni_desc.remote_addr = (uint64_t)req->msg.send_info[0].send_addr + head_len;
+ txd->gni_desc.length = req->msg.send_info[0].send_len - head_len - tail_len;
if (req->msg.recv_flags & GNIX_MSG_GET_TAIL) {
/* The user ended up with a send matching a receive with a
@@ -481,7 +963,8 @@ static int __gnix_rndzv_req(void *arg)
tail_txd->completer_fn = __gnix_rndzv_req_complete;
tail_txd->req = req;
- tail_data = (void *)((req->msg.send_addr + req->msg.send_len) &
+ tail_data = (void *)((req->msg.send_info[0].send_addr +
+ req->msg.send_info[0].send_len) &
~GNI_READ_ALIGN_MASK);
tail_txd->gni_desc.type = GNI_POST_FMA_GET;
@@ -498,7 +981,7 @@ static int __gnix_rndzv_req(void *arg)
GNIX_INFO(FI_LOG_EP_DATA, "Using two GETs\n");
}
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
if (inject_err) {
_gnix_nic_txd_err_inject(nic, txd);
@@ -508,12 +991,14 @@ static int __gnix_rndzv_req(void *arg)
}
if (status != GNI_RC_SUCCESS) {
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
if (tail_txd)
_gnix_nic_tx_free(nic, tail_txd);
_gnix_nic_tx_free(nic, txd);
GNIX_INFO(FI_LOG_EP_DATA, "GNI_PostRdma failed: %s\n",
gni_err_str[status]);
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "\n");
return gnixu_to_fi_errno(status);
}
@@ -527,7 +1012,7 @@ static int __gnix_rndzv_req(void *arg)
}
if (status != GNI_RC_SUCCESS) {
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
_gnix_nic_tx_free(nic, tail_txd);
/* Wait for the first TX to complete, then retransmit
@@ -542,86 +1027,605 @@ static int __gnix_rndzv_req(void *arg)
/* Wait for both TXs to complete, then process the request. */
atomic_set(&req->msg.outstanding_txds, 2);
- req->msg.status = 0;
+ req->msg.status = GNI_RC_SUCCESS;
}
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
GNIX_INFO(FI_LOG_EP_DATA, "Initiated RNDZV GET, req: %p\n", req);
return gnixu_to_fi_errno(status);
}
-/*******************************************************************************
- * GNI SMSG callbacks invoked upon completion of an SMSG message at the sender.
- ******************************************************************************/
-
-static int __comp_eager_msg_w_data(void *data, gni_return_t tx_status)
+/* Attempt to post the iov txds */
+static int __gnix_rndzv_iov_req_post(void *arg)
{
- struct gnix_tx_descriptor *tdesc = (struct gnix_tx_descriptor *)data;
- struct gnix_fab_req *req = tdesc->req;
- int ret = FI_SUCCESS;
+ struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
+ struct gnix_tx_descriptor *txd;
+ gni_return_t status;
+ struct gnix_nic *nic = req->gnix_ep->nic;
+ int i, iov_txd_cnt = atomic_get(&req->msg.outstanding_txds);
- if (tx_status != GNI_RC_SUCCESS) {
- GNIX_INFO(FI_LOG_EP_DATA, "Failed transaction: %p\n", req);
- ret = __gnix_msg_send_err(req->gnix_ep, req);
- if (ret != FI_SUCCESS)
- GNIX_WARN(FI_LOG_EP_DATA,
- "__gnix_msg_send_err() failed: %d\n",
- ret);
- } else {
- /* Successful delivery. Generate completions. */
- ret = __gnix_msg_send_completion(req->gnix_ep, req);
- if (ret != FI_SUCCESS)
- GNIX_WARN(FI_LOG_EP_DATA,
- "__gnix_msg_send_completion() failed: %d\n",
- ret);
+ assert(nic != NULL);
+
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
+ if (unlikely(iov_txd_cnt == 0))
+ return -FI_EAGAIN;
+
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
+
+ for (i = 0, txd = req->iov_txds[0];
+ i < iov_txd_cnt;
+ txd = req->iov_txds[++i]) {
+ if (txd->gni_desc.type & GNI_POST_RDMA_GET) {
+ status = GNI_PostRdma(req->vc->gni_ep,
+ &txd->gni_desc);
+ } else {
+ status = GNI_CtPostFma(req->vc->gni_ep,
+ &txd->gni_desc);
+ }
+
+ if (status != GNI_RC_SUCCESS) {
+ COND_RELEASE(nic->requires_lock, &nic->lock);
+
+ GNIX_WARN(FI_LOG_EP_DATA, "%s failed on txd %p: with "
+ "%s\n",
+ txd->gni_desc.type & GNI_POST_RDMA_GET ?
+ "GNI_PostRdma" : "GNI_CtPostFma", txd,
+ gni_err_str[status]);
+
+ return gnixu_to_fi_errno(status);
+ }
}
- __gnix_msg_send_fr_complete(req, tdesc);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
return FI_SUCCESS;
}
-static int __comp_eager_msg_w_data_ack(void *data, gni_return_t tx_status)
+/*
+ * Dequeued a iov work req on the remote endpoint, initiate the pull of data.
+ */
+static int __gnix_rndzv_iov_req_build(void *arg)
{
- return -FI_ENOSYS;
-}
+ int ret = FI_SUCCESS, send_idx, recv_idx, use_tx_cq_blk;
+ struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
+ struct gnix_fid_ep *ep = req->gnix_ep;
+ struct gnix_nic *nic = ep->nic;
+ gni_ep_handle_t gni_ep = req->vc->gni_ep;
+ struct gnix_tx_descriptor *txd = NULL, *ct_txd = NULL;
+ size_t recv_len, send_len, get_len, send_cnt, recv_cnt, txd_cnt, ht_len;
+ uint64_t recv_ptr = 0UL, send_ptr = 0UL;
+ /* TODO: Should this be the sender's rndzv thresh instead? */
+ size_t rndzv_thresh = ep->domain->params.msg_rendezvous_thresh;
+ gni_ct_get_post_descriptor_t *cur_ct = NULL;
+ void **next_ct = NULL;
+ int head_off, head_len, tail_len;
-static int __comp_eager_msg_data_at_src(void *data, gni_return_t tx_status)
-{
- return -FI_ENOSYS;
-}
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+ /*
+ * TODO: xpmem intercept here
+ */
-static int __comp_eager_msg_data_at_src_ack(void *data, gni_return_t tx_status)
-{
- return -FI_ENOSYS;
-}
+ if (req->vc->modes & GNIX_VC_MODE_XPMEM)
+ return __gnix_rndzv_req_xpmem(req);
-static int __comp_rndzv_msg_rts(void *data, gni_return_t tx_status)
-{
- return -FI_ENOSYS;
-}
+ txd_cnt = 0;
+ send_cnt = req->msg.send_iov_cnt;
-static int __comp_rndzv_msg_rtr(void *data, gni_return_t tx_status)
-{
- return -FI_ENOSYS;
-}
+ recv_ptr = req->msg.recv_info[0].recv_addr;
+ recv_len = req->msg.recv_info[0].recv_len;
+ recv_cnt = req->msg.recv_iov_cnt;
-static int __comp_rndzv_msg_cookie(void *data, gni_return_t tx_status)
-{
- return -FI_ENOSYS;
-}
+ send_ptr = req->msg.send_info[0].send_addr;
+ send_len = req->msg.send_info[0].send_len;
-static int __comp_rndzv_msg_send_done(void *data, gni_return_t tx_status)
-{
- return -FI_ENOSYS;
-}
+ use_tx_cq_blk = (ep->domain->data_progress == FI_PROGRESS_AUTO);
-static int __comp_rndzv_msg_recv_done(void *data, gni_return_t tx_status)
-{
- return -FI_ENOSYS;
+ GNIX_DEBUG(FI_LOG_EP_DATA, "send_cnt = %lu, recv_cnt = %lu\n",
+ send_cnt, recv_cnt);
+
+ /* Ensure the user's recv buffer is registered for recv/recvv */
+ if (!req->msg.recv_md[0]) {
+ struct fid_mr *auto_mr;
+
+ for (recv_idx = 0; recv_idx < recv_cnt; recv_idx++) {
+ auto_mr = NULL;
+ ret = gnix_mr_reg(&ep->domain->domain_fid.fid,
+ (void *)
+ req->msg.recv_info[recv_idx].recv_addr,
+ req->msg.recv_info[recv_idx].recv_len,
+ FI_READ | FI_WRITE, 0, 0, 0,
+ &auto_mr, NULL);
+
+ if (ret != FI_SUCCESS) {
+ GNIX_DEBUG(FI_LOG_EP_DATA,
+ "Failed to auto-register"
+ " local buffer: %s\n",
+ fi_strerror(-ret));
+
+ for (recv_idx--; recv_idx >= 0; recv_idx--) {
+ fi_close(&req->msg.recv_md[recv_idx]->mr_fid.fid);
+ }
+
+ return ret;
+ }
+
+ req->msg.recv_md[recv_idx] = container_of(
+ (void *) auto_mr,
+ struct gnix_fid_mem_desc,
+ mr_fid);
+
+ req->msg.recv_info[recv_idx].mem_hndl =
+ req->msg.recv_md[recv_idx]->mem_hndl;
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "auto-reg MR: %p\n",
+ req->msg.recv_md[recv_idx]);
+
+ }
+ req->msg.recv_flags |= FI_LOCAL_MR;
+ }
+
+ recv_idx = send_idx = 0;
+
+ /* Iterate through the buffers and build the Fma and Rdma requests! */
+ while (send_idx < send_cnt) {
+ get_len = MIN(recv_len, send_len);
+
+ /* Begin alignment checks
+ *
+ * Each "mid-head" and "mid-tail" (resulting from the send pointer
+ * and length being adjusted to match the smaller posted recv buf in
+ * this loop) will be added to one or more chained transactions
+ * below.
+ *
+ * The original heads and tails (sent across in the control
+ * message) must be accounted for below in order to GET the
+ * correct, now four byte aligned, "body" section of the
+ * message.
+ */
+ if (send_ptr & GNI_READ_ALIGN_MASK ||
+ (send_ptr + get_len) & GNI_READ_ALIGN_MASK) {
+ if (req->msg.htd_buf_e == NULL) {
+ req->msg.htd_buf_e = _gnix_ep_get_htd_buf(ep);
+
+ /* There are no available htd bufs */
+ if (req->msg.htd_buf_e == NULL) {
+ atomic_set(&req->msg.outstanding_txds, 0);
+ req->work_fn = __gnix_rndzv_iov_req_post;
+ return _gnix_vc_queue_work_req(req);
+ }
+
+ req->msg.htd_buf = ((struct gnix_htd_buf *) req->msg.htd_buf_e)->buf;
+ req->msg.htd_mdh = _gnix_ep_get_htd_mdh(ep);
+ GNIX_DEBUG(FI_LOG_EP_DATA, "req->msg.htd_buf = %p\n", req->msg.htd_buf);
+ }
+
+ head_off = send_ptr & GNI_READ_ALIGN_MASK;
+ head_len = head_off ? GNI_READ_ALIGN - head_off : 0;
+ tail_len = (send_ptr + get_len) & GNI_READ_ALIGN_MASK;
+
+ ht_len = (size_t) (head_len + tail_len);
+
+ /* TODO: handle this. */
+ if (ht_len > recv_len) {
+ GNIX_FATAL(FI_LOG_EP_DATA, "The head tail data "
+ "length exceeds the matching receive"
+ "buffer length.\n");
+ }
+
+ /* found a mid-head? (see "Begin alignment" comment block) */
+ req->msg.recv_info[recv_idx].head_len = (send_ptr != req->msg.send_info[send_idx].send_addr) ?
+ head_len : 0;
+
+ /* found a mid-tail? (see "Begin alignment" comment block) */
+ req->msg.recv_info[recv_idx].tail_len = (send_len > recv_len) ?
+ tail_len : 0;
+
+ /* Update the local and remote addresses */
+ get_len -= ht_len;
+ send_len -= ht_len;
+ recv_len -= ht_len;
+
+ send_ptr += head_len;
+ recv_ptr += head_len;
+
+ /* Add to existing ct */
+ if (ct_txd) {
+ if (req->msg.recv_info[recv_idx].tail_len) {
+ cur_ct = *next_ct = malloc(sizeof(gni_ct_get_post_descriptor_t));
+
+ if (cur_ct == NULL) {
+ GNIX_DEBUG(FI_LOG_EP_DATA,
+ "Failed to allocate "
+ "gni FMA get chained "
+ "descriptor.");
+
+ /* +1 to ensure we free the
+ * current chained txd */
+ __gnix_msg_free_iov_txds(req, txd_cnt + 1);
+ return -FI_ENOSPC;
+ }
+
+ cur_ct->ep_hndl = gni_ep;
+ cur_ct->remote_mem_hndl = req->msg.send_info[send_idx].mem_hndl;
+ cur_ct->local_mem_hndl = req->msg.htd_mdh;
+ cur_ct->length = GNI_READ_ALIGN;
+ cur_ct->remote_addr = (send_ptr + get_len + tail_len) & ~GNI_READ_ALIGN_MASK;
+ cur_ct->local_addr = (uint64_t) (((uint8_t *) req->msg.htd_buf) + (GNI_READ_ALIGN * recv_idx));
+ next_ct = &cur_ct->next_descr;
+ }
+
+ if (req->msg.recv_info[recv_idx].head_len) {
+ cur_ct = *next_ct = malloc(sizeof(gni_ct_get_post_descriptor_t));
+
+ if (cur_ct == NULL) {
+ GNIX_DEBUG(FI_LOG_EP_DATA,
+ "Failed to allocate "
+ "gni FMA get chained "
+ "descriptor.");
+
+ /* +1 to ensure we free the
+ * current chained txd */
+ __gnix_msg_free_iov_txds(req, txd_cnt + 1);
+ return -FI_ENOSPC;
+ }
+
+ cur_ct->ep_hndl = gni_ep;
+ cur_ct->remote_mem_hndl = req->msg.send_info[send_idx].mem_hndl;
+ cur_ct->local_mem_hndl = req->msg.htd_mdh;
+ cur_ct->length = GNI_READ_ALIGN;
+ cur_ct->remote_addr = send_ptr - GNI_READ_ALIGN;
+ cur_ct->local_addr = (uint64_t) (((uint8_t *) req->msg.htd_buf) +
+ (GNI_READ_ALIGN * (recv_idx + GNIX_MAX_MSG_IOV_LIMIT)));
+ next_ct = &cur_ct->next_descr;
+ }
+ } else { /* Start a new ct */
+ if (req->msg.recv_info[recv_idx].tail_len) {
+ GNIX_DEBUG(FI_LOG_EP_DATA, "New FMA"
+ " CT\n");
+ ret = _gnix_nic_tx_alloc(nic, &ct_txd);
+
+ if (ret != FI_SUCCESS) {
+ /* We'll try again. */
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "_gnix_nic_tx_alloc()"
+ " returned %s\n",
+ fi_strerror(-ret));
+
+ __gnix_msg_free_iov_txds(req, txd_cnt);
+ return -FI_ENOSPC;
+ }
+
+ ct_txd->completer_fn = __gnix_rndzv_iov_req_complete;
+ ct_txd->req = req;
+
+ ct_txd->gni_desc.type = GNI_POST_FMA_GET;
+ ct_txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT;
+ ct_txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE;
+ ct_txd->gni_desc.rdma_mode = 0;
+ ct_txd->gni_desc.src_cq_hndl = (use_tx_cq_blk) ? nic->tx_cq_blk : nic->tx_cq;
+
+ ct_txd->gni_desc.remote_addr = (send_ptr + get_len + tail_len) & ~GNI_READ_ALIGN_MASK;
+ ct_txd->gni_desc.remote_mem_hndl = req->msg.send_info[send_idx].mem_hndl;
+
+ ct_txd->gni_desc.local_addr = (uint64_t) ((uint8_t *) req->msg.htd_buf + (GNI_READ_ALIGN * recv_idx));
+ ct_txd->gni_desc.local_mem_hndl = req->msg.htd_mdh;
+
+ ct_txd->gni_desc.length = GNI_READ_ALIGN;
+
+ next_ct = &ct_txd->gni_desc.next_descr;
+ }
+
+ if (req->msg.recv_info[recv_idx].head_len) {
+ if (req->msg.recv_info[recv_idx].tail_len) { /* Existing FMA CT */
+ cur_ct = *next_ct = malloc(sizeof(gni_ct_get_post_descriptor_t));
+ if (cur_ct == NULL) {
+ GNIX_DEBUG(FI_LOG_EP_DATA,
+ "Failed to allocate "
+ "gni FMA get chained "
+ "descriptor.");
+
+ /* +1 to ensure we free the
+ * current chained txd */
+ __gnix_msg_free_iov_txds(req, txd_cnt + 1);
+ return -FI_ENOSPC;
+ }
+
+ cur_ct->ep_hndl = gni_ep;
+ cur_ct->remote_mem_hndl = req->msg.send_info[send_idx].mem_hndl;
+ cur_ct->local_mem_hndl = req->msg.htd_mdh;
+ cur_ct->length = GNI_READ_ALIGN;
+ cur_ct->remote_addr = send_ptr - GNI_READ_ALIGN;
+ cur_ct->local_addr = (uint64_t) (((uint8_t *) req->msg.htd_buf) +
+ (GNI_READ_ALIGN * (recv_idx + GNIX_MAX_MSG_IOV_LIMIT)));
+ next_ct = &cur_ct->next_descr;
+ } else { /* New FMA ct */
+ GNIX_DEBUG(FI_LOG_EP_DATA, "New FMA"
+ " CT\n");
+ ret = _gnix_nic_tx_alloc(nic, &ct_txd);
+
+ if (ret != FI_SUCCESS) {
+ /* We'll try again. */
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "_gnix_nic_tx_alloc()"
+ " returned %s\n",
+ fi_strerror(-ret));
+
+ __gnix_msg_free_iov_txds(req, txd_cnt);
+ return -FI_ENOSPC;
+ }
+
+ ct_txd->completer_fn = __gnix_rndzv_iov_req_complete;
+ ct_txd->req = req;
+
+ ct_txd->gni_desc.type = GNI_POST_FMA_GET;
+ ct_txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT;
+ ct_txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE;
+ ct_txd->gni_desc.rdma_mode = 0;
+ ct_txd->gni_desc.src_cq_hndl = (use_tx_cq_blk) ? nic->tx_cq_blk : nic->tx_cq;
+
+ ct_txd->gni_desc.remote_addr = send_ptr - GNI_READ_ALIGN;
+ ct_txd->gni_desc.remote_mem_hndl = req->msg.send_info[send_idx].mem_hndl;
+
+ ct_txd->gni_desc.local_addr = (uint64_t) ((uint8_t *) req->msg.htd_buf +
+ (GNI_READ_ALIGN * (recv_idx + GNIX_MAX_MSG_IOV_LIMIT)));
+ ct_txd->gni_desc.local_mem_hndl = req->msg.htd_mdh;
+
+ ct_txd->gni_desc.length = GNI_READ_ALIGN;
+
+ next_ct = &ct_txd->gni_desc.next_descr;
+ }
+ }
+ }
+ } else { /* no head/tail found */
+ head_len = tail_len = 0;
+ req->msg.recv_info[recv_idx].head_len = req->msg.recv_info[recv_idx].tail_len = 0;
+ }
+ /* End alignment checks */
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "send_info[%d].send_len = %lu,"
+ " recv_len = %lu, get_len = %lu, head_len = %d,"
+ " tail_len = %d, req->msg.recv_info[%d].tail_len = %u\n"
+ "req->msg.recv_info[%d].head_len = %u, "
+ "recv_ptr(head) = %p, recv_ptr(tail) = %p\n", send_idx,
+ send_len, recv_len, get_len, head_len, tail_len, recv_idx,
+ req->msg.recv_info[recv_idx].tail_len, recv_idx,
+ req->msg.recv_info[recv_idx].head_len, (void *) (recv_ptr - head_len),
+ (void *) (recv_ptr + get_len));
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "txd = %p, send_ptr = %p, "
+ "send_ptr + get_len = %p, recv_ptr = %p\n",
+ txd, (void *) send_ptr, (void *)(send_ptr + get_len),
+ recv_ptr);
+
+ if (get_len >= rndzv_thresh) { /* Build the rdma txd */
+ ret = _gnix_nic_tx_alloc(nic, &txd);
+
+ if (ret != FI_SUCCESS) {
+ /* We'll try again. */
+ GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc()"
+ " returned %s\n",
+ fi_strerror(-ret));
+
+ __gnix_msg_free_iov_txds(req, txd_cnt);
+ return -FI_ENOSPC;
+ }
+
+ txd->completer_fn = __gnix_rndzv_iov_req_complete;
+ txd->req = req;
+
+ txd->gni_desc.type = GNI_POST_RDMA_GET;
+ txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT;
+ txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE;
+ txd->gni_desc.local_mem_hndl = req->msg.recv_info[recv_idx].mem_hndl;
+ txd->gni_desc.remote_mem_hndl = req->msg.send_info[send_idx].mem_hndl;
+ txd->gni_desc.rdma_mode = 0;
+ txd->gni_desc.src_cq_hndl = (use_tx_cq_blk) ? nic->tx_cq_blk : nic->tx_cq;
+
+ txd->gni_desc.local_addr = recv_ptr;
+ txd->gni_desc.remote_addr = send_ptr;
+ txd->gni_desc.length = get_len;
+
+ req->iov_txds[txd_cnt++] = txd;
+ txd = NULL;
+ } else if (get_len) { /* Build the Ct txd */
+ if (!ct_txd) {
+ GNIX_DEBUG(FI_LOG_EP_DATA, "New FMA"
+ " CT\n");
+ ret = _gnix_nic_tx_alloc(nic, &ct_txd);
+ if (ret != FI_SUCCESS) {
+ /* We'll try again. */
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "_gnix_nic_tx_alloc()"
+ " returned %s\n",
+ fi_strerror(-ret));
+
+ __gnix_msg_free_iov_txds(req, txd_cnt);
+ return -FI_ENOSPC;
+ }
+
+ ct_txd->completer_fn = __gnix_rndzv_iov_req_complete;
+ ct_txd->req = req;
+
+ ct_txd->gni_desc.type = GNI_POST_FMA_GET;
+ ct_txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT;
+ ct_txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE;
+ ct_txd->gni_desc.local_mem_hndl = req->msg.recv_info[recv_idx]. mem_hndl;
+
+ ct_txd->gni_desc.remote_mem_hndl = req->msg.send_info[send_idx].mem_hndl;
+ ct_txd->gni_desc.rdma_mode = 0;
+ ct_txd->gni_desc.src_cq_hndl = (use_tx_cq_blk) ? nic->tx_cq_blk : nic->tx_cq;
+
+ ct_txd->gni_desc.local_addr = recv_ptr;
+ ct_txd->gni_desc.remote_addr = send_ptr;
+ ct_txd->gni_desc.length = get_len;
+
+ next_ct = &ct_txd->gni_desc.next_descr;
+ } else {
+ cur_ct = *next_ct = malloc(sizeof(gni_ct_get_post_descriptor_t));
+
+ if (cur_ct == NULL) {
+ GNIX_DEBUG(FI_LOG_EP_DATA,
+ "Failed to allocate "
+ "gni FMA get chained "
+ "descriptor.");
+
+ /* +1 to ensure we free the
+ * current chained txd */
+ __gnix_msg_free_iov_txds(req, txd_cnt + 1);
+ return -FI_ENOSPC;
+ }
+
+ cur_ct->ep_hndl = gni_ep;
+ cur_ct->length = get_len;
+ cur_ct->remote_addr = send_ptr;
+ cur_ct->remote_mem_hndl = req->msg.send_info[send_idx].mem_hndl;
+ cur_ct->local_addr = (uint64_t) recv_ptr;
+ cur_ct->local_mem_hndl = req->msg.recv_info[recv_idx].mem_hndl;
+
+ next_ct = &cur_ct->next_descr;
+ }
+ }
+
+ /* Update the recv len */
+ recv_len -= get_len;
+
+ /* We have exhausted the current recv (and possibly send)
+ * buffer */
+ if (recv_len == 0) {
+ recv_idx++;
+
+ /* We cannot receive any more. */
+ if (recv_idx == recv_cnt)
+ break;
+
+ recv_ptr = req->msg.recv_info[recv_idx].recv_addr;
+ recv_len = req->msg.recv_info[recv_idx].recv_len;
+
+ /* Also exhausted send buffer */
+ if (get_len == send_len) {
+ send_idx++;
+ send_ptr = req->msg.send_info[send_idx].send_addr;
+ send_len = req->msg.send_info[send_idx].send_len;
+ } else {
+ send_ptr += (get_len + tail_len);
+ send_len -= get_len;
+ }
+ } else { /* Just exhausted current send buffer. */
+ send_idx++;
+ send_ptr = req->msg.send_info[send_idx].send_addr;
+ send_len = req->msg.send_info[send_idx].send_len;
+ recv_ptr += (get_len + tail_len);
+ }
+ GNIX_DEBUG(FI_LOG_EP_DATA, "send_idx = %d, recv_idx = %d\n",
+ send_idx, recv_idx);
+ }
+
+ /*
+ * If we ran out of buffer space on the sender's/receiver's side in the
+ * middle of building the ct, we must terminate and add that ct to the
+ * queue. Note that if the last txd built was a rdma txd then the txd
+ * will have been queued and txd will have a NULL value.
+ */
+ if (ct_txd) {
+ *next_ct = NULL;
+ req->iov_txds[txd_cnt++] = ct_txd;
+ }
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "txd_cnt = %lu\n", txd_cnt);
+ atomic_set(&req->msg.outstanding_txds, txd_cnt);
+
+ /* All the txd's are built, update the work_fn */
+ req->work_fn = __gnix_rndzv_iov_req_post;
+
+ /* Put this request back on work Q.
+ * TODO: Should we put it at the beginning of the work Q? */
+ ret = _gnix_vc_queue_work_req(req);
+ return ret;
+}
+
+/*******************************************************************************
+ * GNI SMSG callbacks invoked upon completion of an SMSG message at the sender.
+ ******************************************************************************/
+
+static int __comp_eager_msg_w_data(void *data, gni_return_t tx_status)
+{
+ struct gnix_tx_descriptor *tdesc = (struct gnix_tx_descriptor *)data;
+ struct gnix_fab_req *req = tdesc->req;
+ int ret = FI_SUCCESS;
+
+ if (tx_status != GNI_RC_SUCCESS) {
+ GNIX_INFO(FI_LOG_EP_DATA, "Failed transaction: %p\n", req);
+ ret = __gnix_msg_send_err(req->gnix_ep, req);
+ if (ret != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "__gnix_msg_send_err() failed: %s\n",
+ fi_strerror(-ret));
+ } else {
+ /* Successful delivery. Generate completions. */
+ ret = __gnix_msg_send_completion(req->gnix_ep, req);
+
+ /*
+ * For fi_sendv we must free the temporary buf used to flatten
+ * the user's iovec.
+ */
+ if (req->type == GNIX_FAB_RQ_SENDV ||
+ req->type == GNIX_FAB_RQ_TSENDV) {
+ free((void *) req->msg.send_info[0].send_addr);
+ }
+
+ if (ret != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "__gnix_msg_send_completion() failed: %d\n",
+ ret);
+ }
+
+ __gnix_msg_send_fr_complete(req, tdesc);
+
+ return FI_SUCCESS;
+}
+
+static int __comp_eager_msg_w_data_ack(void *data, gni_return_t tx_status)
+{
+ return -FI_ENOSYS;
+}
+
+static int __comp_eager_msg_data_at_src(void *data, gni_return_t tx_status)
+{
+ return -FI_ENOSYS;
+}
+
+static int __comp_eager_msg_data_at_src_ack(void *data, gni_return_t tx_status)
+{
+ return -FI_ENOSYS;
+}
+
+static int __comp_rndzv_msg_rts(void *data, gni_return_t tx_status)
+{
+ return -FI_ENOSYS;
+}
+
+static int __comp_rndzv_msg_rtr(void *data, gni_return_t tx_status)
+{
+ return -FI_ENOSYS;
+}
+
+static int __comp_rndzv_msg_cookie(void *data, gni_return_t tx_status)
+{
+ return -FI_ENOSYS;
+}
+
+static int __comp_rndzv_msg_send_done(void *data, gni_return_t tx_status)
+{
+ return -FI_ENOSYS;
+}
+
+static int __comp_rndzv_msg_recv_done(void *data, gni_return_t tx_status)
+{
+ return -FI_ENOSYS;
}
/* Completed request to start rendezvous send. */
@@ -636,8 +1640,8 @@ static int __comp_rndzv_start(void *data, gni_return_t tx_status)
ret = __gnix_msg_send_err(req->gnix_ep, req);
if (ret != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
- "__gnix_msg_send_err() failed: %d\n",
- ret);
+ "__gnix_msg_send_err() failed: %s\n",
+ fi_strerror(-ret));
__gnix_msg_send_fr_complete(req, txd);
} else {
/* Just free the TX descriptor for now. The request remains
@@ -660,7 +1664,7 @@ static int __comp_rndzv_fin(void *data, gni_return_t tx_status)
struct gnix_tx_descriptor *tdesc = (struct gnix_tx_descriptor *)data;
struct gnix_fab_req *req = tdesc->req;
- if (tx_status != GNI_RC_SUCCESS) {
+ if (tx_status != GNI_RC_SUCCESS || req->msg.status != GNI_RC_SUCCESS) {
/* TODO should this be fatal? A request will sit waiting at the
* peer. */
GNIX_WARN(FI_LOG_EP_DATA, "Failed transaction: %p\n", req);
@@ -698,6 +1702,7 @@ smsg_completer_fn_t gnix_ep_smsg_completers[] = {
[GNIX_SMSG_T_RNDZV_RDONE] = __comp_rndzv_msg_recv_done,
[GNIX_SMSG_T_RNDZV_START] = __comp_rndzv_start,
[GNIX_SMSG_T_RNDZV_FIN] = __comp_rndzv_fin,
+ [GNIX_SMSG_T_RNDZV_IOV_START] = __comp_rndzv_start
};
@@ -706,7 +1711,6 @@ smsg_completer_fn_t gnix_ep_smsg_completers[] = {
* These callback functions are invoked with the lock for the nic
* associated with the vc already held.
******************************************************************************/
-
/*
* Handle SMSG message with tag GNIX_SMSG_T_EGR_W_DATA
*/
@@ -725,6 +1729,8 @@ static int __smsg_eager_msg_w_data(void *data, void *msg)
fastlock_t *queue_lock;
int tagged;
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
ep = vc->ep;
assert(ep);
@@ -733,7 +1739,7 @@ static int __smsg_eager_msg_w_data(void *data, void *msg)
tagged = !!(hdr->flags & FI_TAGGED);
__gnix_msg_queues(ep, tagged, &queue_lock, &posted_queue, &unexp_queue);
- fastlock_acquire(queue_lock);
+ COND_ACQUIRE(ep->requires_lock, queue_lock);
/* Lookup a matching posted request. */
req = _gnix_match_tag(posted_queue, hdr->msg_tag, 0, FI_PEEK, NULL,
@@ -743,29 +1749,38 @@ static int __smsg_eager_msg_w_data(void *data, void *msg)
req->gnix_ep = ep;
req->vc = vc;
- req->msg.send_len = MIN(hdr->len, req->msg.recv_len);
+ req->msg.cum_send_len = hdr->len;
req->msg.send_flags = hdr->flags;
+ req->msg.send_iov_cnt = 1;
req->msg.tag = hdr->msg_tag;
req->msg.imm = hdr->imm;
- GNIX_INFO(FI_LOG_EP_DATA, "Matched req: %p (%p, %u)\n",
- req, req->msg.recv_addr, req->msg.send_len);
+ GNIX_DEBUG(FI_LOG_EP_DATA, "Matched req: %p (%p, %u)\n",
+ req, req->msg.recv_info[0].recv_addr,
+ req->msg.cum_send_len);
+
+ __gnix_msg_copy_data_to_recv_addr(req, data_ptr);
+
+ if ((req->msg.recv_flags & FI_MULTI_RECV) &&
+ ((req->msg.cum_recv_len - req->msg.cum_send_len) <
+ ep->min_multi_recv))
+ req->msg.recv_flags &= ~GNIX_MSG_MULTI_RECV_SUP;
- memcpy((void *)req->msg.recv_addr, data_ptr, req->msg.send_len);
__gnix_msg_recv_completion(ep, req);
/* Check if we're using FI_MULTI_RECV and there is space left
* in the receive buffer. */
if ((req->msg.recv_flags & FI_MULTI_RECV) &&
- ((req->msg.recv_len - req->msg.send_len) >=
- ep->min_multi_recv)) {
- GNIX_INFO(FI_LOG_EP_DATA, "Re-using req: %p\n", req);
+ ((req->msg.cum_recv_len - req->msg.cum_send_len) >=
+ ep->min_multi_recv)) {
+ GNIX_DEBUG(FI_LOG_EP_DATA, "Re-using req: %p\n", req);
/* Adjust receive buffer for the next match. */
- req->msg.recv_addr += req->msg.send_len;
- req->msg.recv_len -= req->msg.send_len;
+ req->msg.recv_info[0].recv_addr += req->msg.cum_send_len;
+ req->msg.recv_info[0].recv_len -= req->msg.cum_send_len;
+ req->msg.cum_recv_len = req->msg.recv_info[0].recv_len;
} else {
- GNIX_INFO(FI_LOG_EP_DATA, "Freeing req: %p\n", req);
+ GNIX_DEBUG(FI_LOG_EP_DATA, "Freeing req: %p\n", req);
/* Dequeue and free the request. */
_gnix_remove_tag(posted_queue, req);
@@ -775,13 +1790,14 @@ static int __smsg_eager_msg_w_data(void *data, void *msg)
/* Add new unexpected receive request. */
req = _gnix_fr_alloc(ep);
if (req == NULL) {
- fastlock_release(queue_lock);
+ COND_RELEASE(ep->requires_lock, queue_lock);
return -FI_ENOMEM;
}
- req->msg.send_addr = (uint64_t)malloc(hdr->len);
- if (unlikely(req->msg.send_addr == 0ULL)) {
- fastlock_release(queue_lock);
+ /* TODO: Buddy alloc */
+ req->msg.send_info[0].send_addr = (uint64_t)malloc(hdr->len);
+ if (unlikely(req->msg.send_info[0].send_addr == 0ULL)) {
+ COND_RELEASE(ep->requires_lock, queue_lock);
_gnix_fr_free(ep, req);
return -FI_ENOMEM;
}
@@ -791,21 +1807,23 @@ static int __smsg_eager_msg_w_data(void *data, void *msg)
req->gnix_ep = ep;
req->vc = vc;
- req->msg.send_len = hdr->len;
+ req->msg.cum_send_len = hdr->len;
+ req->msg.send_info[0].send_len = hdr->len;
+ req->msg.send_iov_cnt = 1;
req->msg.send_flags = hdr->flags;
req->msg.tag = hdr->msg_tag;
req->msg.imm = hdr->imm;
- memcpy((void *)req->msg.send_addr, data_ptr, hdr->len);
+ memcpy((void *)req->msg.send_info[0].send_addr, data_ptr, hdr->len);
req->addr = vc->peer_addr;
_gnix_insert_tag(unexp_queue, req->msg.tag, req, ~0);
- GNIX_INFO(FI_LOG_EP_DATA, "New req: %p (%u)\n",
- req, req->msg.send_len);
+ GNIX_DEBUG(FI_LOG_EP_DATA, "New req: %p (%u)\n",
+ req, req->msg.cum_send_len);
}
- fastlock_release(queue_lock);
+ COND_RELEASE(ep->requires_lock, queue_lock);
status = GNI_SmsgRelease(vc->gni_ep);
if (unlikely(status != GNI_RC_SUCCESS)) {
@@ -886,16 +1904,19 @@ static int __smsg_rndzv_start(void *data, void *msg)
fastlock_t *queue_lock;
int tagged;
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
ep = vc->ep;
assert(ep);
tagged = !!(hdr->flags & FI_TAGGED);
__gnix_msg_queues(ep, tagged, &queue_lock, &posted_queue, &unexp_queue);
- fastlock_acquire(queue_lock);
+ COND_ACQUIRE(ep->requires_lock, queue_lock);
req = _gnix_match_tag(posted_queue, hdr->msg_tag, 0, FI_PEEK, NULL,
&vc->peer_addr);
+
if (req) {
req->addr = vc->peer_addr;
req->gnix_ep = ep;
@@ -903,56 +1924,86 @@ static int __smsg_rndzv_start(void *data, void *msg)
req->tx_failures = 0;
/* Check if a second GET for unaligned data is needed. */
- if (hdr->len > req->msg.recv_len &&
- ((hdr->addr + req->msg.recv_len) & GNI_READ_ALIGN_MASK)) {
+ if (hdr->len > req->msg.recv_info[0].recv_len &&
+ ((hdr->addr + req->msg.recv_info[0].recv_len) & GNI_READ_ALIGN_MASK)) {
req->msg.recv_flags |= GNIX_MSG_GET_TAIL;
}
- req->msg.send_addr = hdr->addr;
- req->msg.send_len = MIN(hdr->len, req->msg.recv_len);
+ req->msg.send_info[0].send_addr = hdr->addr;
+ req->msg.send_info[0].send_len =
+ MIN(hdr->len, req->msg.cum_recv_len);
+ req->msg.send_info[0].mem_hndl = hdr->mdh;
+ req->msg.cum_send_len = req->msg.send_info[0].send_len;
+ req->msg.send_iov_cnt = 1;
req->msg.send_flags = hdr->flags;
req->msg.tag = hdr->msg_tag;
req->msg.imm = hdr->imm;
req->msg.rma_mdh = hdr->mdh;
req->msg.rma_id = hdr->req_addr;
- req->msg.rndzv_head = hdr->head;
- req->msg.rndzv_tail = hdr->tail;
+ req->msg.send_info[0].head = hdr->head;
+ req->msg.send_info[0].tail = hdr->tail;
+
+ if (req->type == GNIX_FAB_RQ_RECV) {
+ /* fi_send is rndzv with recv */
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "recv rndzv start - POSTED,"
+ " req = %p\n", req);
+ req->work_fn = __gnix_rndzv_req;
+ } else if (req->type == GNIX_FAB_RQ_RECVV ||
+ req->type == GNIX_FAB_RQ_TRECVV) {
+ /* fi_send is rndzv with recvv */
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "recvv rndzv start - POSTED,"
+ " req = %p\n", req);
+
+ req->work_fn = __gnix_rndzv_iov_req_build;
+ }
GNIX_INFO(FI_LOG_EP_DATA, "Matched req: %p (%p, %u)\n",
- req, req->msg.recv_addr, req->msg.send_len);
+ req, req->msg.recv_info[0].recv_addr,
+ req->msg.send_info[0].send_len);
/* Check if we're using FI_MULTI_RECV and there is space left
* in the receive buffer. */
- if ((req->msg.recv_flags & FI_MULTI_RECV) &&
- ((req->msg.recv_len - req->msg.send_len) >=
- ep->min_multi_recv)) {
+ if (req->type == GNIX_FAB_RQ_RECV &&
+ (req->msg.recv_flags & FI_MULTI_RECV) &&
+ ((req->msg.cum_recv_len - req->msg.cum_send_len) >=
+ ep->min_multi_recv)) {
/* Allocate new request for this transfer. */
dup_req = __gnix_msg_dup_req(req);
if (!dup_req) {
- fastlock_release(queue_lock);
+ COND_RELEASE(ep->requires_lock, queue_lock);
return -FI_ENOMEM;
}
/* Adjust receive buffer for the next match. */
- req->msg.recv_addr += req->msg.send_len;
- req->msg.recv_len -= req->msg.send_len;
+ req->msg.recv_info[0].recv_addr += req->msg.send_info[0].send_len;
+ req->msg.recv_info[0].recv_len -= req->msg.send_info[0].send_len;
+ req->msg.cum_recv_len = req->msg.recv_info[0].recv_len;
/* 'req' remains queued for more matches while the
* duplicated request is processed. */
req = dup_req;
} else {
+ /*
+ * doesn't hurt if its not FI_MULI_RECV
+ */
+ req->msg.recv_flags &= ~GNIX_MSG_MULTI_RECV_SUP;
/* Dequeue the request. */
_gnix_remove_tag(posted_queue, req);
}
/* Queue request to initiate pull of source data. */
- req->work_fn = __gnix_rndzv_req;
ret = _gnix_vc_queue_work_req(req);
+
+ GNIX_DEBUG(FI_LOG_EP_DATA,
+ "_gnix_vc_queue_work_req returned %s\n",
+ fi_strerror(-ret));
} else {
/* Add new unexpected receive request. */
req = _gnix_fr_alloc(ep);
if (req == NULL) {
- fastlock_release(queue_lock);
+ COND_RELEASE(ep->requires_lock, queue_lock);
return -FI_ENOMEM;
}
@@ -961,26 +2012,125 @@ static int __smsg_rndzv_start(void *data, void *msg)
req->gnix_ep = ep;
req->vc = vc;
- req->msg.send_addr = hdr->addr;
- req->msg.send_len = hdr->len;
+ req->msg.send_info[0].send_addr = hdr->addr;
+ req->msg.send_info[0].send_len = hdr->len;
+ req->msg.send_info[0].mem_hndl = hdr->mdh;
+ req->msg.send_iov_cnt = 1;
+ req->msg.cum_send_len = req->msg.send_info[0].send_len;
req->msg.send_flags = hdr->flags;
req->msg.tag = hdr->msg_tag;
req->msg.imm = hdr->imm;
req->msg.rma_mdh = hdr->mdh;
req->msg.rma_id = hdr->req_addr;
- req->msg.rndzv_head = hdr->head;
- req->msg.rndzv_tail = hdr->tail;
+ req->msg.send_info[0].head = hdr->head;
+ req->msg.send_info[0].tail = hdr->tail;
atomic_initialize(&req->msg.outstanding_txds, 0);
_gnix_insert_tag(unexp_queue, req->msg.tag, req, ~0);
GNIX_INFO(FI_LOG_EP_DATA, "New req: %p (%u)\n",
- req, req->msg.send_len);
+ req, req->msg.send_info[0].send_len);
+ }
+
+ COND_RELEASE(ep->requires_lock, queue_lock);
+
+ status = GNI_SmsgRelease(vc->gni_ep);
+ if (unlikely(status != GNI_RC_SUCCESS)) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "GNI_SmsgRelease returned %s\n",
+ gni_err_str[status]);
+ ret = gnixu_to_fi_errno(status);
+ }
+
+ return ret;
+}
+
+static int __smsg_rndzv_iov_start(void *data, void *msg)
+{
+ int ret = FI_SUCCESS;
+ gni_return_t status;
+ struct gnix_vc *vc = (struct gnix_vc *) data;
+ struct gnix_smsg_rndzv_iov_start_hdr *hdr = msg;
+ void *data_ptr = (void *) ((uint8_t *) msg + sizeof(*hdr));
+ struct gnix_fid_ep *ep;
+ struct gnix_fab_req *req = NULL;
+ struct gnix_tag_storage *unexp_queue;
+ struct gnix_tag_storage *posted_queue;
+ fastlock_t *queue_lock;
+ char is_req_posted = 0;
+
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
+#if ENABLE_DEBUG
+ int i;
+
+ for (i = 0; i < hdr->iov_cnt; i++) {
+ GNIX_DEBUG(FI_LOG_EP_DATA, "send_addr[%d] = %p, send_len[%d] = %lu\n", i,
+ ((struct send_info_t *)data_ptr)[i].send_addr, i,
+ ((struct send_info_t *)data_ptr)[i].send_len);
+ }
+#endif
+ ep = vc->ep;
+ assert(ep != NULL);
+
+ __gnix_msg_queues(ep, hdr->flags & FI_TAGGED, &queue_lock,
+ &posted_queue, &unexp_queue);
+
+ COND_ACQUIRE(ep->requires_lock, queue_lock);
+
+ req = _gnix_match_tag(posted_queue, hdr->msg_tag, 0, FI_PEEK, NULL,
+ &vc->peer_addr);
+
+ if (req) { /* Found a request in the posted queue */
+ is_req_posted = 1;
+ req->tx_failures = 0;
+ req->msg.cum_send_len = hdr->send_len;
+
+ GNIX_INFO(FI_LOG_EP_DATA, "Matched req: %p (%p, %u)\n",
+ req, req->msg.recv_info[0].recv_addr, hdr->send_len);
+ _gnix_remove_tag(posted_queue, req);
+ } else { /* Unexpected receive, enqueue it */
+ req = _gnix_fr_alloc(ep);
+ if (req == NULL) {
+ COND_RELEASE(ep->requires_lock, queue_lock);
+ return -FI_ENOMEM;
+ }
+
+ atomic_initialize(&req->msg.outstanding_txds, 0);
+
+ GNIX_INFO(FI_LOG_EP_DATA, "New req: %p (%u)\n",
+ req, hdr->send_len);
+
+ req->msg.cum_send_len = hdr->send_len;
}
- fastlock_release(queue_lock);
+ req->addr = vc->peer_addr;
+ req->gnix_ep = ep;
+ req->vc = vc;
+ req->work_fn = __gnix_rndzv_iov_req_build;
+
+ req->msg.send_flags = hdr->flags;
+ req->msg.imm = hdr->imm;
+ req->msg.tag = hdr->msg_tag;
+ req->msg.send_iov_cnt = hdr->iov_cnt;
+ req->msg.rma_id = hdr->req_addr;
+ memcpy(req->msg.send_info, data_ptr,
+ sizeof(struct send_info_t) * hdr->iov_cnt);
+ if (is_req_posted)
+ ret = _gnix_vc_queue_work_req(req);
+ else
+ _gnix_insert_tag(unexp_queue, req->msg.tag, req, ~0);
+
+
+ COND_RELEASE(ep->requires_lock, queue_lock);
+
+ /*
+ * Release the message buffer on the nic, need to copy the data
+ * section out before this.
+ */
status = GNI_SmsgRelease(vc->gni_ep);
+
if (unlikely(status != GNI_RC_SUCCESS)) {
GNIX_WARN(FI_LOG_EP_DATA,
"GNI_SmsgRelease returned %s\n",
@@ -993,11 +2143,20 @@ static int __smsg_rndzv_start(void *data, void *msg)
static int __gnix_rndzv_fin_cleanup(void *arg)
{
+ int i;
struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
- GNIX_INFO(FI_LOG_EP_DATA, "freeing auto-reg MR: %p\n",
- req->msg.send_md);
- fi_close(&req->msg.send_md->mr_fid.fid);
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
+ for (i = 0; i < req->msg.send_iov_cnt; i++) {
+ GNIX_INFO(FI_LOG_EP_DATA, "freeing auto-reg MR: %p\n",
+ req->msg.send_md[i]);
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "req->msg.send_md[%d] ="
+ " %p\n", i, req->msg.send_md[i]);
+ fi_close(&req->msg.send_md[i]->mr_fid.fid);
+ }
+
_gnix_fr_free(req->gnix_ep, req);
return FI_SUCCESS;
@@ -1015,6 +2174,8 @@ static int __smsg_rndzv_fin(void *data, void *msg)
struct gnix_fab_req *req;
struct gnix_fid_ep *ep;
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
req = (struct gnix_fab_req *)hdr->req_addr;
assert(req);
@@ -1023,7 +2184,16 @@ static int __smsg_rndzv_fin(void *data, void *msg)
ep = req->gnix_ep;
assert(ep != NULL);
- __gnix_msg_send_completion(ep, req);
+ if (hdr->status == GNI_RC_SUCCESS) {
+ __gnix_msg_send_completion(ep, req);
+ } else {
+ ret = __gnix_msg_send_err(ep, req);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "__gnix_msg_send_err() failed: %s\n",
+ fi_strerror(-ret));
+ }
+ }
atomic_dec(&req->vc->outstanding_tx_reqs);
@@ -1049,37 +2219,6 @@ static int __smsg_rndzv_fin(void *data, void *msg)
return ret;
}
-/* TODO: This is kind of out of place. */
-static int __smsg_rma_data(void *data, void *msg)
-{
- int ret = FI_SUCCESS;
- struct gnix_vc *vc = (struct gnix_vc *)data;
- struct gnix_smsg_rma_data_hdr *hdr =
- (struct gnix_smsg_rma_data_hdr *)msg;
- struct gnix_fid_ep *ep = vc->ep;
- gni_return_t status;
-
- if (ep->recv_cq) {
- ret = _gnix_cq_add_event(ep->recv_cq, NULL, hdr->flags, 0,
- 0, hdr->data, 0, FI_ADDR_NOTAVAIL);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_DATA,
- "_gnix_cq_add_event returned %d\n",
- ret);
- }
- }
-
- status = GNI_SmsgRelease(vc->gni_ep);
- if (unlikely(status != GNI_RC_SUCCESS)) {
- GNIX_WARN(FI_LOG_EP_DATA,
- "GNI_SmsgRelease returned %s\n",
- gni_err_str[status]);
- ret = gnixu_to_fi_errno(status);
- }
-
- return ret;
-}
-
smsg_callback_fn_t gnix_ep_smsg_callbacks[] = {
[GNIX_SMSG_T_EGR_W_DATA] = __smsg_eager_msg_w_data,
[GNIX_SMSG_T_EGR_W_DATA_ACK] = __smsg_eager_msg_w_data_ack,
@@ -1092,14 +2231,17 @@ smsg_callback_fn_t gnix_ep_smsg_callbacks[] = {
[GNIX_SMSG_T_RNDZV_RDONE] = __smsg_rndzv_msg_recv_done,
[GNIX_SMSG_T_RNDZV_START] = __smsg_rndzv_start,
[GNIX_SMSG_T_RNDZV_FIN] = __smsg_rndzv_fin,
- [GNIX_SMSG_T_RMA_DATA] = __smsg_rma_data
+ [GNIX_SMSG_T_RMA_DATA] = __smsg_rma_data, /* defined in gnix_rma.c */
+ [GNIX_SMSG_T_AMO_CNTR] = __smsg_amo_cntr, /* defined in gnix_amo.c */
+ [GNIX_SMSG_T_RNDZV_IOV_START] = __smsg_rndzv_iov_start
};
static int __gnix_peek_request(struct gnix_fab_req *req)
{
struct gnix_fid_cq *recv_cq = req->gnix_ep->recv_cq;
int rendezvous = !!(req->msg.send_flags & GNIX_MSG_RENDEZVOUS);
- int ret;
+ int ret, send_idx, recv_idx, copy_len;
+ uint64_t send_ptr, recv_ptr, send_len, recv_len;
/* All claim work is performed by the tag storage, so nothing special
* here. If no CQ, no data is to be returned. Just inform the user
@@ -1111,15 +2253,50 @@ static int __gnix_peek_request(struct gnix_fab_req *req)
/* Rendezvous messages on the unexpected queue won't have data.
* Additionally, if the CQ format doesn't support passing a buffer
* location and length, then data will not be copied. */
- if (!rendezvous && req->msg.recv_addr &&
+ if (!rendezvous && req->msg.recv_info[0].recv_addr &&
!INVALID_PEEK_FORMAT(recv_cq->attr.format)) {
- int copy_len = MIN(req->msg.send_len, req->msg.recv_len);
+ send_len = req->msg.send_info[0].send_len;
+ send_ptr = req->msg.send_info[0].send_addr;
+ recv_len = req->msg.recv_info[0].recv_len;
+ recv_ptr = req->msg.recv_info[0].recv_addr;
+ send_idx = recv_idx = 0;
+
+ while (1) {
+ copy_len = MIN(send_len, recv_len);
+ memcpy((void *)recv_ptr, (void *)send_ptr, copy_len);
+
+ /* Update lengths/addresses */
+ send_len -= copy_len;
+ recv_len -= copy_len;
+
+ if (send_len == 0) {
+ send_idx++;
+
+ if (send_idx == req->msg.send_iov_cnt)
+ break;
+
+ send_ptr = req->msg.send_info[send_idx].send_addr;
+ send_len = req->msg.send_info[send_idx].send_len;
+ } else {
+ send_ptr += copy_len;
+ }
+
+ if (recv_len == 0) {
+ recv_idx++;
+
+ if (recv_idx == req->msg.recv_iov_cnt)
+ break;
+
+ recv_ptr = req->msg.recv_info[recv_idx].recv_addr;
+ recv_len = req->msg.recv_info[recv_idx].recv_len;
+ } else {
+ recv_ptr += copy_len;
+ }
- memcpy((void *)req->msg.recv_addr, (void *)req->msg.send_addr,
- copy_len);
+ }
} else {
/* The CQE should not contain a valid buffer. */
- req->msg.recv_addr = 0;
+ req->msg.recv_info[0].recv_addr = 0;
}
ret = __gnix_msg_recv_completion(req->gnix_ep, req);
@@ -1137,8 +2314,8 @@ static int __gnix_discard_request(struct gnix_fab_req *req)
int rendezvous = !!(req->msg.send_flags & GNIX_MSG_RENDEZVOUS);
/* The CQE should not contain a valid buffer. */
- req->msg.recv_addr = 0;
- req->msg.send_len = 0;
+ req->msg.recv_info[0].recv_addr = 0;
+ req->msg.cum_send_len = req->msg.send_info[0].send_len = 0;
GNIX_INFO(FI_LOG_EP_DATA, "discarding req=%p\n", req);
if (rendezvous) {
@@ -1158,7 +2335,7 @@ static int __gnix_discard_request(struct gnix_fab_req *req)
ret);
/* Free unexpected eager receive buffer. */
- free((void *)req->msg.send_addr);
+ free((void *)req->msg.send_info[0].send_addr);
_gnix_fr_free(req->gnix_ep, req);
}
@@ -1170,7 +2347,7 @@ static int __gnix_msg_addr_lookup(struct gnix_fid_ep *ep, uint64_t src_addr,
{
int ret;
struct gnix_fid_av *av;
- struct gnix_av_addr_entry *av_entry;
+ struct gnix_av_addr_entry av_entry;
/* Translate source address. */
if (GNIX_EP_RDM_DGM(ep->type)) {
@@ -1185,7 +2362,7 @@ static int __gnix_msg_addr_lookup(struct gnix_fid_ep *ep, uint64_t src_addr,
ret);
return ret;
}
- *gnix_addr = av_entry->gnix_addr;
+ *gnix_addr = av_entry.gnix_addr;
} else {
*(uint64_t *)gnix_addr = FI_ADDR_UNSPEC;
}
@@ -1214,8 +2391,9 @@ ssize_t _gnix_recv(struct gnix_fid_ep *ep, uint64_t buf, size_t len,
uint64_t match_flags;
struct gnix_fid_mem_desc *md = NULL;
int tagged = !!(flags & FI_TAGGED);
+ bool try_again = false;
- if (!ep->recv_cq) {
+ if (!ep->recv_cq && !ep->recv_cntr) {
return -FI_ENOCQ;
}
@@ -1235,14 +2413,17 @@ ssize_t _gnix_recv(struct gnix_fid_ep *ep, uint64_t buf, size_t len,
__gnix_msg_queues(ep, tagged, &queue_lock, &posted_queue, &unexp_queue);
+ GNIX_DEBUG(FI_LOG_EP_DATA, "posted_queue = %p\n", posted_queue);
+
if (!tagged) {
tag = 0;
ignore = ~0;
}
- fastlock_acquire(queue_lock);
+ COND_ACQUIRE(ep->requires_lock, queue_lock);
retry_match:
+ try_again = false;
/* Look for a matching unexpected receive request. */
req = _gnix_match_tag(unexp_queue, tag, ignore,
match_flags, context, &gnix_addr);
@@ -1250,16 +2431,21 @@ retry_match:
/* Found matching request, populate local fields. */
req->gnix_ep = ep;
req->user_context = context;
+ req->msg.recv_info[0].recv_addr = (uint64_t)buf;
+ req->msg.recv_info[0].recv_len = len;
+ req->msg.cum_recv_len = len;
- req->msg.recv_addr = (uint64_t)buf;
- req->msg.recv_len = len;
if (mdesc) {
md = container_of(mdesc,
- struct gnix_fid_mem_desc,
- mr_fid);
+ struct gnix_fid_mem_desc,
+ mr_fid);
+ req->msg.recv_info[0].mem_hndl = md->mem_hndl;
}
- req->msg.recv_md = md;
+ req->msg.recv_md[0] = md;
+ req->msg.recv_iov_cnt = 1;
req->msg.recv_flags = flags;
+ if (flags & FI_MULTI_RECV)
+ req->msg.recv_flags |= GNIX_MSG_MULTI_RECV_SUP;
req->msg.ignore = ignore;
if ((flags & GNIX_SUPPRESS_COMPLETION) ||
@@ -1281,7 +2467,7 @@ retry_match:
if (req->msg.send_flags & GNIX_MSG_RENDEZVOUS) {
/* Matched rendezvous request. Start data movement. */
- GNIX_INFO(FI_LOG_EP_DATA, "matched RNDZV, req: %p\n",
+ GNIX_DEBUG(FI_LOG_EP_DATA, "matched RNDZV, req: %p\n",
req);
/*
@@ -1293,48 +2479,64 @@ retry_match:
}
/* Check if second GET for unaligned data is needed. */
- if (req->msg.send_len > req->msg.recv_len &&
- ((req->msg.send_addr + req->msg.recv_len) &
+ if (req->msg.send_info[0].send_len > req->msg.recv_info[0].recv_len &&
+ ((req->msg.send_info[0].send_addr + req->msg.recv_info[0].recv_len) &
GNI_READ_ALIGN_MASK)) {
req->msg.recv_flags |= GNIX_MSG_GET_TAIL;
}
/* Send length is truncated to receive buffer size. */
- req->msg.send_len = MIN(req->msg.send_len,
- req->msg.recv_len);
+ req->msg.cum_send_len =
+ MIN(req->msg.cum_send_len,
+ req->msg.recv_info[0].recv_len);
/* Initiate pull of source data. */
- req->work_fn = __gnix_rndzv_req;
- ret = _gnix_vc_queue_work_req(req);
+ req->work_fn = req->msg.send_iov_cnt == 1 ?
+ __gnix_rndzv_req : __gnix_rndzv_iov_req_build;
+
+
+ /*
+ * Check if we're using FI_MULTI_RECV and there is
+ * space left in the receive buffer.
+ */
- /* If using FI_MULTI_RECV and there is space left in
- * the receive buffer, try to match another unexpected
- * request. */
if ((req->msg.recv_flags & FI_MULTI_RECV) &&
- ((len - req->msg.send_len) >= ep->min_multi_recv)) {
- buf += req->msg.send_len;
- len -= req->msg.send_len;
+ ((len - req->msg.cum_send_len) >= ep->min_multi_recv)) {
- GNIX_INFO(FI_LOG_EP_DATA,
- "Attempting additional matches, "
- "req: %p (%p %u)\n",
- req, buf, len);
- goto retry_match;
+ buf += req->msg.cum_send_len;
+ len -= req->msg.cum_send_len;
+ try_again = true;
+
+ } else {
+ req->msg.recv_flags &= ~GNIX_MSG_MULTI_RECV_SUP;
}
+
+ ret = _gnix_vc_queue_work_req(req);
+ if (try_again)
+ goto retry_match;
+
} else {
/* Matched eager request. Copy data and generate
* completions. */
- GNIX_INFO(FI_LOG_EP_DATA, "Matched recv, req: %p\n",
+ GNIX_DEBUG(FI_LOG_EP_DATA, "Matched recv, req: %p\n",
req);
/* Send length is truncated to receive buffer size. */
- req->msg.send_len = MIN(req->msg.send_len,
- req->msg.recv_len);
+ req->msg.send_info[0].send_len =
+ MIN(req->msg.send_info[0].send_len,
+ req->msg.recv_info[0].recv_len);
+
+ req->msg.cum_send_len = req->msg.send_info[0].send_len;
/* Copy data from unexpected eager receive buffer. */
- memcpy((void *)buf, (void *)req->msg.send_addr,
- req->msg.send_len);
- free((void *)req->msg.send_addr);
+ memcpy((void *)buf, (void *)req->msg.send_info[0].send_addr,
+ req->msg.send_info[0].send_len);
+ free((void *)req->msg.send_info[0].send_addr);
+
+ if ((req->msg.recv_flags & FI_MULTI_RECV) &&
+ ((len - req->msg.cum_send_len) <
+ ep->min_multi_recv))
+ req->msg.recv_flags &= ~GNIX_MSG_MULTI_RECV_SUP;
__gnix_msg_recv_completion(ep, req);
@@ -1342,11 +2544,13 @@ retry_match:
* the receive buffer, try to match another unexpected
* request. */
if ((req->msg.recv_flags & FI_MULTI_RECV) &&
- ((len - req->msg.send_len) >= ep->min_multi_recv)) {
- buf += req->msg.send_len;
- len -= req->msg.send_len;
-
- GNIX_INFO(FI_LOG_EP_DATA,
+ ((len - req->msg.cum_send_len) >= ep->min_multi_recv)) {
+ buf += req->msg.cum_send_len;
+ len -= req->msg.cum_send_len;
+ req->msg.recv_info[0].recv_addr = buf;
+ req->msg.recv_info[0].recv_len -= req->msg.cum_send_len;
+ req->msg.cum_recv_len = req->msg.recv_info[0].recv_len;
+ GNIX_DEBUG(FI_LOG_EP_DATA,
"Attempting additional matches, "
"req: %p (%p %u)\n",
req, buf, len);
@@ -1378,7 +2582,7 @@ retry_match:
goto err;
}
- GNIX_INFO(FI_LOG_EP_DATA, "New recv, req: %p\n", req);
+ GNIX_DEBUG(FI_LOG_EP_DATA, "New recv, req: %p\n", req);
req->type = GNIX_FAB_RQ_RECV;
@@ -1386,15 +2590,23 @@ retry_match:
req->gnix_ep = ep;
req->user_context = context;
- req->msg.recv_addr = (uint64_t)buf;
- req->msg.recv_len = len;
+ req->msg.recv_info[0].recv_addr = (uint64_t)buf;
+ req->msg.recv_info[0].recv_len = len;
+ req->msg.cum_recv_len = len;
+
if (mdesc) {
md = container_of(mdesc,
struct gnix_fid_mem_desc,
mr_fid);
+
+ req->msg.recv_info[0].mem_hndl = md->mem_hndl;
}
- req->msg.recv_md = md;
+
+ req->msg.recv_md[0] = md;
+ req->msg.send_iov_cnt = req->msg.recv_iov_cnt = 1;
req->msg.recv_flags = flags;
+ if (flags & FI_MULTI_RECV)
+ req->msg.recv_flags |= GNIX_MSG_MULTI_RECV_SUP;
req->msg.tag = tag;
req->msg.ignore = ignore;
atomic_initialize(&req->msg.outstanding_txds, 0);
@@ -1406,13 +2618,12 @@ retry_match:
} else {
req->msg.recv_flags |= FI_COMPLETION;
}
-
_gnix_insert_tag(posted_queue, tag, req, ignore);
}
pdc_exit:
err:
- fastlock_release(queue_lock);
+ COND_RELEASE(ep->requires_lock, queue_lock);
return ret;
}
@@ -1449,64 +2660,121 @@ static int _gnix_send_req(void *arg)
}
assert(rc == FI_SUCCESS);
- if (rendezvous) {
- assert(req->msg.send_md);
-
- tag = GNIX_SMSG_T_RNDZV_START;
- tdesc->rndzv_start_hdr.flags = req->msg.send_flags;
- tdesc->rndzv_start_hdr.imm = req->msg.imm;
- tdesc->rndzv_start_hdr.msg_tag = req->msg.tag;
- tdesc->rndzv_start_hdr.mdh = req->msg.send_md->mem_hndl;
- tdesc->rndzv_start_hdr.addr = req->msg.send_addr;
- tdesc->rndzv_start_hdr.len = req->msg.send_len;
- tdesc->rndzv_start_hdr.req_addr = (uint64_t)req;
-
- if (req->msg.send_addr & GNI_READ_ALIGN_MASK) {
- tdesc->rndzv_start_hdr.head =
- *(uint32_t *)(req->msg.send_addr &
- ~GNI_READ_ALIGN_MASK);
- GNIX_INFO(FI_LOG_EP_DATA,
- "Sending %d unaligned head bytes (%x)\n",
- GNI_READ_ALIGN -
- (req->msg.send_addr & GNI_READ_ALIGN_MASK),
- tdesc->rndzv_start_hdr.head);
- }
+ if (unlikely(rendezvous)) {
+ switch(req->type) {
+
+ case GNIX_FAB_RQ_SEND:
+ case GNIX_FAB_RQ_TSEND:
+ assert(req->msg.send_md);
+ tag = GNIX_SMSG_T_RNDZV_START;
+ tdesc->rndzv_start_hdr.flags = req->msg.send_flags;
+ tdesc->rndzv_start_hdr.imm = req->msg.imm;
+ tdesc->rndzv_start_hdr.msg_tag = req->msg.tag;
+ tdesc->rndzv_start_hdr.mdh = req->msg.send_info[0].mem_hndl;
+ tdesc->rndzv_start_hdr.addr = req->msg.send_info[0].send_addr;
+ tdesc->rndzv_start_hdr.len = req->msg.send_info[0].send_len;
+ tdesc->rndzv_start_hdr.req_addr = (uint64_t)req;
+
+ if (req->msg.send_info[0].send_addr & GNI_READ_ALIGN_MASK) {
+ tdesc->rndzv_start_hdr.head =
+ *(uint32_t *)(req->msg.send_info[0].send_addr &
+ ~GNI_READ_ALIGN_MASK);
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "Sending %d unaligned head bytes (%x)\n",
+ GNI_READ_ALIGN -
+ (req->msg.send_info[0].send_addr &
+ GNI_READ_ALIGN_MASK),
+ tdesc->rndzv_start_hdr.head);
+ }
- if ((req->msg.send_addr + req->msg.send_len) &
- GNI_READ_ALIGN_MASK) {
- tdesc->rndzv_start_hdr.tail =
- *(uint32_t *)((req->msg.send_addr +
- req->msg.send_len) &
- ~GNI_READ_ALIGN_MASK);
- GNIX_INFO(FI_LOG_EP_DATA,
- "Sending %d unaligned tail bytes (%x)\n",
- (req->msg.send_addr + req->msg.send_len) &
- GNI_READ_ALIGN_MASK,
- tdesc->rndzv_start_hdr.tail);
- }
+ if ((req->msg.send_info[0].send_addr +
+ req->msg.send_info[0].send_len) &
+ GNI_READ_ALIGN_MASK) {
+ tdesc->rndzv_start_hdr.tail =
+ *(uint32_t *)((req->msg.send_info[0].send_addr +
+ req->msg.send_info[0].send_len) &
+ ~GNI_READ_ALIGN_MASK);
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "Sending %d unaligned tail bytes (%x)\n",
+ (req->msg.send_info[0].send_addr +
+ req->msg.send_info[0].send_len) &
+ GNI_READ_ALIGN_MASK,
+ tdesc->rndzv_start_hdr.tail);
+ }
+
+ hdr = &tdesc->rndzv_start_hdr;
+ hdr_len = sizeof(tdesc->rndzv_start_hdr);
+ /* TODO: Unify send&sendv/recv&recvv, so data will be
+ * req->msg.send_info */
+ data = NULL;
+ data_len = 0;
+
+ break;
+
+ case GNIX_FAB_RQ_SENDV:
+ case GNIX_FAB_RQ_TSENDV:
+ assert(req->msg.send_md[0]);
+ tag = GNIX_SMSG_T_RNDZV_IOV_START;
+ tdesc->rndzv_iov_start_hdr.flags = req->msg.send_flags;
+ tdesc->rndzv_iov_start_hdr.imm = req->msg.imm;
+ tdesc->rndzv_iov_start_hdr.msg_tag = req->msg.tag;
+ tdesc->rndzv_iov_start_hdr.iov_cnt =
+ req->msg.send_iov_cnt;
+ tdesc->rndzv_iov_start_hdr.req_addr = (uint64_t) req;
+ tdesc->rndzv_iov_start_hdr.send_len = req->msg.cum_send_len;
+
+ /* Send data at unaligned bytes in the iov addresses
+ * within the data section of the control message so
+ * that the remote peer can pull from four byte aligned
+ * addresses and still transfer all the data. */
+ __gnix_msg_send_alignment(req);
+
+ data_len = sizeof(struct send_info_t) * req->msg.send_iov_cnt;
+ data = (void *) req->msg.send_info;
+ hdr_len = sizeof(tdesc->rndzv_iov_start_hdr);
+ hdr = &tdesc->rndzv_iov_start_hdr;
+
+#if ENABLE_DEBUG
+ int i;
+ for (i = 0; i < req->msg.send_iov_cnt; i++) {
+ GNIX_DEBUG(FI_LOG_EP_DATA,
+ "data[%d].send_addr = 0x%ux, "
+ "data[%d].send_len = %lu, "
+ "data[%d].mem_hndl = %hxx\n", i,
+ ((struct send_info_t *)data)[i].send_addr,
+ i,
+ ((struct send_info_t *)data)[i].send_len,
+ i,
+ ((struct send_info_t *)data)[i].mem_hndl);
- hdr = &tdesc->rndzv_start_hdr;
- hdr_len = sizeof(tdesc->rndzv_start_hdr);
- data = NULL;
- data_len = 0;
+ }
+#endif
+ break;
+ default:
+ GNIX_FATAL(FI_LOG_EP_DATA, "Invalid request type: %d\n", req->type);
+ return -FI_EINVAL;
+ }
} else {
tag = GNIX_SMSG_T_EGR_W_DATA;
+
tdesc->eager_hdr.flags = req->msg.send_flags;
tdesc->eager_hdr.imm = req->msg.imm;
tdesc->eager_hdr.msg_tag = req->msg.tag;
- tdesc->eager_hdr.len = req->msg.send_len;
+ tdesc->eager_hdr.len = req->msg.cum_send_len;
hdr = &tdesc->eager_hdr;
hdr_len = sizeof(tdesc->eager_hdr);
- data = (void *)req->msg.send_addr;
- data_len = req->msg.send_len;
+ data = (void *)req->msg.send_info[0].send_addr;
+ /* If this is not rndzv the send length should always be the
+ * cumulative length of all the send_info lengths */
+ data_len = req->msg.cum_send_len;
}
tdesc->req = req;
tdesc->completer_fn = gnix_ep_smsg_completers[tag];
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
- if (inject_err) {
+ if (unlikely(inject_err)) {
_gnix_nic_txd_err_inject(nic, tdesc);
status = GNI_RC_SUCCESS;
} else {
@@ -1518,12 +2786,14 @@ static int _gnix_send_req(void *arg)
/*
* if this is a rendezvous message, we want to generate
* IRQ at remote peer.
+ * TODO: Do we need to do this for sendv?
*/
if ((status == GNI_RC_SUCCESS) &&
- (tag == GNIX_SMSG_T_RNDZV_START))
+ (tag == GNIX_SMSG_T_RNDZV_START ||
+ tag == GNIX_SMSG_T_RNDZV_IOV_START))
_gnix_rma_post_irq(req->vc);
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
if (status == GNI_RC_NOT_DONE) {
_gnix_nic_tx_free(nic, tdesc);
@@ -1551,12 +2821,17 @@ ssize_t _gnix_send(struct gnix_fid_ep *ep, uint64_t loc_addr, size_t len,
int rendezvous;
struct fid_mr *auto_mr = NULL;
- if (!ep) {
- return -FI_EINVAL;
+ if (!ep->send_cq && !ep->send_cntr) {
+ return -FI_ENOCQ;
}
- if (!ep->send_cq) {
- return -FI_ENOCQ;
+ if (flags & FI_TRIGGER) {
+ struct fi_triggered_context *trigger_context =
+ (struct fi_triggered_context *)context;
+ if ((trigger_context->event_type != FI_TRIGGER_THRESHOLD) ||
+ (flags & FI_INJECT)) {
+ return -FI_EINVAL;
+ }
}
if ((flags & FI_INJECT) && (len > GNIX_INJECT_SIZE)) {
@@ -1582,14 +2857,14 @@ ssize_t _gnix_send(struct gnix_fid_ep *ep, uint64_t loc_addr, size_t len,
len, FI_READ | FI_WRITE, 0, 0, 0,
&auto_mr, NULL);
if (ret != FI_SUCCESS) {
- GNIX_INFO(FI_LOG_EP_DATA,
- "Failed to auto-register local buffer: %d\n",
- ret);
+ GNIX_DEBUG(FI_LOG_EP_DATA,
+ "Failed to auto-register local buffer: %s\n",
+ fi_strerror(-ret));
return ret;
}
flags |= FI_LOCAL_MR;
mdesc = (void *)auto_mr;
- GNIX_INFO(FI_LOG_EP_DATA, "auto-reg MR: %p\n", auto_mr);
+ GNIX_DEBUG(FI_LOG_EP_DATA, "auto-reg MR: %p\n", auto_mr);
}
ret = _gnix_vc_ep_get_vc(ep, dest_addr, &vc);
@@ -1618,19 +2893,22 @@ ssize_t _gnix_send(struct gnix_fid_ep *ep, uint64_t loc_addr, size_t len,
if (mdesc) {
md = container_of(mdesc, struct gnix_fid_mem_desc, mr_fid);
+ req->msg.send_info[0].mem_hndl = md->mem_hndl;
}
- req->msg.send_md = md;
- req->msg.send_len = len;
+ req->msg.send_md[0] = md;
+ req->msg.send_iov_cnt = 1;
req->msg.send_flags = flags;
+ req->msg.send_info[0].send_len = len;
+ req->msg.cum_send_len = len;
req->msg.imm = data;
req->flags = 0;
if (flags & FI_INJECT) {
memcpy(req->inject_buf, (void *)loc_addr, len);
- req->msg.send_addr = (uint64_t)req->inject_buf;
+ req->msg.send_info[0].send_addr = (uint64_t)req->inject_buf;
req->flags |= FI_INJECT;
} else {
- req->msg.send_addr = loc_addr;
+ req->msg.send_info[0].send_addr = loc_addr;
}
if ((flags & GNIX_SUPPRESS_COMPLETION) ||
@@ -1651,7 +2929,7 @@ ssize_t _gnix_send(struct gnix_fid_ep *ep, uint64_t loc_addr, size_t len,
req->msg.send_flags |= GNIX_MSG_RENDEZVOUS;
}
- GNIX_INFO(FI_LOG_EP_DATA, "Queuing (%p %d)\n",
+ GNIX_DEBUG(FI_LOG_EP_DATA, "Queuing (%p %d)\n",
(void *)loc_addr, len);
return _gnix_vc_queue_tx_req(req);
@@ -1664,3 +2942,459 @@ err_get_vc:
return ret;
}
+ssize_t _gnix_recvv(struct gnix_fid_ep *ep, const struct iovec *iov,
+ void **desc, size_t count, uint64_t src_addr, void *context,
+ uint64_t flags, uint64_t ignore, uint64_t tag)
+{
+ int i, ret = FI_SUCCESS;
+ size_t cum_len = 0;
+ struct gnix_fab_req *req = NULL;
+ struct gnix_address gnix_addr;
+ fastlock_t *queue_lock = NULL;
+ struct gnix_tag_storage *posted_queue = NULL;
+ struct gnix_tag_storage *unexp_queue = NULL;
+ uint64_t match_flags;
+ int tagged = flags & FI_TAGGED;
+ struct fid_mr *auto_mr;
+
+ if (!ep->recv_cq && !ep->recv_cntr) {
+ return -FI_ENOCQ;
+ }
+
+ if (!tagged) {
+ if (!ep->ep_ops.msg_send_allowed)
+ return -FI_EOPNOTSUPP;
+
+ tag = 0;
+ ignore = ~0;
+ } else {
+ if (!ep->ep_ops.tagged_send_allowed)
+ return -FI_EOPNOTSUPP;
+ }
+
+ match_flags = flags & (FI_CLAIM | FI_DISCARD | FI_PEEK);
+
+ /*
+ * Lookup the gni addr in the av_table or av_hashtable.
+ * If the gni addr doesn't exist the addr is FI_ADDR_UNSPEC,
+ * meaning this remote node wants to receive from all senders?
+ */
+ ret = __gnix_msg_addr_lookup(ep, src_addr, &gnix_addr);
+ if (ret != FI_SUCCESS)
+ return ret;
+
+ /* calculate cumulative size of the iovec buf lens */
+ for (i = 0; i < count; i++) {
+ cum_len += iov[i].iov_len;
+ }
+
+ /*
+ * Initialize the tag storage objects.
+ * The posted_queue holds information about receives that have
+ * been posted on the remote endpoint.
+ *
+ * The unexp_queue holds information about data that has arrived
+ * prior to posting a receive on the remote endpoint.
+ *
+ * Both {unexp,posted}_queue objects have two sets, one for tagged
+ * messages and the other for untagged messages.
+ *
+ * The untagged queues match based off the source address.
+ *
+ * The tagged queues match based off the tag and source address (when
+ * the ep is created with FI_DIRECTED_RECV).
+ *
+ * A "message" is added to the unexpected queue when it arrives at a
+ * remote endpoint and the completer_fn doesn't find an existing request
+ * in the posted queue (i.e. no fi_recvs have been called (or posted)
+ * on the remote endpoint).
+ */
+ __gnix_msg_queues(ep, tagged, &queue_lock, &posted_queue, &unexp_queue);
+
+ COND_ACQUIRE(ep->requires_lock, queue_lock);
+
+ /*
+ * Posting a recv, look for an existing request in the
+ * unexpected queue.
+ */
+ req = _gnix_match_tag(unexp_queue, tag, ignore,
+ match_flags, context, &gnix_addr);
+
+ if (req) {
+ GNIX_DEBUG(FI_LOG_EP_DATA, "UNEXPECTED, req = %p\n", req);
+ /* Found a matching request in the unexpected queue. */
+
+ /*
+ * reset ep, it might be different than the ep the message came
+ * in on.
+ */
+ req->gnix_ep = ep;
+ req->user_context = context;
+ req->flags = 0;
+ req->msg.recv_flags = flags;
+ req->msg.recv_iov_cnt = count;
+ req->msg.cum_recv_len = cum_len;
+ /* req->msg.cum_send_len = MIN(req->msg.cum_send_len, cum_len); */
+
+ if (tagged) {
+ req->type = GNIX_FAB_RQ_TRECVV;
+ } else {
+ req->type = GNIX_FAB_RQ_RECVV;
+ }
+
+ if ((flags & GNIX_SUPPRESS_COMPLETION) ||
+ (ep->recv_selective_completion &&
+ !(flags & FI_COMPLETION))) {
+ req->msg.recv_flags &= ~FI_COMPLETION;
+ } else {
+ req->msg.recv_flags |= FI_COMPLETION;
+ }
+
+ /* Check to see if we are using P/C/D matching flags. */
+ if (match_flags & FI_DISCARD) {
+ ret = __gnix_discard_request(req);
+ goto pdc_exit;
+ } else if (match_flags & FI_PEEK) {
+ ret = __gnix_peek_request(req);
+ goto pdc_exit;
+ }
+
+ if (req->msg.send_flags & GNIX_MSG_RENDEZVOUS) {
+ req->work_fn = __gnix_rndzv_iov_req_build;
+ if (!desc) { /* Register the memory for the user */
+ for (i = 0; i < count; i++) {
+ auto_mr = NULL;
+ ret = gnix_mr_reg(&ep->domain->
+ domain_fid.fid,
+ iov[i].iov_base,
+ iov[i].iov_len,
+ FI_READ | FI_WRITE, 0,
+ 0, 0, &auto_mr,
+ NULL);
+
+ if (ret != FI_SUCCESS) {
+ GNIX_DEBUG(FI_LOG_EP_DATA,
+ "Failed to "
+ "auto-register"
+ " local buffer: %s\n"
+ , fi_strerror(-ret));
+
+ for (i--; i >= 0; i--) {
+ fi_close(&req->msg.recv_md[i]->mr_fid.fid);
+ }
+
+ return ret;
+ }
+
+ req->msg.recv_md[i] = container_of(
+ (void *) auto_mr,
+ struct gnix_fid_mem_desc,
+ mr_fid);
+
+ req->msg.recv_info[i].recv_addr = (uint64_t) iov[i].iov_base;
+ req->msg.recv_info[i].recv_len = iov[i].iov_len;
+ req->msg.recv_info[i].mem_hndl =
+ req->msg.recv_md[i]->mem_hndl;
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "auto-reg MR"
+ ": %p\n",
+ req->msg.recv_md[i]);
+
+ }
+
+ req->msg.send_flags |= FI_LOCAL_MR;
+ } else { /* User registered their memory */
+ for (i = 0; i < count; i++) {
+ if (!desc[i]) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "invalid memory reg"
+ "istration (%p).\n",
+ desc[i]);
+ return -FI_EINVAL;
+ }
+
+ req->msg.recv_md[i] =
+ container_of(desc[i],
+ struct gnix_fid_mem_desc,
+ mr_fid);
+
+ req->msg.recv_info[i].mem_hndl =
+ req->msg.recv_md[i]->mem_hndl;
+ }
+ }
+
+ ret = _gnix_vc_queue_work_req(req);
+ } else {
+ for (i = 0; i < count; i++) {
+ req->msg.recv_info[i].recv_addr = (uint64_t) iov[i].iov_base;
+ req->msg.recv_info[i].recv_len = iov[i].iov_len;
+ }
+
+ /*
+ * This request is associate with a regular eager smsg,
+ * the rndzv threshold on the sender was not reached or
+ * exceeded.
+ */
+ __gnix_msg_unpack_data_into_iov(req->msg.recv_info,
+ count,
+ req->msg.send_info[0].send_addr,
+ req->msg.send_info[0].send_len);
+
+ __gnix_msg_recv_completion(ep, req);
+ _gnix_fr_free(ep, req);
+ }
+ } else {
+ /* if peek/claim/discard, we didn't find what we
+ * were looking for, return FI_ENOMSG
+ */
+ if (match_flags) {
+ __recv_err(ep, context, flags, cum_len,
+ (void *) iov, 0, tag, cum_len, FI_ENOMSG,
+ FI_ENOMSG, NULL);
+
+ /* if handling trecvmsg flags, return here
+ * Never post a receive request from this type of
+ * context
+ */
+ ret = FI_SUCCESS;
+ goto pdc_exit;
+ }
+
+ /*
+ * No matching requests found, create a new one and enqueue
+ * it in the posted queue.
+ */
+ req = _gnix_fr_alloc(ep);
+ if (req == NULL) {
+ ret = -FI_EAGAIN;
+ goto err;
+ }
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "EXPECTED, req = %p\n", req);
+
+ if (tagged) {
+ req->type = GNIX_FAB_RQ_TRECVV;
+ } else {
+ req->type = GNIX_FAB_RQ_RECVV;
+ }
+
+ if ((flags & GNIX_SUPPRESS_COMPLETION) ||
+ (ep->recv_selective_completion &&
+ !(flags & FI_COMPLETION))) {
+ req->msg.recv_flags &= ~FI_COMPLETION;
+ } else {
+ req->msg.recv_flags |= FI_COMPLETION;
+ }
+
+ req->addr = gnix_addr;
+ req->gnix_ep = ep;
+ req->user_context = context;
+ req->flags = 0;
+
+ for (i = 0; i < count; i++) {
+ req->msg.recv_info[i].recv_addr = (uint64_t) iov[i].iov_base;
+ req->msg.recv_info[i].recv_len = iov[i].iov_len;
+ req->msg.recv_md[i] = NULL;
+ }
+
+ req->msg.recv_iov_cnt = count;
+ req->msg.recv_flags = flags;
+ req->msg.cum_recv_len = cum_len;
+ req->msg.tag = tag;
+ req->msg.ignore = ignore;
+ atomic_initialize(&req->msg.outstanding_txds, 0);
+
+
+ if ((flags & GNIX_SUPPRESS_COMPLETION) ||
+ (ep->recv_selective_completion &&
+ !(flags & FI_COMPLETION))) {
+ req->msg.recv_flags &= ~FI_COMPLETION;
+ } else {
+ req->msg.recv_flags |= FI_COMPLETION;
+ }
+
+ _gnix_insert_tag(posted_queue, tag, req, ignore);
+ }
+
+pdc_exit:
+err:
+ COND_RELEASE(ep->requires_lock, queue_lock);
+
+ return ret;
+}
+
+ssize_t _gnix_sendv(struct gnix_fid_ep *ep, const struct iovec *iov,
+ void **mdesc, size_t count, uint64_t dest_addr,
+ void *context, uint64_t flags, uint64_t tag)
+{
+ int i, ret = FI_SUCCESS;
+ unsigned long long cum_len = 0;
+ void *tmp = NULL;
+ struct gnix_vc *vc = NULL;
+ struct gnix_fab_req *req = NULL;
+ struct fid_mr *auto_mr;
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "iov_count = %lu\n", count);
+
+ if (!ep->send_cq && !ep->send_cntr) {
+ return -FI_ENOCQ;
+ }
+
+ if (!(flags & FI_TAGGED)) {
+ if (!ep->ep_ops.msg_send_allowed)
+ return -FI_EOPNOTSUPP;
+ } else {
+ if (!ep->ep_ops.tagged_send_allowed)
+ return -FI_EOPNOTSUPP;
+ }
+
+ ret = _gnix_vc_ep_get_vc(ep, dest_addr, &vc);
+ if (ret != FI_SUCCESS) {
+ return ret;
+ }
+
+ req = _gnix_fr_alloc(ep);
+ if (req == NULL) {
+ return -FI_ENOSPC;
+ }
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "Created req - %p\n", req);
+
+ /* calculate cumulative size of the iovec buf lens */
+ for (i = 0; i < count; i++) {
+ /* TODO: handle possible overflow */
+ cum_len += iov[i].iov_len;
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "iov[%d].iov_len = %lu\n", i, iov[i].iov_len);
+ }
+
+ /* Fill out fabric request */
+ if (flags & FI_TAGGED) {
+ req->type = GNIX_FAB_RQ_TSENDV;
+ req->msg.tag = tag;
+ req->msg.ignore = 0;
+ } else {
+ req->type = GNIX_FAB_RQ_SENDV;
+ req->msg.tag = 0;
+ req->msg.ignore = ~0;
+ }
+
+ req->gnix_ep = ep;
+ req->user_context = context;
+ req->vc = vc;
+ req->work_fn = _gnix_send_req;
+ req->flags = 0; /* Flags that apply to all message types? */
+ req->msg.send_flags = flags;
+ req->msg.imm = 0;
+
+ /*
+ * If the cum_len is >= ep->domain->params.msg_rendezvous_thresh
+ * transfer the iovec entries individually.
+ *
+ * For this case, use CtPostFma for iovec lengths that are smaller than
+ * the rendezvous thresh. For CtPostFma:
+ * the sum of the iov lens must be either <= 1GB or <= 1MB if the comm
+ * dom is configured with FmaSharing.
+ * otherwise use PostRdma.
+ */
+ if (cum_len >= ep->domain->params.msg_rendezvous_thresh) {
+ if (!mdesc) { /* Register the memory for the user */
+ for (i = 0; i < count; i++) {
+ auto_mr = NULL;
+ ret = gnix_mr_reg(&ep->domain->domain_fid.fid,
+ iov[i].iov_base,
+ iov[i].iov_len,
+ FI_READ | FI_WRITE, 0, 0, 0,
+ &auto_mr, NULL);
+
+ if (ret != FI_SUCCESS) {
+ GNIX_DEBUG(FI_LOG_EP_DATA,
+ "Failed to auto-register"
+ " local buffer: %s\n",
+ fi_strerror(-ret));
+
+ for (i--; i >= 0; i--) {
+ fi_close(&req->msg.send_md[i]->mr_fid.fid);
+ }
+
+ return ret;
+ }
+
+ req->msg.send_md[i] = container_of(
+ (void *) auto_mr,
+ struct gnix_fid_mem_desc,
+ mr_fid);
+
+ req->msg.send_info[i].send_addr = (uint64_t) iov[i].iov_base;
+ req->msg.send_info[i].send_len = iov[i].iov_len;
+ req->msg.send_info[i].mem_hndl =
+ req->msg.send_md[i]->mem_hndl;
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "iov[%d].iov_len = %lu,"
+ " req->msg.send_info[%d].send_addr = "
+ "%p, req->msg.send_info[%d].send_len "
+ "= %lu\n", i, iov[i].iov_len, i,
+ (void *) req->msg.send_info[i].send_addr,
+ i, req->msg.send_info[i].send_len);
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "req->msg.send_md[%d] "
+ "= %p\n", i,
+ req->msg.send_md[i]);
+
+ GNIX_DEBUG(FI_LOG_EP_DATA, "auto-reg MR: %p\n",
+ req->msg.send_md[i]);
+
+ }
+
+ req->msg.send_flags |= FI_LOCAL_MR;
+ } else { /* User registered their memory */
+ for (i = 0; i < count; i++) {
+ if (!mdesc[i]) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "invalid memory reg"
+ "istration (%p).\n",
+ mdesc[i]);
+ return -FI_EINVAL;
+ }
+
+ req->msg.send_md[i] =
+ container_of(mdesc[i],
+ struct gnix_fid_mem_desc,
+ mr_fid);
+
+ req->msg.send_info[i].send_addr = (uint64_t) iov[i].iov_base;
+ req->msg.send_info[i].send_len = iov[i].iov_len;
+ req->msg.send_info[i].mem_hndl =
+ req->msg.send_md[i]->mem_hndl;
+ }
+ }
+
+ req->msg.send_iov_cnt = count;
+ req->msg.send_flags |= GNIX_MSG_RENDEZVOUS;
+ } else {
+ /*
+ * TODO: Use buddy allocator with max alloc lim of
+ * ep->domain->params.msg_rendezvous_thresh
+ */
+ /* This is freed in __comp_eager_msg_w_data */
+ tmp = malloc(cum_len);
+ assert(tmp != NULL);
+
+ __gnix_msg_pack_data_from_iov((uint64_t) tmp, cum_len,
+ iov, count);
+ req->msg.send_info[0].send_addr = (uint64_t) tmp;
+ req->msg.send_info[0].send_len = cum_len;
+ }
+
+ if ((flags & GNIX_SUPPRESS_COMPLETION) ||
+ (ep->send_selective_completion &&
+ !(flags & FI_COMPLETION))) {
+ req->msg.send_flags &= ~FI_COMPLETION;
+ } else {
+ req->msg.send_flags |= FI_COMPLETION;
+ }
+
+ req->msg.cum_send_len = (size_t) cum_len;
+
+ return _gnix_vc_queue_tx_req(req);
+}
diff --git a/prov/gni/src/gnix_nameserver.c b/prov/gni/src/gnix_nameserver.c
index 59fec0a..488eeab 100644
--- a/prov/gni/src/gnix_nameserver.c
+++ b/prov/gni/src/gnix_nameserver.c
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2014 Intel Corporation, Inc. All rights reserved.
- * Copyright (c) 2015 Los Alamos National Security, LLC. Allrights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC.
+ * Allrights reserved.
* Copyright (c) 2015 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -70,6 +71,8 @@ static int __gnix_ipaddr_from_iface(const char *iface, struct sockaddr_in *sin)
struct ifreq ifr = { { { 0 } } };
int sock = -1;
+ GNIX_TRACE(FI_LOG_FABRIC, "\n");
+
sock = socket(AF_INET, SOCK_DGRAM, 0);
if (sock == -1) {
GNIX_WARN(FI_LOG_FABRIC, "Socket creation failed: %s\n",
@@ -110,7 +113,8 @@ exit_w_sock:
* returns 0 if ipogif entry found
* otherwise -errno
*/
-static int gnixu_get_pe_from_ip(const char *ip_addr, uint32_t *gni_nic_addr)
+static int gnixu_get_pe_from_ip(const char *iface_name, const char *ip_addr,
+ uint32_t *gni_nic_addr)
{
int scount;
/* return this if no ipgogif for this ip-addr found */
@@ -121,6 +125,8 @@ static int gnixu_get_pe_from_ip(const char *ip_addr, uint32_t *gni_nic_addr)
char mac_str[64];
int w, x, y;
+ GNIX_TRACE(FI_LOG_FABRIC, "\n");
+
fd = fopen("/proc/net/arp", "r");
if (fd == NULL) {
return -errno;
@@ -140,7 +146,7 @@ static int gnixu_get_pe_from_ip(const char *ip_addr, uint32_t *gni_nic_addr)
* check for a match
*/
if ((strstr(line, ip_addr) != NULL) &&
- (strstr(line, "ipogif") != NULL)) {
+ (strstr(line, iface_name) != NULL)) {
ret = 0;
scount = sscanf(line, "%s%s%s%s%s%s", fnd_ip_addr,
dummy, dummy, mac_str, dummy, iface);
@@ -208,6 +214,8 @@ int gnix_resolve_name(IN const char *node, IN const char *service,
.ai_flags = AI_CANONNAME
};
+ GNIX_TRACE(FI_LOG_FABRIC, "\n");
+
if (flags & FI_SOURCE)
hints.ai_flags |= AI_PASSIVE;
@@ -229,11 +237,13 @@ int gnix_resolve_name(IN const char *node, IN const char *service,
ret = __gnix_ipaddr_from_iface("ipogif0", &sin);
if (ret != FI_SUCCESS)
ret = __gnix_ipaddr_from_iface("br0", &sin);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_FABRIC,
- "Unable to obtain local iface addr\n");
- goto err;
- }
+
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_FABRIC,
+ "Unable to obtain local iface addr\n");
+ goto err;
+ }
+
ret = getaddrinfo(node, service, &hints, &result);
if (ret != 0) {
@@ -264,10 +274,17 @@ int gnix_resolve_name(IN const char *node, IN const char *service,
}
} else {
ret =
- gnixu_get_pe_from_ip(inet_ntoa(sa->sin_addr), &pe);
+ gnixu_get_pe_from_ip("ipogif0",
+ inet_ntoa(sa->sin_addr), &pe);
if (ret == 0) {
break;
+ } else {
+ ret = gnixu_get_pe_from_ip("br0",
+ inet_ntoa(sa->sin_addr),
+ &pe);
}
+ if (ret == 0)
+ break;
}
}
diff --git a/prov/gni/src/gnix_nic.c b/prov/gni/src/gnix_nic.c
index 96224c3..8933ea9 100644
--- a/prov/gni/src/gnix_nic.c
+++ b/prov/gni/src/gnix_nic.c
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC.
+ * All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -294,12 +295,16 @@ static int __nic_rx_overrun(struct gnix_nic *nic)
GNIX_WARN(FI_LOG_EP_DATA, "\n");
/* clear out the CQ */
+ /*
+ * TODO: really need to process CQEs better for error reporting,
+ * etc.
+ */
while ((status = GNI_CqGetEvent(nic->rx_cq, &cqe)) == GNI_RC_SUCCESS);
assert(status == GNI_RC_NOT_DONE);
- fastlock_acquire(&nic->vc_id_lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->vc_id_lock);
max_id = nic->vc_id_table_count;
- fastlock_release(&nic->vc_id_lock);
+ COND_RELEASE(nic->requires_lock, &nic->vc_id_lock);
/*
* TODO: optimization would
* be to keep track of last time
@@ -332,13 +337,13 @@ static int __process_rx_cqe(struct gnix_nic *nic, gni_cq_entry_t cqe)
#if 1 /* Process RX inline with arrival of an RX CQE. */
if (unlikely(vc->conn_state != GNIX_VC_CONNECTED)) {
- GNIX_INFO(FI_LOG_EP_DATA,
+ GNIX_DEBUG(FI_LOG_EP_DATA,
"Scheduling VC for RX processing (%p)\n",
vc);
ret = _gnix_vc_rx_schedule(vc);
assert(ret == FI_SUCCESS);
} else {
- GNIX_INFO(FI_LOG_EP_DATA,
+ GNIX_DEBUG(FI_LOG_EP_DATA,
"Processing VC RX (%p)\n",
vc);
ret = _gnix_vc_dequeue_smsg(vc);
@@ -366,7 +371,7 @@ static int __nic_rx_progress(struct gnix_nic *nic)
if (status == GNI_RC_NOT_DONE)
return FI_SUCCESS;
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
do {
status = GNI_CqGetEvent(nic->rx_cq, &cqe);
@@ -398,7 +403,7 @@ static int __nic_rx_progress(struct gnix_nic *nic)
}
} while (1);
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
return ret;
}
@@ -434,10 +439,11 @@ static void __nic_get_completed_txd(struct gnix_nic *nic,
{
gni_post_descriptor_t *gni_desc;
struct gnix_tx_descriptor *txd_p = NULL;
+ struct gnix_fab_req *req;
gni_return_t status;
int msg_id;
gni_cq_entry_t cqe;
- uint32_t recov;
+ uint32_t recov = 1;
if (__gnix_nic_txd_err_get(nic, &txd_p)) {
*txd = txd_p;
@@ -455,13 +461,22 @@ static void __nic_get_completed_txd(struct gnix_nic *nic,
assert(status == GNI_RC_SUCCESS ||
status == GNI_RC_TRANSACTION_ERROR);
- if (status == GNI_RC_TRANSACTION_ERROR) {
+ if (unlikely(status == GNI_RC_TRANSACTION_ERROR)) {
status = GNI_CqErrorRecoverable(cqe, &recov);
- if (status != GNI_RC_SUCCESS || !recov) {
- char ebuf[512];
- GNI_CqErrorStr(cqe, ebuf, sizeof(ebuf));
- GNIX_FATAL(FI_LOG_EP_DATA, "CQ error status: %s\n",
- ebuf);
+ if (status == GNI_RC_SUCCESS) {
+ if (!recov) {
+ char ebuf[512];
+
+ GNI_CqErrorStr(cqe, ebuf, sizeof(ebuf));
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "CQ error status: %s\n",
+ ebuf);
+ }
+ } else {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "GNI_CqErrorRecover returned: %s\n",
+ gni_err_str[status]);
+ recov = 0; /* assume something bad has happened */
}
}
@@ -479,8 +494,18 @@ static void __nic_get_completed_txd(struct gnix_nic *nic,
txd_p = __desc_lkup_by_id(nic, msg_id);
}
- if (!txd_p) {
+ if (unlikely(txd_p == NULL))
GNIX_FATAL(FI_LOG_EP_DATA, "Unexpected CQE: 0x%lx", cqe);
+
+ /*
+ * set retry count on the request to max to force
+ * delivering error'd CQ event to application
+ */
+ if (!recov) {
+ status = GNI_RC_TRANSACTION_ERROR;
+ req = txd_p->req;
+ if (req)
+ req->tx_failures = UINT_MAX;
}
*tx_status = status;
@@ -497,10 +522,10 @@ static int __nic_tx_progress(struct gnix_nic *nic, gni_cq_handle_t cq)
do {
txd = NULL;
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
__nic_get_completed_txd(nic, cq, &txd,
&tx_status);
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
if (txd && txd->completer_fn) {
ret = txd->completer_fn(txd, tx_status);
@@ -535,11 +560,11 @@ int _gnix_nic_progress(struct gnix_nic *nic)
}
ret = __nic_rx_progress(nic);
- if (unlikely(ret != FI_SUCCESS))
+ if (ret != FI_SUCCESS)
return ret;
ret = _gnix_vc_nic_progress(nic);
- if (unlikely(ret != FI_SUCCESS))
+ if (ret != FI_SUCCESS)
return ret;
return ret;
@@ -576,7 +601,7 @@ int _gnix_nic_get_rem_id(struct gnix_nic *nic, int *remote_id, void *entry)
* bit before resizing the table
*/
- fastlock_acquire(&nic->vc_id_lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->vc_id_lock);
if (nic->vc_id_table_capacity == nic->vc_id_table_count) {
table_base = realloc(nic->vc_id_table,
2 * nic->vc_id_table_capacity *
@@ -587,6 +612,13 @@ int _gnix_nic_get_rem_id(struct gnix_nic *nic, int *remote_id, void *entry)
}
nic->vc_id_table_capacity *= 2;
nic->vc_id_table = table_base;
+
+ ret = _gnix_realloc_bitmap(&nic->vc_id_bitmap,
+ nic->vc_id_table_capacity);
+ if (ret != FI_SUCCESS) {
+ assert(ret == -FI_ENOMEM);
+ goto err;
+ }
}
nic->vc_id_table[nic->vc_id_table_count] = entry;
@@ -600,7 +632,7 @@ int _gnix_nic_get_rem_id(struct gnix_nic *nic, int *remote_id, void *entry)
++(nic->vc_id_table_count);
err:
- fastlock_release(&nic->vc_id_lock);
+ COND_RELEASE(nic->requires_lock, &nic->vc_id_lock);
return ret;
}
@@ -613,9 +645,9 @@ int _gnix_nic_tx_alloc(struct gnix_nic *nic,
{
struct dlist_entry *entry;
- fastlock_acquire(&nic->tx_desc_lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->tx_desc_lock);
if (dlist_empty(&nic->tx_desc_free_list)) {
- fastlock_release(&nic->tx_desc_lock);
+ COND_RELEASE(nic->requires_lock, &nic->tx_desc_lock);
return -FI_ENOSPC;
}
@@ -623,7 +655,7 @@ int _gnix_nic_tx_alloc(struct gnix_nic *nic,
dlist_remove_init(entry);
dlist_insert_head(entry, &nic->tx_desc_active_list);
*desc = dlist_entry(entry, struct gnix_tx_descriptor, list);
- fastlock_release(&nic->tx_desc_lock);
+ COND_RELEASE(nic->requires_lock, &nic->tx_desc_lock);
return FI_SUCCESS;
}
@@ -636,10 +668,10 @@ int _gnix_nic_tx_alloc(struct gnix_nic *nic,
int _gnix_nic_tx_free(struct gnix_nic *nic,
struct gnix_tx_descriptor *desc)
{
- fastlock_acquire(&nic->tx_desc_lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->tx_desc_lock);
dlist_remove_init(&desc->list);
dlist_insert_head(&desc->list, &nic->tx_desc_free_list);
- fastlock_release(&nic->tx_desc_lock);
+ COND_RELEASE(nic->requires_lock, &nic->tx_desc_lock);
return FI_SUCCESS;
}
@@ -688,7 +720,8 @@ static int __gnix_nic_tx_freelist_init(struct gnix_nic *nic, int n_descs)
for (i = 0, desc_ptr = desc_base; i < n_descs; i++, desc_ptr++) {
desc_ptr->id = i;
- desc_ptr->int_buf = int_bufs + (i * GNIX_CACHELINE_SIZE);
+ desc_ptr->int_buf = (void *) ((uint8_t *) int_bufs +
+ (i * GNIX_CACHELINE_SIZE));
dlist_insert_tail(&desc_ptr->list,
&nic->tx_desc_free_list);
}
@@ -907,14 +940,16 @@ int gnix_nic_alloc(struct gnix_fid_domain *domain,
struct gnix_nic *nic = NULL;
uint32_t device_addr;
gni_return_t status;
- uint32_t fake_cdm_id, seed;
+ uint32_t fake_cdm_id;
gni_smsg_attr_t smsg_mbox_attr;
struct gnix_nic_attr *nic_attr = &default_attr;
+ uint32_t num_corespec_cpus = 0;
bool must_alloc_nic = false;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
*nic_ptr = NULL;
+ nic_attr->gni_cdm_modes = gnix_cdm_modes;
if (attr) {
ret = __gnix_nic_check_attr_sanity(attr);
@@ -974,15 +1009,7 @@ int gnix_nic_alloc(struct gnix_fid_domain *domain,
}
if (nic_attr->use_cdm_id == false) {
- ret = _gnix_cm_nic_get_cdm_seed_set(domain, 1, &seed);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "_gnix_cm_nic_get_cdm_seed_set returned %s\n",
- fi_strerror(-ret));
- goto err;
- }
- ret = _gnix_cm_nic_create_cdm_id(domain, seed,
- &fake_cdm_id);
+ ret = _gnix_cm_nic_create_cdm_id(domain, &fake_cdm_id);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_cm_nic_create_cdm_id returned %s\n",
@@ -1005,8 +1032,9 @@ int gnix_nic_alloc(struct gnix_fid_domain *domain,
goto err1;
}
nic->allocd_gni_res |= GNIX_NIC_CDM_ALLOCD;
- } else
+ } else {
nic->gni_cdm_hndl = nic_attr->gni_cdm_hndl;
+ }
/*
* Okay, now go for the attach
@@ -1020,6 +1048,7 @@ int gnix_nic_alloc(struct gnix_fid_domain *domain,
if (status != GNI_RC_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL, "GNI_CdmAttach returned %s\n",
gni_err_str[status]);
+ _gnix_dump_gni_res(domain->ptag);
ret = gnixu_to_fi_errno(status);
goto err1;
}
@@ -1043,6 +1072,7 @@ int gnix_nic_alloc(struct gnix_fid_domain *domain,
GNIX_WARN(FI_LOG_EP_CTRL,
"GNI_CqCreate returned %s\n",
gni_err_str[status]);
+ _gnix_dump_gni_res(domain->ptag);
ret = gnixu_to_fi_errno(status);
goto err1;
}
@@ -1059,6 +1089,7 @@ int gnix_nic_alloc(struct gnix_fid_domain *domain,
GNIX_WARN(FI_LOG_EP_CTRL,
"GNI_CqCreate returned %s\n",
gni_err_str[status]);
+ _gnix_dump_gni_res(domain->ptag);
ret = gnixu_to_fi_errno(status);
goto err1;
}
@@ -1079,6 +1110,7 @@ int gnix_nic_alloc(struct gnix_fid_domain *domain,
GNIX_WARN(FI_LOG_EP_CTRL,
"GNI_CqCreate returned %s\n",
gni_err_str[status]);
+ _gnix_dump_gni_res(domain->ptag);
ret = gnixu_to_fi_errno(status);
goto err1;
}
@@ -1095,6 +1127,7 @@ int gnix_nic_alloc(struct gnix_fid_domain *domain,
GNIX_WARN(FI_LOG_EP_CTRL,
"GNI_CqCreate returned %s\n",
gni_err_str[status]);
+ _gnix_dump_gni_res(domain->ptag);
ret = gnixu_to_fi_errno(status);
goto err1;
}
@@ -1113,6 +1146,15 @@ int gnix_nic_alloc(struct gnix_fid_domain *domain,
goto err1;
}
+ ret = _gnix_alloc_bitmap(&nic->vc_id_bitmap,
+ nic->vc_id_table_capacity);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "alloc_bitmap returned %d\n", ret);
+ goto err1;
+ }
+ fastlock_init(&nic->vc_id_lock);
+
fastlock_init(&nic->lock);
ret = __gnix_nic_tx_freelist_init(nic, domain->gni_tx_cq_size);
@@ -1127,13 +1169,6 @@ int gnix_nic_alloc(struct gnix_fid_domain *domain,
dlist_init(&nic->tx_vcs);
_gnix_ref_init(&nic->ref_cnt, 1, __nic_destruct);
- ret = _gnix_alloc_bitmap(&nic->vc_id_bitmap, 1000);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "alloc_bitmap returned %d\n", ret);
- goto err1;
- }
- fastlock_init(&nic->vc_id_lock);
smsg_mbox_attr.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
smsg_mbox_attr.mbox_maxcredit = domain->params.mbox_maxcredit;
@@ -1159,6 +1194,7 @@ int gnix_nic_alloc(struct gnix_fid_domain *domain,
(size_t)nic->mem_per_mbox,
domain->params.mbox_num_per_slab,
&nic->mbox_hndl);
+
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_mbox_alloc returned %s\n",
@@ -1230,10 +1266,20 @@ int gnix_nic_alloc(struct gnix_fid_domain *domain,
* though...
*/
- ret = _gnix_job_disable_affinity_apply();
+ ret = _gnix_get_num_corespec_cpus(&num_corespec_cpus);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "failed to get num corespec cpus\n");
+ }
+ if (num_corespec_cpus > 0) {
+ ret = _gnix_job_disable_affinity_apply();
+ } else {
+ ret = _gnix_job_enable_unassigned_cpus();
+ }
if (ret != 0)
GNIX_WARN(FI_LOG_EP_CTRL,
- "_gnix_job_disable call returned %d\n", ret);
+ "job_disable/unassigned cpus returned %d\n",
+ ret);
ret = pthread_create(&nic->progress_thread,
NULL,
@@ -1252,6 +1298,10 @@ int gnix_nic_alloc(struct gnix_fid_domain *domain,
GNIX_INFO(FI_LOG_EP_CTRL, "Allocated NIC:%p\n", nic);
}
+ if (nic) {
+ nic->requires_lock = domain->thread_model != FI_THREAD_COMPLETION;
+ }
+
*nic_ptr = nic;
goto out;
diff --git a/prov/gni/src/gnix_queue.c b/prov/gni/src/gnix_queue.c
index 2e49bc4..ac2ea72 100644
--- a/prov/gni/src/gnix_queue.c
+++ b/prov/gni/src/gnix_queue.c
@@ -129,5 +129,5 @@ void _gnix_queue_enqueue(struct gnix_queue *queue, struct slist_entry *item)
void _gnix_queue_enqueue_free(struct gnix_queue *queue,
struct slist_entry *item)
{
- gnix_slist_insert_tail(item, &queue->free_list);
+ slist_insert_head(item, &queue->free_list);
}
diff --git a/prov/gni/src/gnix_rma.c b/prov/gni/src/gnix_rma.c
index ca92c76..7fd4f5f 100644
--- a/prov/gni/src/gnix_rma.c
+++ b/prov/gni/src/gnix_rma.c
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC.
+ * All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -128,7 +129,7 @@ static void __gnix_rma_copy_indirect_get_data(struct gnix_tx_descriptor *txd)
int head_off = req->rma.rem_addr & GNI_READ_ALIGN_MASK;
memcpy((void *)req->rma.loc_addr,
- txd->int_buf + head_off,
+ (void *) ((uint8_t *) txd->int_buf + head_off),
req->rma.len);
}
@@ -146,25 +147,24 @@ static void __gnix_rma_copy_chained_get_data(struct gnix_tx_descriptor *txd)
GNIX_INFO(FI_LOG_EP_DATA, "writing %d bytes to %p\n",
head_len, req->rma.loc_addr);
memcpy((void *)req->rma.loc_addr,
- txd->int_buf + head_off,
+ (void *) ((uint8_t *) txd->int_buf + head_off),
head_len);
}
if (tail_len) {
- addr = (void *)req->rma.loc_addr +
- req->rma.len -
- tail_len;
+ addr = (void *) ((uint8_t *) req->rma.loc_addr +
+ req->rma.len -
+ tail_len);
GNIX_INFO(FI_LOG_EP_DATA, "writing %d bytes to %p\n",
tail_len, addr);
memcpy((void *)addr,
- txd->int_buf + GNI_READ_ALIGN,
+ (void *) ((uint8_t *) txd->int_buf + GNI_READ_ALIGN),
tail_len);
}
}
-static void __gnix_rma_fr_complete(struct gnix_fab_req *req,
- struct gnix_tx_descriptor *txd)
+static void __gnix_rma_fr_complete(struct gnix_fab_req *req)
{
if (req->flags & FI_LOCAL_MR) {
GNIX_INFO(FI_LOG_EP_DATA, "freeing auto-reg MR: %p\n",
@@ -173,7 +173,6 @@ static void __gnix_rma_fr_complete(struct gnix_fab_req *req,
}
atomic_dec(&req->vc->outstanding_tx_reqs);
- _gnix_nic_tx_free(req->vc->ep->nic, txd);
/* Schedule VC TX queue in case the VC is 'fenced'. */
_gnix_vc_tx_schedule(req->vc);
@@ -181,30 +180,35 @@ static void __gnix_rma_fr_complete(struct gnix_fab_req *req,
_gnix_fr_free(req->vc->ep, req);
}
-static int __gnix_rma_post_err(struct gnix_tx_descriptor *txd, int error)
+static int __gnix_rma_post_err_no_retrans(struct gnix_fab_req *req, int error)
{
- struct gnix_fab_req *req = txd->req;
int rc;
- req->tx_failures++;
- if (GNIX_EP_RDM(req->gnix_ep->type) &&
- req->tx_failures < req->gnix_ep->domain->params.max_retransmits) {
- _gnix_nic_tx_free(req->gnix_ep->nic, txd);
+ rc = __gnix_rma_send_err(req->vc->ep, req, error);
+ if (rc != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "__gnix_rma_send_err() failed: %d\n",
+ rc);
+ __gnix_rma_fr_complete(req);
+ return FI_SUCCESS;
+}
+
+static int __gnix_rma_post_err(struct gnix_fab_req *req, int error)
+{
+ if (GNIX_EP_RDM(req->gnix_ep->type) &&
+ _gnix_req_replayable(req)) {
+ req->tx_failures++;
GNIX_INFO(FI_LOG_EP_DATA,
"Requeueing failed request: %p\n", req);
return _gnix_vc_queue_work_req(req);
}
- GNIX_INFO(FI_LOG_EP_DATA, "Failed %d transmits: %p error: %d\n",
+ GNIX_WARN(FI_LOG_EP_DATA, "Failed %u transmits: %p error: %d\n",
req->tx_failures, req, error);
- rc = __gnix_rma_send_err(req->vc->ep, req, error);
- if (rc != FI_SUCCESS)
- GNIX_WARN(FI_LOG_EP_DATA,
- "__gnix_rma_send_err() failed: %d\n",
- rc);
- __gnix_rma_fr_complete(req, txd);
+ __gnix_rma_post_err_no_retrans(req, error);
+
return FI_SUCCESS;
}
@@ -223,6 +227,54 @@ static int __gnix_rma_post_irq_complete(void *arg, gni_return_t tx_status)
return FI_SUCCESS;
}
+/* SMSG callback for RMA data control message. */
+int __smsg_rma_data(void *data, void *msg)
+{
+ int ret = FI_SUCCESS;
+ struct gnix_vc *vc = (struct gnix_vc *)data;
+ struct gnix_smsg_rma_data_hdr *hdr =
+ (struct gnix_smsg_rma_data_hdr *)msg;
+ struct gnix_fid_ep *ep = vc->ep;
+ gni_return_t status;
+
+ if (hdr->flags & FI_REMOTE_CQ_DATA && ep->recv_cq) {
+ ret = _gnix_cq_add_event(ep->recv_cq, NULL, hdr->user_flags, 0,
+ 0, hdr->user_data, 0,
+ FI_ADDR_NOTAVAIL);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_cq_add_event returned %d\n",
+ ret);
+ }
+ }
+
+ if (hdr->flags & FI_REMOTE_WRITE && ep->rwrite_cntr) {
+ ret = _gnix_cntr_inc(ep->rwrite_cntr);
+ if (ret != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_cntr_inc() failed: %d\n",
+ ret);
+ }
+
+ if (hdr->flags & FI_REMOTE_READ && ep->rread_cntr) {
+ ret = _gnix_cntr_inc(ep->rread_cntr);
+ if (ret != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_cntr_inc() failed: %d\n",
+ ret);
+ }
+
+ status = GNI_SmsgRelease(vc->gni_ep);
+ if (unlikely(status != GNI_RC_SUCCESS)) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "GNI_SmsgRelease returned %s\n",
+ gni_err_str[status]);
+ ret = gnixu_to_fi_errno(status);
+ }
+
+ return ret;
+}
+
/* __gnix_rma_txd_data_complete() should match __gnix_rma_txd_complete() except
* for checking whether to send immediate data. */
static int __gnix_rma_txd_data_complete(void *arg, gni_return_t tx_status)
@@ -231,9 +283,10 @@ static int __gnix_rma_txd_data_complete(void *arg, gni_return_t tx_status)
struct gnix_fab_req *req = txd->req;
int rc;
- if (tx_status != GNI_RC_SUCCESS) {
- return __gnix_rma_post_err(txd, FI_ECANCELED);
- }
+ _gnix_nic_tx_free(req->gnix_ep->nic, txd);
+
+ if (tx_status != GNI_RC_SUCCESS)
+ return __gnix_rma_post_err(req, FI_ECANCELED);
/* Successful data delivery. Generate local completion. */
rc = __gnix_rma_send_completion(req->gnix_ep, req);
@@ -242,7 +295,7 @@ static int __gnix_rma_txd_data_complete(void *arg, gni_return_t tx_status)
"__gnix_rma_send_completion() failed: %d\n",
rc);
- __gnix_rma_fr_complete(req, txd);
+ __gnix_rma_fr_complete(req);
return FI_SUCCESS;
}
@@ -267,16 +320,28 @@ static int __gnix_rma_send_data_req(void *arg)
txd->req = req;
txd->completer_fn = __gnix_rma_txd_data_complete;
+ txd->rma_data_hdr.flags = 0;
- txd->rma_data_hdr.flags = FI_RMA | FI_REMOTE_CQ_DATA;
- if (req->type == GNIX_FAB_RQ_RDMA_WRITE) {
- txd->rma_data_hdr.flags |= FI_REMOTE_WRITE;
- } else {
- txd->rma_data_hdr.flags |= FI_REMOTE_READ;
+ if (req->flags & FI_REMOTE_CQ_DATA) {
+ txd->rma_data_hdr.flags |= FI_REMOTE_CQ_DATA;
+ txd->rma_data_hdr.user_flags = FI_RMA | FI_REMOTE_CQ_DATA;
+ if (req->type == GNIX_FAB_RQ_RDMA_WRITE) {
+ txd->rma_data_hdr.user_flags |= FI_REMOTE_WRITE;
+ } else {
+ txd->rma_data_hdr.user_flags |= FI_REMOTE_READ;
+ }
+ txd->rma_data_hdr.user_data = req->rma.imm;
}
- txd->rma_data_hdr.data = req->rma.imm;
- fastlock_acquire(&nic->lock);
+ if (req->vc->peer_caps & FI_RMA_EVENT) {
+ if (req->type == GNIX_FAB_RQ_RDMA_WRITE) {
+ txd->rma_data_hdr.flags |= FI_REMOTE_WRITE;
+ } else {
+ txd->rma_data_hdr.flags |= FI_REMOTE_READ;
+ }
+ }
+
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
if (inject_err) {
_gnix_nic_txd_err_inject(nic, txd);
status = GNI_RC_SUCCESS;
@@ -287,7 +352,7 @@ static int __gnix_rma_send_data_req(void *arg)
NULL, 0, txd->id,
GNIX_SMSG_T_RMA_DATA);
}
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
if (status == GNI_RC_NOT_DONE) {
_gnix_nic_tx_free(nic, txd);
@@ -338,24 +403,27 @@ static int __gnix_rma_txd_complete(void *arg, gni_return_t tx_status)
}
if (tx_status != GNI_RC_SUCCESS) {
- GNIX_INFO(FI_LOG_EP_DATA, "calling __gnix_rma_post_err\n");
- return __gnix_rma_post_err(txd, FI_ECANCELED);
+ _gnix_nic_tx_free(req->gnix_ep->nic, txd);
+ return __gnix_rma_post_err(req, FI_ECANCELED);
}
/* Successful delivery. Progress request. */
- if (req->flags & FI_REMOTE_CQ_DATA) {
- /* initiate immediate data transfer */
+ if (req->flags & GNIX_RMA_INDIRECT) {
+ __gnix_rma_copy_indirect_get_data(txd);
+ } else if (req->flags & GNIX_RMA_CHAINED &&
+ !(req->flags & GNIX_RMA_RDMA)) {
+ __gnix_rma_copy_chained_get_data(txd);
+ }
+
+ _gnix_nic_tx_free(req->gnix_ep->nic, txd);
+
+ if (req->flags & FI_REMOTE_CQ_DATA ||
+ req->vc->peer_caps & FI_RMA_EVENT) {
+ /* control message needed for imm. data or a counter event. */
req->tx_failures = 0;
req->work_fn = __gnix_rma_send_data_req;
_gnix_vc_queue_work_req(req);
} else {
- if (req->flags & GNIX_RMA_INDIRECT) {
- __gnix_rma_copy_indirect_get_data(txd);
- } else if (req->flags & GNIX_RMA_CHAINED &&
- !(req->flags & GNIX_RMA_RDMA)) {
- __gnix_rma_copy_chained_get_data(txd);
- }
-
/* complete request */
rc = __gnix_rma_send_completion(req->vc->ep, req);
if (rc != FI_SUCCESS)
@@ -363,7 +431,7 @@ static int __gnix_rma_txd_complete(void *arg, gni_return_t tx_status)
"__gnix_rma_send_completion() failed: %d\n",
rc);
- __gnix_rma_fr_complete(req, txd);
+ __gnix_rma_fr_complete(req);
}
return FI_SUCCESS;
@@ -519,11 +587,12 @@ int _gnix_rma_post_rdma_chain_req(void *data)
int fma_chain = 0;
if (!gnix_ops_allowed(ep, req->vc->peer_caps, req->flags)) {
- rc = __gnix_rma_send_err(req->vc->ep, req, FI_EOPNOTSUPP);
+ rc = __gnix_rma_post_err_no_retrans(req, FI_EOPNOTSUPP);
if (rc != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
- "__gnix_rma_send_err() failed: %d\n", rc);
- return FI_SUCCESS;
+ "__gnix_rma_post_err_no_retrans() failed: %d\n",
+ rc);
+ return -FI_ECANCELED;
}
rc = _gnix_nic_tx_alloc(nic, &bte_txd);
@@ -613,7 +682,7 @@ int _gnix_rma_post_rdma_chain_req(void *data)
GNIX_LOG_DUMP_TXD(ct_txd);
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
/*
* TODO: need work here too!
@@ -628,7 +697,7 @@ int _gnix_rma_post_rdma_chain_req(void *data)
}
if (status != GNI_RC_SUCCESS) {
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
_gnix_nic_tx_free(nic, ct_txd);
_gnix_nic_tx_free(nic, bte_txd);
@@ -649,7 +718,7 @@ int _gnix_rma_post_rdma_chain_req(void *data)
}
if (status != GNI_RC_SUCCESS) {
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
_gnix_nic_tx_free(nic, ct_txd);
/* Wait for the first TX to complete, then retransmit the
@@ -662,7 +731,7 @@ int _gnix_rma_post_rdma_chain_req(void *data)
return FI_SUCCESS;
}
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
/* Wait for both TXs to complete, then process the request. */
atomic_set(&req->rma.outstanding_txds, 2);
@@ -686,6 +755,15 @@ int _gnix_rma_post_req(void *data)
int chained = !!(fab_req->flags & GNIX_RMA_CHAINED);
int inject_err = _gnix_req_inject_err(fab_req);
+ if (!gnix_ops_allowed(ep, fab_req->vc->peer_caps, fab_req->flags)) {
+ rc = __gnix_rma_post_err_no_retrans(fab_req, FI_EOPNOTSUPP);
+ if (rc != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "__gnix_rma_post_err_no_retrans() failed: %d\n",
+ rc);
+ return -FI_ECANCELED;
+ }
+
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA,
@@ -697,10 +775,6 @@ int _gnix_rma_post_req(void *data)
txd->completer_fn = __gnix_rma_txd_complete;
txd->req = fab_req;
- if (!gnix_ops_allowed(ep, fab_req->vc->peer_caps, fab_req->flags)) {
- return __gnix_rma_post_err(txd, FI_EOPNOTSUPP);
- }
-
if (rdma) {
_gnix_convert_key_to_mhdl(
(gnix_mr_key_t *)&fab_req->rma.rem_mr_key,
@@ -738,7 +812,7 @@ int _gnix_rma_post_req(void *data)
GNIX_LOG_DUMP_TXD(txd);
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
if (unlikely(inject_err)) {
_gnix_nic_txd_err_inject(nic, txd);
@@ -751,11 +825,11 @@ int _gnix_rma_post_req(void *data)
status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc);
}
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
- GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
+ GNIX_WARN(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n",
gni_err_str[status]);
}
@@ -808,12 +882,19 @@ ssize_t _gnix_rma(struct gnix_fid_ep *ep, enum gnix_fab_req_type fr_type,
int rdma;
struct fid_mr *auto_mr = NULL;
- if (!ep) {
- return -FI_EINVAL;
+ if (!(flags & FI_INJECT) && !ep->send_cq &&
+ (((fr_type == GNIX_FAB_RQ_RDMA_WRITE) && !ep->write_cntr) ||
+ ((fr_type == GNIX_FAB_RQ_RDMA_READ) && !ep->read_cntr))) {
+ return -FI_ENOCQ;
}
- if (!ep->send_cq && !(flags & FI_INJECT)) {
- return -FI_ENOCQ;
+ if (flags & FI_TRIGGER) {
+ struct fi_triggered_context *trigger_context =
+ (struct fi_triggered_context *)context;
+ if ((trigger_context->event_type != FI_TRIGGER_THRESHOLD) ||
+ (flags & FI_INJECT)) {
+ return -FI_EINVAL;
+ }
}
if ((flags & FI_INJECT) && (len > GNIX_INJECT_SIZE)) {
@@ -914,7 +995,7 @@ ssize_t _gnix_rma(struct gnix_fid_ep *ep, enum gnix_fab_req_type fr_type,
req->flags |= GNIX_RMA_RDMA;
}
- GNIX_INFO(FI_LOG_EP_DATA, "Queuing (%p %p %d)\n",
+ GNIX_DEBUG(FI_LOG_EP_DATA, "Queuing (%p %p %d)\n",
(void *)loc_addr, (void *)rem_addr, len);
return _gnix_vc_queue_tx_req(req);
diff --git a/prov/gni/src/gnix_tags.c b/prov/gni/src/gnix_tags.c
index c908b41..c0e3c0a 100644
--- a/prov/gni/src/gnix_tags.c
+++ b/prov/gni/src/gnix_tags.c
@@ -608,7 +608,9 @@ static int __gnix_tag_list_insert_tag(
struct gnix_tag_list_element *element;
element = &req->msg.tle;
- element->free.next = NULL;
+ if (!dlist_empty(&element->free))
+ return -FI_EALREADY;
+
element->context = NULL;
dlist_insert_tail(&element->free, &ts->list.list);
@@ -651,7 +653,6 @@ static struct gnix_fab_req *__gnix_tag_list_remove_tag(
return NULL;
req = __to_gnix_fab_req(element);
- element->free.next = NULL;
return req;
}
@@ -666,7 +667,6 @@ static void __gnix_tag_list_remove_tag_by_req(
element = &req->msg.tle;
item = (struct dlist_entry *) &element->free;
dlist_remove(item);
- element->free.next = NULL;
}
static struct gnix_fab_req *__gnix_tag_list_remove_req_by_context(
@@ -684,7 +684,6 @@ static struct gnix_fab_req *__gnix_tag_list_remove_req_by_context(
return NULL;
req = __to_gnix_fab_req(element);
- element->free.next = NULL;
return req;
}
@@ -746,6 +745,9 @@ static int __gnix_tag_hlist_insert_tag(
int bucket = get_bucket(ts, tag);
element = &req->msg.tle;
+ if (!dlist_empty(&element->free))
+ return -FI_EALREADY;
+
dlist_init(&element->free);
element->context = NULL;
element->seq = ++ts->hlist.last_inserted_id;
@@ -836,7 +838,7 @@ int _gnix_insert_tag(
{
int ret;
- GNIX_INFO(FI_LOG_EP_CTRL, "inserting a message by tag, "
+ GNIX_DEBUG(FI_LOG_EP_CTRL, "inserting a message by tag, "
"ts=%p tag=%llx req=%p\n", ts, tag, req);
req->msg.tag = tag;
if (ts->match_func == _gnix_match_posted_tag) {
@@ -845,7 +847,7 @@ int _gnix_insert_tag(
ret = ts->ops->insert_tag(ts, tag, req);
- GNIX_INFO(FI_LOG_EP_CTRL, "ret=%i\n", ret);
+ GNIX_DEBUG(FI_LOG_EP_CTRL, "ret=%i\n", ret);
return ret;
}
@@ -864,12 +866,12 @@ static struct gnix_fab_req *__remove_by_tag_and_addr(
struct gnix_fab_req *ret;
/* assuming that flags and context are correct */
- GNIX_INFO(FI_LOG_EP_CTRL, "removing a message by tag, "
+ GNIX_DEBUG(FI_LOG_EP_CTRL, "removing a message by tag, "
"ts=%p tag=%llx ignore=%llx flags=%llx context=%p "
"addr=%p\n",
ts, tag, ignore, flags, context, addr);
ret = ts->ops->remove_tag(ts, tag, ignore, flags, context, addr);
- GNIX_INFO(FI_LOG_EP_CTRL, "ret=%p\n", ret);
+ GNIX_DEBUG(FI_LOG_EP_CTRL, "ret=%p\n", ret);
return ret;
}
@@ -885,7 +887,7 @@ static struct gnix_fab_req *__peek_by_tag_and_addr(
struct gnix_fab_req *ret;
/* assuming that flags and context are correct */
- GNIX_INFO(FI_LOG_EP_CTRL, "peeking a message by tag, "
+ GNIX_DEBUG(FI_LOG_EP_CTRL, "peeking a message by tag, "
"ts=%p tag=%llx ignore=%llx flags=%llx context=%p "
"addr=%p\n",
ts, tag, ignore, flags, context, addr);
@@ -896,7 +898,7 @@ static struct gnix_fab_req *__peek_by_tag_and_addr(
ret->msg.tle.context = context;
}
- GNIX_INFO(FI_LOG_EP_CTRL, "ret=%p\n", ret);
+ GNIX_DEBUG(FI_LOG_EP_CTRL, "ret=%p\n", ret);
return ret;
}
diff --git a/prov/gni/src/gnix_trigger.c b/prov/gni/src/gnix_trigger.c
new file mode 100644
index 0000000..2e2c223
--- /dev/null
+++ b/prov/gni/src/gnix_trigger.c
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2016 Cray Inc. All rights reserved.
+ * Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved.
+ *
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*
+ * Triggered operations handling.
+ */
+
+#include "gnix_trigger.h"
+#include "gnix_vc.h"
+#include "gnix.h"
+
+int _gnix_trigger_queue_req(struct gnix_fab_req *req)
+{
+ struct fi_triggered_context *trigger_context;
+ struct fi_trigger_threshold *threshold;
+ struct gnix_fid_cntr *cntr;
+ struct gnix_fab_req *r;
+ size_t req_thresh;
+
+ trigger_context = (struct fi_triggered_context *)
+ req->user_context;
+ threshold = &trigger_context->trigger.threshold;
+ cntr = container_of(threshold->cntr, struct gnix_fid_cntr, cntr_fid);
+
+ if (atomic_get(&cntr->cnt) >= threshold->threshold) {
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "Trigger condition met: %p\n",
+ req);
+
+ /* Trigger condition has already been met. */
+ return 1;
+ }
+
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "Queueing triggered op: %p\n",
+ req);
+
+ fastlock_acquire(&cntr->trigger_lock);
+ if (dlist_empty(&cntr->trigger_list)) {
+ dlist_init(&req->dlist);
+ dlist_insert_head(&req->dlist, &cntr->trigger_list);
+ } else {
+ req_thresh = threshold->threshold;
+
+ dlist_for_each(&cntr->trigger_list, r, dlist) {
+ trigger_context = (struct fi_triggered_context *)
+ r->user_context;
+ threshold = &trigger_context->trigger.threshold;
+
+ /* Insert new req. after those with equal threshold and
+ * before those with greater threshold. */
+ if (req_thresh < threshold->threshold) {
+ break;
+ }
+ }
+
+ dlist_init(&req->dlist);
+ dlist_insert_before(&req->dlist, &r->dlist);
+ }
+ fastlock_release(&cntr->trigger_lock);
+
+ return FI_SUCCESS;
+}
+
+void _gnix_trigger_check_cntr(struct gnix_fid_cntr *cntr)
+{
+ struct fi_triggered_context *trigger_context;
+ struct fi_trigger_threshold *threshold;
+ struct gnix_fab_req *req, *req2;
+ size_t count;
+
+ if (likely(dlist_empty(&cntr->trigger_list))) {
+ return;
+ }
+
+ count = atomic_get(&cntr->cnt);
+
+ fastlock_acquire(&cntr->trigger_lock);
+ dlist_for_each_safe(&cntr->trigger_list, req, req2, dlist) {
+ trigger_context = (struct fi_triggered_context *)
+ req->user_context;
+ threshold = &trigger_context->trigger.threshold;
+
+ if (count >= threshold->threshold) {
+ GNIX_INFO(FI_LOG_EP_DATA,
+ "Trigger condition met: %p\n",
+ req);
+
+ dlist_remove_init(&req->dlist);
+ req->flags &= ~FI_TRIGGER;
+ _gnix_vc_queue_tx_req(req);
+ } else {
+ break;
+ }
+ }
+ fastlock_release(&cntr->trigger_lock);
+}
diff --git a/prov/gni/src/gnix_util.c b/prov/gni/src/gnix_util.c
index 475d064..8782003 100644
--- a/prov/gni/src/gnix_util.c
+++ b/prov/gni/src/gnix_util.c
@@ -1,7 +1,7 @@
/*
* Copyright (c) 2014 Intel Corporation, Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -63,32 +63,47 @@
#include "gnix.h"
#include "gnix_util.h"
-static int alps_init;
-static uint64_t gnix_apid;
-static alpsAppLayout_t gnix_appLayout;
+static bool app_init;
+/* Filled in by __gnix_app_init */
static uint8_t gnix_app_ptag;
static uint32_t gnix_app_cookie;
+static uint32_t gnix_pes_on_node;
+/* CCM/ccmlogin specific stuff */
+static bool ccm_init;
+/* This file provides ccm_alps_info */
+#define CCM_ALPS_INFO_FILE "/tmp/ccm_alps_info"
+typedef struct ccm_alps_info {
+ uint32_t version;
+ uint8_t ptag;
+ uint32_t cookie;
+} ccm_alps_info_t;
+/* Format for the nodelist filename: $HOME/.crayccm/ccmnodlist.<WLM jobid> */
+#define CCM_NODELIST_FN ".crayccm/ccm_nodelist."
+/* alps specific stuff */
+static uint64_t gnix_apid;
+static alpsAppLayout_t gnix_appLayout;
static uint32_t gnix_device_id;
-
+/* These are not used currently and could be static to gnix_alps_init */
+static int alps_init;
static int *gnix_app_placementList;
static int *gnix_app_targetNids;
static int *gnix_app_targetPes;
static int *gnix_app_targetLen;
-struct in_addr *gnix_app_targetIps;
+static struct in_addr *gnix_app_targetIps;
static int *gnix_app_startPe;
static int *gnix_app_totalPes;
static int *gnix_app_nodePes;
static int *gnix_app_peCpus;
-void _gnix_alps_cleanup(void)
+
+static inline void __gnix_ccm_cleanup(void)
{
- alps_app_lli_lock();
+ ccm_init = false;
+}
- if (alps_init) {
- /* alps lli lock protects alps_init for now */
- alps_app_lli_unlock();
- return;
- }
+static inline void __gnix_alps_cleanup(void)
+{
+ alps_app_lli_lock();
if (gnix_app_placementList)
free(gnix_app_placementList);
@@ -109,12 +124,113 @@ void _gnix_alps_cleanup(void)
if (gnix_app_peCpus)
free(gnix_app_peCpus);
- alps_init = 0;
+ alps_init = false;
alps_app_lli_unlock();
}
-static int gnix_alps_init(void)
+void _gnix_app_cleanup(void)
+{
+ if (alps_init) {
+ __gnix_alps_cleanup();
+ } else if (ccm_init) {
+ __gnix_ccm_cleanup();
+ }
+}
+
+/* There are two types of errors that can happen in this function:
+ * - CCM ALPS info file not found
+ * - Failure while trying to get ptag, cookie and PEs/node
+ * Currently we don't distinguish between the two.
+ */
+static int __gnix_ccm_init(void)
+{
+ int rc, fd;
+ FILE *f;
+ char ccm_nodelist[PATH_MAX];
+ const char *home, *jobid;
+ ccm_alps_info_t info;
+ uint32_t num_nids = 0;
+
+ GNIX_DEBUG(FI_LOG_FABRIC, "Reading job info file %s\n",
+ CCM_ALPS_INFO_FILE);
+
+ fd = open(CCM_ALPS_INFO_FILE, O_RDONLY);
+ if (fd < 0) {
+ return -FI_EIO;
+ }
+
+ rc = read(fd, &info, sizeof(ccm_alps_info_t));
+ if (rc != sizeof(ccm_alps_info_t))
+ return -FI_EIO;
+
+ gnix_app_ptag = info.ptag;
+ gnix_app_cookie = info.cookie;
+
+ close(fd);
+ GNIX_DEBUG(FI_LOG_FABRIC, "Ptag=0x%x, cookie=0x%x\n",
+ gnix_app_ptag, gnix_app_cookie);
+
+ home = getenv("HOME");
+ jobid = getenv("PBS_JOBID");
+ if (!jobid) {
+ /* This path hasn't been tested yet (no access to resources) */
+ jobid = getenv("SLURM_JOB_ID");
+ if (!jobid) {
+ jobid = getenv("SLURM_JOBID");
+ }
+ /* Should we do something else if this fails? */
+ }
+ snprintf(ccm_nodelist, PATH_MAX, "%s/%s%s", home ? home : ".",
+ CCM_NODELIST_FN, jobid ? jobid : "sdb");
+ f = fopen(ccm_nodelist, "r");
+ if (f) {
+ char mynid[PATH_MAX];
+ char next_nid[PATH_MAX];
+
+ rc = gethostname(mynid, PATH_MAX);
+ if (rc) {
+ /* use the first address */
+ rc = fscanf(f, "%s\n", mynid);
+ /* assume this one worked, error case is same */
+ num_nids++;
+ }
+ while (true) {
+ rc = fscanf(f, "%s\n", next_nid);
+ if (rc == 1) {
+ if (strcmp(mynid, next_nid) == 0) {
+ num_nids++;
+ }
+ } else {
+ break;
+ }
+ }
+ gnix_pes_on_node = num_nids;
+ fclose(f);
+ } else {
+ /* what would be a better default? */
+ GNIX_WARN(FI_LOG_FABRIC,
+ "CCM nodelist not found. Assuming 1 PE per node");
+ gnix_pes_on_node = 1;
+ }
+ GNIX_DEBUG(FI_LOG_FABRIC, "pes per node=%u\n", gnix_pes_on_node);
+
+ /* Don't really need to do this here, but wanted to be clear */
+ gnix_app_placementList = NULL;
+ gnix_app_targetNids = NULL;
+ gnix_app_targetPes = NULL;
+ gnix_app_targetLen = NULL;
+ gnix_app_targetIps = NULL;
+ gnix_app_startPe = NULL;
+ gnix_app_totalPes = NULL;
+ gnix_app_nodePes = NULL;
+ gnix_app_peCpus = NULL;
+
+ ccm_init = true;
+ return FI_SUCCESS;
+}
+
+static int __gnix_alps_init(void)
{
int ret = FI_SUCCESS;
int alps_status = 0;
@@ -230,8 +346,9 @@ static int gnix_alps_init(void)
goto err;
}
- gnix_device_id = 0;
- alps_init++;
+ gnix_pes_on_node = gnix_appLayout.numPesHere;
+
+ alps_init = true;
ret = 0;
err:
@@ -243,34 +360,27 @@ err:
return ret;
}
-typedef struct ccm_alps_info {
- uint32_t version;
- uint8_t ptag;
- uint32_t cookie;
-} ccm_alps_info_t;
-
-int get_ccm_ptag_cookie(char *filename)
+static int __gnix_app_init(void)
{
- int rc, fd;
- ccm_alps_info_t info;
- GNIX_INFO(FI_LOG_FABRIC, "Reading job info file %s", filename);
+ int ret;
- fd = open(filename, O_RDONLY);
- if (fd < 0)
- return -FI_EIO;
+ if (app_init) {
+ return FI_SUCCESS;
+ }
- rc = read(fd, &info, sizeof(ccm_alps_info_t));
- if (rc != sizeof(ccm_alps_info_t))
- return -FI_EIO;
+ /* Try CCM first */
+ ret = __gnix_ccm_init();
+ if (ret) {
+ ret = __gnix_alps_init();
+ }
- gnix_app_ptag = info.ptag;
- gnix_app_cookie = info.cookie;
+ if (ret == FI_SUCCESS) {
+ app_init = true;
+ }
- close(fd);
- GNIX_INFO(FI_LOG_FABRIC, "Ptag=0x%x, cookie=0x%x",
- gnix_app_ptag, gnix_app_cookie);
+ gnix_device_id = 0;
+ return ret;
- return FI_SUCCESS;
}
int gnixu_get_rdma_credentials(void *addr, uint8_t *ptag, uint32_t *cookie)
@@ -280,15 +390,13 @@ int gnixu_get_rdma_credentials(void *addr, uint8_t *ptag, uint32_t *cookie)
if ((ptag == NULL) || (cookie == NULL)) {
return -FI_EINVAL;
}
- ret = get_ccm_ptag_cookie("/tmp/ccm_alps_info");
+
+ ret = __gnix_app_init();
if (ret) {
- ret = gnix_alps_init();
- if (ret) {
- GNIX_WARN(FI_LOG_FABRIC,
- "gnix_alps_init() failed, ret=%d(%s)\n",
- ret, strerror(errno));
- return ret;
- }
+ GNIX_WARN(FI_LOG_FABRIC,
+ "__gnix_app_init() failed, ret=%d(%s)\n",
+ ret, strerror(errno));
+ return ret;
}
/*
@@ -390,14 +498,14 @@ static int gnix_write_proc_job(char *val_str)
* assigned to the app and not assigned to corespec. */
int _gnix_job_enable_unassigned_cpus(void)
{
- return gnix_write_proc_job("enable_unassigned_cpus");
+ return gnix_write_proc_job("enable_affinity_unassigned_cpus");
}
/* Indicate that the next task spawned will be restricted to CPUs that are
* assigned to the app. */
int _gnix_job_disable_unassigned_cpus(void)
{
- return gnix_write_proc_job("disable_unassigned_cpus");
+ return gnix_write_proc_job("disable_affinity_unassigned_cpus");
}
/* Indicate that the next task spawned should adhere to the the affinity rules. */
@@ -468,15 +576,15 @@ int _gnix_pes_on_node(uint32_t *num_pes)
return -FI_EINVAL;
}
- rc = gnix_alps_init();
+ rc = __gnix_app_init();
if (rc) {
GNIX_WARN(FI_LOG_FABRIC,
- "gnix_alps_init() failed, ret=%d(%s)\n",
+ "__gnix_app_init() failed, ret=%d(%s)\n",
rc, strerror(errno));
return rc;
}
- *num_pes = gnix_appLayout.numPesHere;
+ *num_pes = gnix_pes_on_node;
GNIX_INFO(FI_LOG_FABRIC, "num_pes: %u\n", gnix_appLayout.numPesHere);
return FI_SUCCESS;
@@ -491,10 +599,10 @@ int _gnix_nics_per_rank(uint32_t *nics_per_rank)
return -FI_EINVAL;
}
- rc = gnix_alps_init();
+ rc = __gnix_app_init();
if (rc) {
GNIX_WARN(FI_LOG_FABRIC,
- "gnix_alps_init() failed, ret=%d(%s)\n",
+ "__gnix_app_init() failed, ret=%d(%s)\n",
rc, strerror(errno));
return rc;
}
@@ -522,3 +630,100 @@ int _gnix_nics_per_rank(uint32_t *nics_per_rank)
return FI_SUCCESS;
}
+void _gnix_dump_gni_res(uint8_t ptag)
+{
+ int i;
+ gni_return_t status;
+ gni_dev_res_desc_t dev_res_desc;
+ gni_job_res_desc_t job_res_desc;
+#define BUF_SZ 4096
+ char buf[BUF_SZ];
+ int size = BUF_SZ, written = 0;
+
+ if (!fi_log_enabled(&gnix_prov, FI_LOG_WARN, FI_LOG_FABRIC))
+ return;
+
+ written += snprintf(buf + written, size - written,
+ "Device Resources:\n");
+ for (i = GNI_DEV_RES_FIRST+1; i < GNI_DEV_RES_LAST; i++) {
+ status = GNI_GetDevResInfo(0, i, &dev_res_desc);
+ if (status == GNI_RC_SUCCESS) {
+ written += snprintf(buf + written, size - written,
+ "dev res: %9s, avail: %lu res: %lu held: %lu total: %lu\n",
+ gni_dev_res_to_str(i),
+ dev_res_desc.available,
+ dev_res_desc.reserved,
+ dev_res_desc.held,
+ dev_res_desc.total);
+ GNIX_WARN(FI_LOG_FABRIC, "%s", buf);
+ }
+ }
+
+ written = 0;
+ written += snprintf(buf + written, size - written,
+ "Job Resources:\n");
+ for (i = GNI_JOB_RES_FIRST+1; i < GNI_JOB_RES_LAST; i++) {
+ status = GNI_GetJobResInfo(0, ptag, i, &job_res_desc);
+ if (status == GNI_RC_SUCCESS) {
+ written += snprintf(buf + written, size - written,
+ "ptag[%d] job res: %9s used: %lu limit: %lu\n",
+ ptag, gni_job_res_to_str(i),
+ job_res_desc.used,
+ job_res_desc.limit);
+ GNIX_WARN(FI_LOG_FABRIC, "%s", buf);
+ }
+ }
+}
+
+int _gnix_get_num_corespec_cpus(uint32_t *num_core_spec_cpus)
+{
+ int ret = -FI_ENODATA;
+ int ncpus = 0;
+ FILE *fd = NULL;
+ char buffer[4096], *line, *field;
+ static bool already_called;
+ static uint32_t cached_num_corespec_cpus;
+
+ if (num_core_spec_cpus == NULL)
+ return -FI_EINVAL;
+
+ if (already_called == true) {
+ *num_core_spec_cpus = cached_num_corespec_cpus;
+ return FI_SUCCESS;
+ }
+
+ fd = fopen("/proc/job", "r");
+ if (!fd) {
+ GNIX_WARN(FI_LOG_FABRIC,
+ "open of /proc/job returned %s", strerror(errno));
+ return -errno;
+ }
+
+ while (1) {
+ line = fgets(buffer, sizeof(buffer), fd);
+ if (!line)
+ break;
+
+ line = strstr(line, "corespec");
+ if (line != NULL) {
+ field = strtok(line, " ");
+ field = strtok(NULL, " ");
+ if (!strcmp(field, "num_sys_cpus")) {
+ field = strtok(NULL, " ");
+ ncpus = atoi(field);
+ }
+ ret = FI_SUCCESS;
+ break;
+ }
+ }
+
+ *num_core_spec_cpus = ncpus;
+ cached_num_corespec_cpus = ncpus;
+
+ already_called = true;
+
+ fclose(fd);
+
+ return ret;
+}
+
diff --git a/prov/gni/src/gnix_vc.c b/prov/gni/src/gnix_vc.c
index 2de4d1a..e908b9d 100644
--- a/prov/gni/src/gnix_vc.c
+++ b/prov/gni/src/gnix_vc.c
@@ -50,6 +50,9 @@
#include "gnix_mbox_allocator.h"
#include "gnix_hashtable.h"
#include "gnix_av.h"
+#include "gnix_trigger.h"
+#include "gnix_vector.h"
+#include "gnix_xpmem.h"
/*
* forward declarations and local struct defs.
@@ -61,6 +64,7 @@ struct wq_hndl_conn_req {
struct gnix_vc *vc;
uint64_t src_vc_ptr;
gni_mem_handle_t irq_mem_hndl;
+ xpmem_segid_t peer_segid;
};
static int __gnix_vc_conn_ack_prog_fn(void *data, int *complete_ptr);
@@ -71,6 +75,238 @@ static int __gnix_vc_push_tx_reqs(struct gnix_vc *vc);
* Helper functions
******************************************************************************/
+/**
+ * Set key to the given gnix_addr.
+ *
+ * NOTE: If struct gnix_address is ever bit packed or packed by
+ * the compiler this assignment may not set key to the correct
+ * bytes.
+ */
+static inline void __gnix_vc_set_ht_key(void *gnix_addr,
+ gnix_ht_key_t *key)
+{
+ *key = *((gnix_ht_key_t *)gnix_addr);
+}
+
+static struct gnix_vc *_gnix_ep_vc_lookup(struct gnix_fid_ep *ep, uint64_t key)
+{
+ struct gnix_vc *vc = NULL;
+ int ret;
+ int i;
+
+ assert(ep->av);
+
+
+ for (i = 0; i < GNIX_ADDR_CACHE_SIZE; i++)
+ {
+ if (ep->addr_cache[i].addr == key && ep->addr_cache[i].vc != NULL)
+ return ep->addr_cache[i].vc;
+ }
+
+ if (ep->av->type == FI_AV_TABLE) {
+ ret = _gnix_vec_at(ep->vc_table, (void **)&vc, key);
+ if (ret != FI_SUCCESS) {
+ vc = NULL;
+ }
+ } else {
+ vc = (struct gnix_vc *)_gnix_ht_lookup(ep->vc_ht, key);
+ }
+
+ if (vc) {
+ ep->addr_cache[ep->last_cached].addr = key;
+ ep->addr_cache[ep->last_cached].vc = vc;
+ ep->last_cached = (ep->last_cached + 1) % 5;
+ }
+
+ return vc;
+}
+
+static int _gnix_ep_vc_store(struct gnix_fid_ep *ep, struct gnix_vc *vc,
+ uint64_t key)
+{
+ int ret;
+
+ assert(ep->av);
+
+ if (ep->av->type == FI_AV_TABLE) {
+ ret = _gnix_vec_insert_at(ep->vc_table, (void *)vc, key);
+ } else {
+ ret = _gnix_ht_insert(ep->vc_ht, key, vc);
+ }
+
+ return ret;
+}
+
+static int __gnix_vc_gnix_addr_equal(struct dlist_entry *item, const void *arg)
+{
+ struct gnix_vc *vc = dlist_entry(item, struct gnix_vc, list);
+
+ return GNIX_ADDR_EQUAL(vc->peer_addr, *(struct gnix_address *)arg);
+}
+
+/* Find an unmapped VC that matches 'dest_addr' and map it into the EP's VC
+ * look up table. VC lock must be held. */
+static struct gnix_vc *__gnix_vc_lookup_unmapped(struct gnix_fid_ep *ep,
+ fi_addr_t dest_addr)
+{
+ struct gnix_av_addr_entry av_entry;
+ struct dlist_entry *entry;
+ struct gnix_vc *vc;
+ int ret;
+
+ /* Determine if the fi_addr now exists in the AV. */
+ ret = _gnix_av_lookup(ep->av, dest_addr, &av_entry);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_av_lookup for addr 0x%lx returned %s\n",
+ dest_addr, fi_strerror(-ret));
+ return NULL;
+ }
+
+ /* Find a pre-existing, unmapped VC that matches the gnix_address
+ * mapped by dest_addr. */
+ entry = dlist_remove_first_match(&ep->unmapped_vcs,
+ __gnix_vc_gnix_addr_equal,
+ (void *)&av_entry.gnix_addr);
+ if (entry) {
+ /* Found a matching, unmapped VC. Map dest_addr to the VC in
+ * the EP's VC look up table. */
+ vc = dlist_entry(entry, struct gnix_vc, list);
+ GNIX_INFO(FI_LOG_EP_CTRL,
+ "Found unmapped VC: %p gnix_addr: 0x%lx fi_addr: 0x%lx\n",
+ vc, vc->peer_addr, vc->peer_fi_addr);
+
+ ret = _gnix_ep_vc_store(ep, vc, dest_addr);
+ if (unlikely(ret != FI_SUCCESS)) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_ep_vc_store returned %s\n",
+ fi_strerror(-ret));
+ dlist_insert_tail(&vc->list, &ep->unmapped_vcs);
+ return NULL;
+ }
+
+ return vc;
+ }
+
+ return NULL;
+}
+
+/**
+ * Look up the vc by fi_addr_t, if it's found just return it,
+ * otherwise allocate a new vc, insert it into the hashtable,
+ * and vector for FI_AV_TABLE AV type, and start connection setup.
+ *
+ * assumptions: ep is non-null;
+ * dest_addr is valid;
+ * vc_ptr is non-null.
+ */
+static int __gnix_vc_get_vc_by_fi_addr(struct gnix_fid_ep *ep, fi_addr_t dest_addr,
+ struct gnix_vc **vc_ptr)
+{
+ struct gnix_fid_av *av;
+ int ret = FI_SUCCESS;
+ struct gnix_av_addr_entry av_entry;
+ struct gnix_vc *vc;
+
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+
+ GNIX_DEBUG(FI_LOG_EP_CTRL,
+ "ep->vc_table = %p, ep->vc_table->vector = %p\n",
+ ep->vc_table, ep->vc_table->vector);
+
+ av = ep->av;
+ if (unlikely(av == NULL)) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "av field NULL for ep %p\n", ep);
+ return -FI_EINVAL;
+ }
+
+ COND_ACQUIRE(ep->requires_lock, &ep->vc_lock);
+
+ /* Use FI address to lookup in EP VC table. */
+ vc = _gnix_ep_vc_lookup(ep, dest_addr);
+ if (vc) {
+ COND_RELEASE(ep->requires_lock, &ep->vc_lock);
+ *vc_ptr = vc;
+ return FI_SUCCESS;
+ }
+
+ /* VC is not mapped yet. We can receive a connection request from a
+ * remote peer before the target EP has bound to an AV or before the
+ * remote peer has had it's address inserted into the target EP's AV.
+ * Those requests will result in a connection as usual, but the VC will
+ * not be mapped into an EP's AV until the EP attempts to send to the
+ * remote peer. Check the 'unmapped VC' list to see if such a VC
+ * exists and map it into the AV here. */
+ vc = __gnix_vc_lookup_unmapped(ep, dest_addr);
+ if (vc) {
+ COND_RELEASE(ep->requires_lock, &ep->vc_lock);
+ *vc_ptr = vc;
+ return FI_SUCCESS;
+ }
+
+ /* No VC exists for the peer yet. Look up full AV entry for the
+ * destination address. */
+ ret = _gnix_av_lookup(av, dest_addr, &av_entry);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_av_lookup for addr 0x%llx returned %s \n",
+ dest_addr, fi_strerror(-ret));
+ goto err_w_lock;
+ }
+
+ /* Allocate new VC with AV entry. */
+ ret = _gnix_vc_alloc(ep, &av_entry, &vc);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_vc_alloc returned %s\n",
+ fi_strerror(-ret));
+ goto err_w_lock;
+ }
+
+ /* Map new VC through the EP connection table. */
+ ret = _gnix_ep_vc_store(ep, vc, dest_addr);
+ if (unlikely(ret != FI_SUCCESS)) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_ep_vc_store returned %s\n",
+ fi_strerror(-ret));
+ goto err_w_lock;
+ }
+
+ /* Initiate new VC connection. */
+ ret = _gnix_vc_connect(vc);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_vc_connect returned %s\n",
+ fi_strerror(-ret));
+ goto err_w_lock;
+ }
+
+ COND_RELEASE(ep->requires_lock, &ep->vc_lock);
+
+ /* If we just initiated an intra-CM NIC connection, run CM progress.
+ * This will attempt to complete the connection immediately. This may
+ * be required if auto-progress is enabled because CM progress would
+ * not otherwise be run until a datagram is received on the network. */
+ if (GNIX_ADDR_EQUAL(vc->peer_cm_nic_addr,
+ vc->ep->cm_nic->my_name.gnix_addr)) {
+ ret = _gnix_cm_nic_progress(ep->domain->cm_nic);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_cm_nic_progress() returned %s\n",
+ fi_strerror(-ret));
+ }
+ }
+
+ *vc_ptr = vc;
+ return ret;
+
+err_w_lock:
+ COND_RELEASE(ep->requires_lock, &ep->vc_lock);
+ if (vc != NULL)
+ _gnix_vc_destroy(vc);
+ return ret;
+}
+
/*******************************************************************************
* connection request /response message pack/unpack functions
******************************************************************************/
@@ -94,7 +330,8 @@ static void __gnix_vc_pack_conn_req(char *sbuf,
uint64_t src_vc_vaddr,
gni_smsg_attr_t *src_smsg_attr,
gni_mem_handle_t *src_irq_cq_mhdl,
- uint64_t caps)
+ uint64_t caps,
+ xpmem_segid_t my_segid)
{
size_t __attribute__((unused)) len;
char *cptr = sbuf;
@@ -111,7 +348,8 @@ static void __gnix_vc_pack_conn_req(char *sbuf,
sizeof(int) +
sizeof(uint64_t) * 2 +
sizeof(gni_smsg_attr_t) +
- sizeof(gni_mem_handle_t);
+ sizeof(gni_mem_handle_t) +
+ sizeof(xpmem_segid_t);
assert(len <= GNIX_CM_NIC_MAX_MSG_SIZE);
memcpy(cptr, &rtype, sizeof(rtype));
@@ -129,6 +367,8 @@ static void __gnix_vc_pack_conn_req(char *sbuf,
memcpy(cptr, src_irq_cq_mhdl, sizeof(gni_mem_handle_t));
cptr += sizeof(gni_mem_handle_t);
memcpy(cptr, &caps, sizeof(uint64_t));
+ cptr += sizeof(xpmem_segid_t);
+ memcpy(cptr, &my_segid, sizeof(xpmem_segid_t));
}
/*
@@ -141,7 +381,8 @@ static void __gnix_vc_unpack_conn_req(char *rbuf,
uint64_t *src_vc_vaddr,
gni_smsg_attr_t *src_smsg_attr,
gni_mem_handle_t *src_irq_cq_mhndl,
- uint64_t *caps)
+ uint64_t *caps,
+ xpmem_segid_t *peer_segid)
{
size_t __attribute__((unused)) len;
char *cptr = rbuf;
@@ -166,6 +407,8 @@ static void __gnix_vc_unpack_conn_req(char *rbuf,
memcpy(src_irq_cq_mhndl, cptr, sizeof(gni_mem_handle_t));
cptr += sizeof(gni_mem_handle_t);
memcpy(caps, cptr, sizeof(uint64_t));
+ cptr += sizeof(uint64_t);
+ memcpy(peer_segid, cptr, sizeof(xpmem_segid_t));
}
/*
@@ -185,7 +428,8 @@ static void __gnix_vc_pack_conn_resp(char *sbuf,
int resp_vc_id,
gni_smsg_attr_t *resp_smsg_attr,
gni_mem_handle_t *resp_irq_cq_mhndl,
- uint64_t caps)
+ uint64_t caps,
+ xpmem_segid_t my_segid)
{
size_t __attribute__((unused)) len;
char *cptr = sbuf;
@@ -201,7 +445,8 @@ static void __gnix_vc_pack_conn_resp(char *sbuf,
sizeof(uint64_t) * 3 +
sizeof(int) +
sizeof(gni_smsg_attr_t) +
- sizeof(gni_mem_handle_t);
+ sizeof(gni_mem_handle_t) +
+ sizeof(xpmem_segid_t);
assert(len <= GNIX_CM_NIC_MAX_MSG_SIZE);
memcpy(cptr, &rtype, sizeof(rtype));
@@ -217,6 +462,8 @@ static void __gnix_vc_pack_conn_resp(char *sbuf,
memcpy(cptr, resp_irq_cq_mhndl, sizeof(gni_mem_handle_t));
cptr += sizeof(gni_mem_handle_t);
memcpy(cptr, &caps, sizeof(uint64_t));
+ cptr += sizeof(uint64_t);
+ memcpy(cptr, &my_segid, sizeof(xpmem_segid_t));
}
/*
@@ -228,7 +475,8 @@ static void __gnix_vc_unpack_resp(char *rbuf,
int *resp_vc_id,
gni_smsg_attr_t *resp_smsg_attr,
gni_mem_handle_t *resp_irq_cq_mhndl,
- uint64_t *caps)
+ uint64_t *caps,
+ xpmem_segid_t *peer_segid)
{
char *cptr = rbuf;
@@ -245,6 +493,8 @@ static void __gnix_vc_unpack_resp(char *rbuf,
memcpy(resp_irq_cq_mhndl, cptr, sizeof(gni_mem_handle_t));
cptr += sizeof(gni_mem_handle_t);
memcpy(caps, cptr, sizeof(uint64_t));
+ cptr += sizeof(uint64_t);
+ memcpy(peer_segid, cptr, sizeof(xpmem_segid_t));
}
static void __gnix_vc_get_msg_type(char *rbuf,
@@ -297,7 +547,7 @@ static int __gnix_vc_smsg_init(struct gnix_vc *vc,
* now build the SMSG connection
*/
- fastlock_acquire(&ep->nic->lock);
+ COND_ACQUIRE(ep->nic->requires_lock, &ep->nic->lock);
status = GNI_EpCreate(ep->nic->gni_nic_hndl,
ep->nic->tx_cq,
@@ -343,34 +593,25 @@ static int __gnix_vc_smsg_init(struct gnix_vc *vc,
if (peer_irq_mem_hndl != NULL)
vc->peer_irq_mem_hndl = *peer_irq_mem_hndl;
- fastlock_release(&ep->nic->lock);
+ COND_RELEASE(ep->nic->requires_lock, &ep->nic->lock);
return ret;
err1:
GNI_EpDestroy(vc->gni_ep);
err:
- fastlock_release(&ep->nic->lock);
+ COND_RELEASE(ep->nic->requires_lock, &ep->nic->lock);
return ret;
}
-/*
- * connect to self, since we use a lock here
- * the only case we need to deal with is one
- * vc connect request with the peer vc not yet
- * being set up
- */
-static int __gnix_vc_connect_to_same_cm_nic(struct gnix_vc *vc)
+static int __gnix_vc_connect_to_self(struct gnix_vc *vc)
{
int ret = FI_SUCCESS;
struct gnix_fid_domain *dom = NULL;
struct gnix_fid_ep *ep = NULL;
- struct gnix_fid_ep *ep_peer = NULL;
struct gnix_cm_nic *cm_nic = NULL;
- struct gnix_mbox *mbox = NULL, *mbox_peer = NULL;
- struct gnix_vc *vc_peer;
+ struct gnix_mbox *mbox = NULL;
gni_smsg_attr_t smsg_mbox_attr;
- gni_smsg_attr_t smsg_mbox_attr_peer;
- gnix_ht_key_t *key_ptr;
- struct gnix_av_addr_entry entry;
+ xpmem_apid_t peer_apid;
+ xpmem_segid_t my_segid;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
@@ -379,42 +620,26 @@ static int __gnix_vc_connect_to_same_cm_nic(struct gnix_vc *vc)
return -FI_EINVAL;
cm_nic = ep->cm_nic;
- if (cm_nic == NULL) {
- ret = -FI_EINVAL;
- goto exit;
- }
+ if (cm_nic == NULL)
+ return -FI_EINVAL;
dom = ep->domain;
- if (dom == NULL) {
- ret = -FI_EINVAL;
- goto exit;
- }
-
- fastlock_acquire(&ep->vc_ht_lock);
- if ((vc->conn_state == GNIX_VC_CONNECTING) ||
- (vc->conn_state == GNIX_VC_CONNECTED)) {
- fastlock_release(&ep->vc_ht_lock);
- return FI_SUCCESS;
- } else
- vc->conn_state = GNIX_VC_CONNECTING;
-
- fastlock_release(&ep->vc_ht_lock);
+ if (dom == NULL)
+ return -FI_EINVAL;
- GNIX_DEBUG(FI_LOG_EP_CTRL, "moving vc %p state to connecting\n", vc);
+ assert(vc->conn_state == GNIX_VC_CONN_NONE);
+ vc->conn_state = GNIX_VC_CONNECTING;
+ assert(vc->smsg_mbox == NULL);
- if (vc->smsg_mbox == NULL) {
- ret = _gnix_mbox_alloc(vc->ep->nic->mbox_hndl,
- &mbox);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_DATA,
- "_gnix_mbox_alloc returned %s\n",
- fi_strerror(-ret));
- goto exit;
- }
- vc->smsg_mbox = mbox;
- } else
- mbox = vc->smsg_mbox;
+ ret = _gnix_mbox_alloc(vc->ep->nic->mbox_hndl, &mbox);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_mbox_alloc returned %s\n",
+ fi_strerror(-ret));
+ return -FI_ENOSPC;
+ }
+ vc->smsg_mbox = mbox;
smsg_mbox_attr.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
smsg_mbox_attr.msg_buffer = mbox->base;
@@ -424,143 +649,51 @@ static int __gnix_vc_connect_to_same_cm_nic(struct gnix_vc *vc)
smsg_mbox_attr.mbox_maxcredit = dom->params.mbox_maxcredit;
smsg_mbox_attr.msg_maxsize = dom->params.mbox_msg_maxsize;
- key_ptr = (gnix_ht_key_t *)&vc->peer_addr;
- ep_peer = (struct gnix_fid_ep *)_gnix_ht_lookup(cm_nic->addr_to_ep_ht,
- *key_ptr);
- if (ep_peer == NULL) {
- GNIX_WARN(FI_LOG_EP_DATA,
- "_gnix_ht_lookup addr_to_ep failed\n");
- ret = -FI_ENOENT;
- goto exit;
- }
-
- key_ptr = (gnix_ht_key_t *)&ep->my_name.gnix_addr;
-
- fastlock_acquire(&ep_peer->vc_ht_lock);
- vc_peer = (struct gnix_vc *)_gnix_ht_lookup(ep_peer->vc_ht,
- *key_ptr);
-
- /*
- * handle the special case of connecting to self
- */
-
- if (vc_peer == vc) {
- ret = __gnix_vc_smsg_init(vc, vc->vc_id, &smsg_mbox_attr, NULL);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_DATA,
- "_gnix_vc_smsg_init returned %s\n",
- fi_strerror(-ret));
- goto exit_w_lock;
- }
- vc->conn_state = GNIX_VC_CONNECTED;
- vc->peer_id = vc->vc_id;
- vc->peer_caps = ep->caps;
- ret = _gnix_vc_schedule(vc);
- if (ret != FI_SUCCESS)
- GNIX_WARN(FI_LOG_EP_DATA,
- "_gnix_vc_schedule returned %s\n",
- fi_strerror(-ret));
-
- GNIX_DEBUG(FI_LOG_EP_CTRL, "moving vc %p state to connected\n",
- vc);
- goto exit_w_lock;
- }
-
- if ((vc_peer != NULL) &&
- (vc_peer->conn_state != GNIX_VC_CONN_NONE)) {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "_gnix_vc_connect self, vc_peer in inconsistent state\n");
- ret = -FI_ENOSPC;
- goto exit_w_lock;
- }
-
- if (vc_peer == NULL) {
- entry.gnix_addr = ep->my_name.gnix_addr;
- entry.cm_nic_cdm_id = ep->my_name.cm_nic_cdm_id;
- ret = _gnix_vc_alloc(ep_peer,
- &entry,
- &vc_peer);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "_gnix_vc_alloc returned %s\n",
- fi_strerror(-ret));
- goto exit_w_lock;
- }
-
- ret = _gnix_ht_insert(ep_peer->vc_ht,
- *key_ptr,
- vc_peer);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_CTRL,
- "_gnix_ht_insert returned %s\n",
- fi_strerror(-ret));
- goto exit_w_lock;
- }
- vc_peer->modes |= GNIX_VC_MODE_IN_HT;
- }
-
- vc_peer->conn_state = GNIX_VC_CONNECTING;
- GNIX_DEBUG(FI_LOG_EP_CTRL, "moving vc %p state to connecting\n",
- vc_peer);
-
- if (vc_peer->smsg_mbox == NULL) {
- ret = _gnix_mbox_alloc(vc_peer->ep->nic->mbox_hndl,
- &mbox_peer);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_DATA,
- "_gnix_mbox_alloc returned %s\n",
- fi_strerror(-ret));
- goto exit_w_lock;
- }
- vc_peer->smsg_mbox = mbox_peer;
- } else
- mbox_peer = vc_peer->smsg_mbox;
-
- smsg_mbox_attr_peer.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
- smsg_mbox_attr_peer.msg_buffer = mbox_peer->base;
- smsg_mbox_attr_peer.buff_size = vc_peer->ep->nic->mem_per_mbox;
- smsg_mbox_attr_peer.mem_hndl = *mbox_peer->memory_handle;
- smsg_mbox_attr_peer.mbox_offset = (uint64_t)mbox_peer->offset;
- smsg_mbox_attr_peer.mbox_maxcredit = dom->params.mbox_maxcredit;
- smsg_mbox_attr_peer.msg_maxsize = dom->params.mbox_msg_maxsize;
-
- ret = __gnix_vc_smsg_init(vc, vc_peer->vc_id, &smsg_mbox_attr_peer,
- &ep_peer->nic->irq_mem_hndl);
+ ret = __gnix_vc_smsg_init(vc, vc->vc_id, &smsg_mbox_attr, NULL);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_vc_smsg_init returned %s\n",
fi_strerror(-ret));
- goto exit_w_lock;
+ goto err_mbox_init;
}
+ vc->conn_state = GNIX_VC_CONNECTED;
- ret = __gnix_vc_smsg_init(vc_peer, vc->vc_id, &smsg_mbox_attr,
- &ep->nic->irq_mem_hndl);
+ /* TODO: use special send-to-self mechanism to avoid overhead of XPMEM
+ * when just sending a message to oneself. */
+ ret = _gnix_xpmem_get_my_segid(ep->xpmem_hndl, &my_segid);
if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_DATA,
- "_gnix_vc_smsg_init returned %s\n",
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gni_xpmem_get_my_segid returned %s\n",
fi_strerror(-ret));
- goto exit_w_lock;
}
- vc->conn_state = GNIX_VC_CONNECTED;
- vc->peer_id = vc_peer->vc_id;
+ ret = _gnix_xpmem_get_apid(ep->xpmem_hndl, my_segid, &peer_apid);
+ if (ret == FI_SUCCESS) {
+ vc->modes |= GNIX_VC_MODE_XPMEM;
+ vc->peer_apid = peer_apid;
+ } else {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gni_xpmem_get_apiid returned %s\n",
+ fi_strerror(-ret));
+ }
+
+ vc->peer_id = vc->vc_id;
vc->peer_caps = ep->caps;
- GNIX_DEBUG(FI_LOG_EP_CTRL, "moving vc %p state to connected\n",
- vc);
- vc_peer->conn_state = GNIX_VC_CONNECTED;
- vc_peer->peer_id = vc->vc_id;
- ret = _gnix_vc_schedule(vc_peer);
+ vc->peer_irq_mem_hndl = ep->nic->irq_mem_hndl;
+
+ ret = _gnix_vc_schedule(vc);
if (ret != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_vc_schedule returned %s\n",
fi_strerror(-ret));
- GNIX_DEBUG(FI_LOG_EP_CTRL, "moving vc %p state to connected\n",
- vc_peer);
+ GNIX_DEBUG(FI_LOG_EP_CTRL, "moving vc %p state to connected\n", vc);
+ return ret;
-exit_w_lock:
- fastlock_release(&ep_peer->vc_ht_lock);
-exit:
+err_mbox_init:
+ _gnix_mbox_free(vc->smsg_mbox);
+ vc->smsg_mbox = NULL;
+
return ret;
}
@@ -580,6 +713,9 @@ static int __gnix_vc_hndl_conn_resp(struct gnix_cm_nic *cm_nic,
gni_smsg_attr_t peer_smsg_attr;
gni_mem_handle_t tmp_mem_hndl;
uint64_t peer_caps;
+ xpmem_segid_t peer_segid;
+ xpmem_apid_t peer_apid;
+ bool accessible;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
@@ -593,7 +729,8 @@ static int __gnix_vc_hndl_conn_resp(struct gnix_cm_nic *cm_nic,
&peer_id,
&peer_smsg_attr,
&tmp_mem_hndl,
- &peer_caps);
+ &peer_caps,
+ &peer_segid);
GNIX_DEBUG(FI_LOG_EP_CTRL,
"resp rx: (From Aries 0x%x Id %d src vc %p peer vc addr 0x%lx)\n",
@@ -605,7 +742,7 @@ static int __gnix_vc_hndl_conn_resp(struct gnix_cm_nic *cm_nic,
ep = vc->ep;
assert(ep != NULL);
- fastlock_acquire(&ep->vc_ht_lock);
+ COND_ACQUIRE(ep->requires_lock, &ep->vc_lock);
/*
* at this point vc should be in connecting state
@@ -632,6 +769,21 @@ static int __gnix_vc_hndl_conn_resp(struct gnix_cm_nic *cm_nic,
}
/*
+ * see if we can do xpmem with this EP
+ */
+
+ ret = _gnix_xpmem_accessible(ep, src_cm_nic_addr, &accessible);
+ if ((ret == FI_SUCCESS) && (accessible == true)) {
+ ret = _gnix_xpmem_get_apid(ep->xpmem_hndl,
+ peer_segid,
+ &peer_apid);
+ if (ret == FI_SUCCESS) {
+ vc->modes |= GNIX_VC_MODE_XPMEM;
+ vc->peer_apid = peer_apid;
+ }
+ }
+
+ /*
* transition the VC to connected
* put in to the nic's work queue for
* further processing
@@ -643,7 +795,7 @@ static int __gnix_vc_hndl_conn_resp(struct gnix_cm_nic *cm_nic,
" moving vc %p to state connected\n",vc);
vc->peer_caps = peer_caps;
- fastlock_release(&ep->vc_ht_lock);
+ COND_RELEASE(ep->requires_lock, &ep->vc_lock);
ret = _gnix_vc_schedule(vc);
if (ret != FI_SUCCESS)
@@ -654,7 +806,8 @@ static int __gnix_vc_hndl_conn_resp(struct gnix_cm_nic *cm_nic,
return ret;
err:
vc->conn_state = GNIX_VC_CONN_ERROR;
- fastlock_release(&ep->vc_ht_lock);
+
+ COND_RELEASE(ep->requires_lock, &ep->vc_lock);
return ret;
}
@@ -665,11 +818,10 @@ static int __gnix_vc_hndl_conn_req(struct gnix_cm_nic *cm_nic,
int ret = FI_SUCCESS;
gni_return_t __attribute__((unused)) status;
struct gnix_fid_ep *ep = NULL;
- gnix_ht_key_t *key_ptr;
+ gnix_ht_key_t key;
struct gnix_av_addr_entry entry;
struct gnix_address src_addr, target_addr;
struct gnix_vc *vc = NULL;
- struct gnix_vc *vc_try = NULL;
struct gnix_work_req *work_req;
int src_vc_id;
gni_smsg_attr_t src_smsg_attr;
@@ -677,7 +829,11 @@ static int __gnix_vc_hndl_conn_req(struct gnix_cm_nic *cm_nic,
uint64_t peer_caps;
struct wq_hndl_conn_req *data = NULL;
gni_mem_handle_t tmp_mem_hndl;
-
+ int src_mapped = 0;
+ fi_addr_t fi_addr;
+ xpmem_segid_t peer_segid;
+ xpmem_apid_t peer_apid;
+ bool accessible;
ssize_t __attribute__((unused)) len;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
@@ -693,7 +849,8 @@ static int __gnix_vc_hndl_conn_req(struct gnix_cm_nic *cm_nic,
&src_vc_ptr,
&src_smsg_attr,
&tmp_mem_hndl,
- &peer_caps);
+ &peer_caps,
+ &peer_segid);
GNIX_DEBUG(FI_LOG_EP_CTRL,
@@ -709,10 +866,10 @@ static int __gnix_vc_hndl_conn_req(struct gnix_cm_nic *cm_nic,
* in the datagram
*/
- key_ptr = (gnix_ht_key_t *)&target_addr;
+ __gnix_vc_set_ht_key(&target_addr, &key);
ep = (struct gnix_fid_ep *)_gnix_ht_lookup(cm_nic->addr_to_ep_ht,
- *key_ptr);
+ key);
if (ep == NULL) {
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_ht_lookup addr_to_ep failed\n");
@@ -725,18 +882,23 @@ static int __gnix_vc_hndl_conn_req(struct gnix_cm_nic *cm_nic,
* address of the connecting EP.
*/
- key_ptr = (gnix_ht_key_t *)&src_addr;
+ COND_ACQUIRE(ep->requires_lock, &ep->vc_lock);
- fastlock_acquire(&ep->vc_ht_lock);
- vc = (struct gnix_vc *)_gnix_ht_lookup(ep->vc_ht,
- *key_ptr);
+ /* If we already have an AV bound, see if sender's address is already
+ * mapped. */
+ if (ep->av) {
+ ret = _gnix_av_reverse_lookup(ep->av, src_addr, &fi_addr);
+ if (ret == FI_SUCCESS) {
+ src_mapped = 1;
+ vc = _gnix_ep_vc_lookup(ep, fi_addr);
+ }
+ }
/*
- * if there is no corresponding vc in the hash,
- * or there is an entry and its not in connecting state
- * go down the conn req ack route.
- */
-
+ * if there is no corresponding vc in the hash,
+ * or there is an entry and it's not in connecting state
+ * go down the conn req ack route.
+ */
if ((vc == NULL) ||
(vc->conn_state == GNIX_VC_CONN_NONE)) {
if (vc == NULL) {
@@ -744,7 +906,7 @@ static int __gnix_vc_hndl_conn_req(struct gnix_cm_nic *cm_nic,
entry.cm_nic_cdm_id = src_cm_nic_addr.cdm_id;
ret = _gnix_vc_alloc(ep,
&entry,
- &vc_try);
+ &vc);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_vc_alloc returned %s\n",
@@ -752,20 +914,29 @@ static int __gnix_vc_hndl_conn_req(struct gnix_cm_nic *cm_nic,
goto err;
}
- vc_try->conn_state = GNIX_VC_CONNECTING;
- ret = _gnix_ht_insert(ep->vc_ht,
- *key_ptr,
- vc_try);
- if (likely(ret == FI_SUCCESS)) {
- vc = vc_try;
- vc->modes |= GNIX_VC_MODE_IN_HT;
+ vc->conn_state = GNIX_VC_CONNECTING;
+
+ if (src_mapped) {
+ /* We have an AV which maps the incoming
+ * address. Store the new VC in our VC lookup
+ * table. */
+ ret = _gnix_ep_vc_store(ep, vc, fi_addr);
+ if (unlikely(ret != FI_SUCCESS)) {
+ _gnix_vc_destroy(vc);
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_ep_vc_store returned %s\n",
+ fi_strerror(-ret));
+ goto err;
+ }
} else {
- if (ret == -FI_ENOSPC)
- _gnix_vc_destroy(vc_try);
- GNIX_WARN(FI_LOG_EP_DATA,
- "_gnix_ht_insert returned %s\n",
- fi_strerror(-ret));
- goto err;
+ /* We lack an AV and/or the entry to map the
+ * incoming address. Keep VC in special table
+ * until it is mapped for a TX operation. */
+ GNIX_INFO(FI_LOG_EP_CTRL,
+ "Received conn. request from unmapped peer EP, vc: %p addr: 0x%lx\n",
+ vc, src_addr);
+
+ dlist_insert_tail(&vc->list, &ep->unmapped_vcs);
}
} else
vc->conn_state = GNIX_VC_CONNECTING;
@@ -793,6 +964,7 @@ static int __gnix_vc_hndl_conn_req(struct gnix_cm_nic *cm_nic,
data->src_vc_id = src_vc_id;
data->src_vc_ptr = src_vc_ptr;
data->irq_mem_hndl = tmp_mem_hndl;
+ data->peer_segid = peer_segid;
work_req->progress_fn = __gnix_vc_conn_ack_prog_fn;
work_req->data = data;
@@ -804,12 +976,11 @@ static int __gnix_vc_hndl_conn_req(struct gnix_cm_nic *cm_nic,
* cm_nic's work queue, progress the cm_nic.
*/
-
fastlock_acquire(&cm_nic->wq_lock);
dlist_insert_before(&work_req->list, &cm_nic->cm_nic_wq);
fastlock_release(&cm_nic->wq_lock);
- fastlock_release(&ep->vc_ht_lock);
+ COND_RELEASE(ep->requires_lock, &ep->vc_lock);
ret = _gnix_vc_schedule(vc);
if (ret != FI_SUCCESS)
@@ -849,12 +1020,23 @@ static int __gnix_vc_hndl_conn_req(struct gnix_cm_nic *cm_nic,
goto err;
}
+ ret = _gnix_xpmem_accessible(ep, src_cm_nic_addr, &accessible);
+ if ((ret == FI_SUCCESS) && (accessible == true)) {
+ ret = _gnix_xpmem_get_apid(ep->xpmem_hndl,
+ peer_segid,
+ &peer_apid);
+ if (ret == FI_SUCCESS) {
+ vc->modes |= GNIX_VC_MODE_XPMEM;
+ vc->peer_apid = peer_apid;
+ }
+ }
+
vc->conn_state = GNIX_VC_CONNECTED;
vc->peer_id = src_vc_id;
GNIX_DEBUG(FI_LOG_EP_CTRL, "moving vc %p state to connected\n",
vc);
- fastlock_release(&ep->vc_ht_lock);
+ COND_RELEASE(ep->requires_lock, &ep->vc_lock);
ret = _gnix_vc_schedule(vc);
if (ret != FI_SUCCESS)
@@ -925,7 +1107,10 @@ static int __gnix_vc_conn_ack_prog_fn(void *data, int *complete_ptr)
struct gnix_fid_ep *ep = NULL;
struct gnix_fid_domain *dom = NULL;
struct gnix_cm_nic *cm_nic = NULL;
+ xpmem_segid_t my_segid;
char sbuf[GNIX_CM_NIC_MAX_MSG_SIZE] = {0};
+ xpmem_apid_t peer_apid;
+ bool accessible;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
@@ -948,7 +1133,7 @@ static int __gnix_vc_conn_ack_prog_fn(void *data, int *complete_ptr)
if (cm_nic == NULL)
return -FI_EINVAL;
- fastlock_acquire(&ep->vc_ht_lock);
+ COND_ACQUIRE(ep->requires_lock, &ep->vc_lock);
/*
* we may have already been moved to connected or
@@ -980,7 +1165,7 @@ static int __gnix_vc_conn_ack_prog_fn(void *data, int *complete_ptr)
/*
* prep the smsg_mbox_attr
-¬ */
+ */
smsg_mbox_attr.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
smsg_mbox_attr.msg_buffer = mbox->base;
@@ -994,13 +1179,21 @@ static int __gnix_vc_conn_ack_prog_fn(void *data, int *complete_ptr)
* serialize the resp message in the buffer
*/
+ ret = _gnix_xpmem_get_my_segid(ep->xpmem_hndl,
+ &my_segid);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "_gni_xpmem_get_my_segid returned %s\n",
+ fi_strerror(-ret));
+ }
+
__gnix_vc_pack_conn_resp(sbuf,
work_req_data->src_vc_ptr,
(uint64_t)vc,
vc->vc_id,
&smsg_mbox_attr,
&ep->nic->irq_mem_hndl,
- ep->caps);
+ ep->caps,
+ my_segid);
/*
* try to send the message, if it succeeds,
@@ -1023,6 +1216,23 @@ static int __gnix_vc_conn_ack_prog_fn(void *data, int *complete_ptr)
fi_strerror(-ret));
goto exit;
}
+
+ /*
+ * TODO: xpmem setup here
+ */
+
+ ret = _gnix_xpmem_accessible(ep, vc->peer_cm_nic_addr,
+ &accessible);
+ if ((ret == FI_SUCCESS) && (accessible == true)) {
+ ret = _gnix_xpmem_get_apid(ep->xpmem_hndl,
+ work_req_data->peer_segid,
+ &peer_apid);
+ if (ret == FI_SUCCESS) {
+ vc->modes |= GNIX_VC_MODE_XPMEM;
+ vc->peer_apid = peer_apid;
+ }
+ }
+
complete = 1;
vc->conn_state = GNIX_VC_CONNECTED;
vc->peer_id = work_req_data->src_vc_id;
@@ -1048,7 +1258,7 @@ static int __gnix_vc_conn_ack_prog_fn(void *data, int *complete_ptr)
}
exit:
- fastlock_release(&ep->vc_ht_lock);
+ COND_RELEASE(ep->requires_lock, &ep->vc_lock);
*complete_ptr = complete;
return ret;
@@ -1064,6 +1274,7 @@ static int __gnix_vc_conn_req_prog_fn(void *data, int *complete_ptr)
struct gnix_fid_ep *ep = NULL;
struct gnix_fid_domain *dom = NULL;
struct gnix_cm_nic *cm_nic = NULL;
+ xpmem_segid_t my_segid;
char sbuf[GNIX_CM_NIC_MAX_MSG_SIZE] = {0};
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
@@ -1080,7 +1291,7 @@ static int __gnix_vc_conn_req_prog_fn(void *data, int *complete_ptr)
if (cm_nic == NULL)
return -FI_EINVAL;
- fastlock_acquire(&ep->vc_ht_lock);
+ COND_ACQUIRE(ep->requires_lock, &ep->vc_lock);
if ((vc->conn_state == GNIX_VC_CONNECTING) ||
(vc->conn_state == GNIX_VC_CONNECTED)) {
@@ -1089,17 +1300,6 @@ static int __gnix_vc_conn_req_prog_fn(void *data, int *complete_ptr)
}
/*
- * sanity check that the vc is in the hash table
- */
-
- if (!(vc->modes & GNIX_VC_MODE_IN_HT)) {
- GNIX_WARN(FI_LOG_EP_CTRL, "vc not in hashtable\n");
- assert(vc->modes & GNIX_VC_MODE_IN_HT);
- ret = -FI_EINVAL;
- goto err;
- }
-
- /*
* first see if we still need a mailbox
*/
@@ -1116,7 +1316,7 @@ static int __gnix_vc_conn_req_prog_fn(void *data, int *complete_ptr)
/*
* prep the smsg_mbox_attr
-¬ */
+ */
smsg_mbox_attr.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
smsg_mbox_attr.msg_buffer = mbox->base;
@@ -1139,6 +1339,14 @@ static int __gnix_vc_conn_req_prog_fn(void *data, int *complete_ptr)
vc->peer_cm_nic_addr.cdm_id,
vc);
+ ret = _gnix_xpmem_get_my_segid(ep->xpmem_hndl,
+ &my_segid);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gnix_xpmem_get_my_segid returned %s\n",
+ fi_strerror(-ret));
+ }
+
__gnix_vc_pack_conn_req(sbuf,
&vc->peer_addr,
&ep->my_name.gnix_addr,
@@ -1146,7 +1354,8 @@ static int __gnix_vc_conn_req_prog_fn(void *data, int *complete_ptr)
(uint64_t)vc,
&smsg_mbox_attr,
&ep->nic->irq_mem_hndl,
- ep->caps);
+ ep->caps,
+ my_segid);
/*
* try to send the message, if -FI_EAGAIN is returned, okay,
@@ -1176,7 +1385,7 @@ static int __gnix_vc_conn_req_prog_fn(void *data, int *complete_ptr)
fi_strerror(-ret));
err:
- fastlock_release(&ep->vc_ht_lock);
+ COND_RELEASE(ep->requires_lock, &ep->vc_lock);
*complete_ptr = complete;
return ret;
}
@@ -1204,7 +1413,6 @@ static int __gnix_vc_conn_req_comp_fn(void *data)
/*******************************************************************************
* Internal API functions
******************************************************************************/
-
int _gnix_vc_alloc(struct gnix_fid_ep *ep_priv,
struct gnix_av_addr_entry *entry, struct gnix_vc **vc)
@@ -1246,15 +1454,17 @@ int _gnix_vc_alloc(struct gnix_fid_ep *ep_priv,
}
vc_ptr->ep = ep_priv;
- slist_init(&vc_ptr->work_queue);
+ dlist_init(&vc_ptr->work_queue);
fastlock_init(&vc_ptr->work_queue_lock);
- slist_init(&vc_ptr->tx_queue);
+ dlist_init(&vc_ptr->tx_queue);
fastlock_init(&vc_ptr->tx_queue_lock);
dlist_init(&vc_ptr->rx_list);
dlist_init(&vc_ptr->work_list);
dlist_init(&vc_ptr->tx_list);
vc_ptr->peer_fi_addr = FI_ADDR_NOTAVAIL;
+ dlist_init(&vc_ptr->list);
+
atomic_initialize(&vc_ptr->outstanding_tx_reqs, 0);
ret = _gnix_alloc_bitmap(&vc_ptr->flags, 1);
assert(!ret);
@@ -1283,20 +1493,20 @@ static void __gnix_vc_cancel(struct gnix_vc *vc)
{
struct gnix_nic *nic = vc->ep->nic;
- fastlock_acquire(&nic->rx_vc_lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->rx_vc_lock);
if (!dlist_empty(&vc->rx_list))
dlist_remove(&vc->rx_list);
- fastlock_release(&nic->rx_vc_lock);
+ COND_RELEASE(nic->requires_lock, &nic->rx_vc_lock);
- fastlock_acquire(&nic->work_vc_lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->work_vc_lock);
if (!dlist_empty(&vc->work_list))
dlist_remove(&vc->work_list);
- fastlock_release(&nic->work_vc_lock);
+ COND_RELEASE(nic->requires_lock, &nic->work_vc_lock);
- fastlock_acquire(&nic->tx_vc_lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->tx_vc_lock);
if (!dlist_empty(&vc->tx_list))
dlist_remove(&vc->tx_list);
- fastlock_release(&nic->tx_vc_lock);
+ COND_RELEASE(nic->requires_lock, &nic->tx_vc_lock);
}
/* Destroy an unconnected VC. More Support is needed to shutdown and destroy
@@ -1335,9 +1545,9 @@ int _gnix_vc_destroy(struct gnix_vc *vc)
if (vc->gni_ep != NULL) {
while (status == GNI_RC_NOT_DONE) {
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
status = GNI_EpUnbind(vc->gni_ep);
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
if ((status != GNI_RC_NOT_DONE) &&
(status != GNI_RC_SUCCESS)) {
@@ -1350,9 +1560,9 @@ int _gnix_vc_destroy(struct gnix_vc *vc)
if (status == GNI_RC_NOT_DONE)
_gnix_nic_progress(nic);
}
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
status = GNI_EpDestroy(vc->gni_ep);
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
if (status != GNI_RC_SUCCESS)
GNIX_WARN(FI_LOG_EP_CTRL,
"GNI_EpDestroy returned %s\n",
@@ -1384,10 +1594,13 @@ int _gnix_vc_destroy(struct gnix_vc *vc)
* may not be correct for handling fi_shutdown.
*/
- if (!slist_empty(&vc->tx_queue)) {
- GNIX_WARN(FI_LOG_EP_CTRL, "vc sendqueue not empty\n");
- return -FI_EBUSY;
- }
+ if (!dlist_empty(&vc->tx_queue))
+ GNIX_FATAL(FI_LOG_EP_CTRL, "VC TX queue not empty\n");
+
+ if (atomic_get(&vc->outstanding_tx_reqs))
+ GNIX_FATAL(FI_LOG_EP_CTRL,
+ "VC outstanding_tx_reqs out of sync: %d\n",
+ atomic_get(&vc->outstanding_tx_reqs));
fastlock_destroy(&vc->tx_queue_lock);
@@ -1457,14 +1670,11 @@ int _gnix_vc_connect(struct gnix_vc *vc)
return -FI_EINVAL;
/*
- * have to do something special for
- * connect to same cm_nic
+ * check if this EP is connecting to itself
*/
- if (!memcmp(&vc->peer_cm_nic_addr,
- &cm_nic->my_name.gnix_addr,
- sizeof(struct gnix_address))) {
- return __gnix_vc_connect_to_same_cm_nic(vc);
+ if (GNIX_ADDR_EQUAL(ep->my_name.gnix_addr, vc->peer_addr)) {
+ return __gnix_vc_connect_to_self(vc);
}
/*
@@ -1545,9 +1755,9 @@ int _gnix_vc_rx_schedule(struct gnix_vc *vc)
struct gnix_nic *nic = vc->ep->nic;
if (!_gnix_test_and_set_bit(&vc->flags, GNIX_VC_FLAG_RX_SCHEDULED)) {
- fastlock_acquire(&nic->rx_vc_lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->rx_vc_lock);
dlist_insert_tail(&vc->rx_list, &nic->rx_vcs);
- fastlock_release(&nic->rx_vc_lock);
+ COND_RELEASE(nic->requires_lock, &nic->rx_vc_lock);
GNIX_INFO(FI_LOG_EP_CTRL, "Scheduled RX VC (%p)\n", vc);
}
@@ -1581,7 +1791,7 @@ int _gnix_vc_dequeue_smsg(struct gnix_vc *vc)
&tag);
if (status == GNI_RC_SUCCESS) {
- GNIX_INFO(FI_LOG_EP_DATA, "Found RX (%p)\n", vc);
+ GNIX_DEBUG(FI_LOG_EP_DATA, "Found RX (%p)\n", vc);
ret = nic->smsg_callbacks[tag](vc, msg_ptr);
if (ret != FI_SUCCESS) {
/* Stalled, reschedule */
@@ -1617,9 +1827,9 @@ static int __gnix_vc_rx_progress(struct gnix_vc *vc)
}
/* Process pending RXs */
- fastlock_acquire(&vc->ep->nic->lock);
+ COND_ACQUIRE(vc->ep->nic->requires_lock, &vc->ep->nic->lock);
ret = _gnix_vc_dequeue_smsg(vc);
- fastlock_release(&vc->ep->nic->lock);
+ COND_RELEASE(vc->ep->nic->requires_lock, &vc->ep->nic->lock);
if (ret != FI_SUCCESS) {
/* We didn't finish processing RXs. Low memory likely.
@@ -1637,11 +1847,11 @@ static struct gnix_vc *__gnix_nic_next_pending_rx_vc(struct gnix_nic *nic)
{
struct gnix_vc *vc = NULL;
- fastlock_acquire(&nic->rx_vc_lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->rx_vc_lock);
vc = dlist_first_entry(&nic->rx_vcs, struct gnix_vc, rx_list);
if (vc)
dlist_remove_init(&vc->rx_list);
- fastlock_release(&nic->rx_vc_lock);
+ COND_RELEASE(nic->requires_lock, &nic->rx_vc_lock);
if (vc) {
GNIX_INFO(FI_LOG_EP_CTRL, "Dequeued RX VC (%p)\n", vc);
@@ -1680,13 +1890,13 @@ static int __gnix_vc_work_schedule(struct gnix_vc *vc)
struct gnix_nic *nic = vc->ep->nic;
/* Don't bother scheduling if there's no work to do. */
- if (slist_empty(&vc->work_queue))
+ if (dlist_empty(&vc->work_queue))
return FI_SUCCESS;
if (!_gnix_test_and_set_bit(&vc->flags, GNIX_VC_FLAG_WORK_SCHEDULED)) {
- fastlock_acquire(&nic->work_vc_lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->work_vc_lock);
dlist_insert_tail(&vc->work_list, &nic->work_vcs);
- fastlock_release(&nic->work_vc_lock);
+ COND_RELEASE(nic->requires_lock, &nic->work_vc_lock);
GNIX_INFO(FI_LOG_EP_CTRL, "Scheduled work VC (%p)\n", vc);
}
@@ -1698,12 +1908,10 @@ int _gnix_vc_queue_work_req(struct gnix_fab_req *req)
{
struct gnix_vc *vc = req->vc;
- req->slist.next = NULL; /* keep slist happy */
-
- fastlock_acquire(&vc->work_queue_lock);
- slist_insert_tail(&req->slist, &vc->work_queue);
+ COND_ACQUIRE(vc->ep->requires_lock, &vc->work_queue_lock);
+ dlist_insert_tail(&req->dlist, &vc->work_queue);
__gnix_vc_work_schedule(vc);
- fastlock_release(&vc->work_queue_lock);
+ COND_RELEASE(vc->ep->requires_lock, &vc->work_queue_lock);
return FI_SUCCESS;
}
@@ -1712,17 +1920,18 @@ int _gnix_vc_queue_work_req(struct gnix_fab_req *req)
static int __gnix_vc_push_work_reqs(struct gnix_vc *vc)
{
int ret, fi_rc = FI_SUCCESS;
- struct slist_entry *item;
struct gnix_fab_req *req;
while (fi_rc == FI_SUCCESS) {
- fastlock_acquire(&vc->work_queue_lock);
- item = slist_remove_head(&vc->work_queue);
- fastlock_release(&vc->work_queue_lock);
-
- if (item != NULL) {
- req = (struct gnix_fab_req *)
- container_of(item, struct gnix_fab_req, slist);
+ COND_ACQUIRE(vc->ep->requires_lock, &vc->work_queue_lock);
+ req = dlist_first_entry(&vc->work_queue,
+ struct gnix_fab_req,
+ dlist);
+ if (req)
+ dlist_remove_init(&req->dlist);
+ COND_RELEASE(vc->ep->requires_lock, &vc->work_queue_lock);
+
+ if (req) {
ret = req->work_fn(req);
if (ret == FI_SUCCESS) {
GNIX_INFO(FI_LOG_EP_DATA,
@@ -1733,11 +1942,16 @@ static int __gnix_vc_push_work_reqs(struct gnix_vc *vc)
/* Work failed. Reschedule to put this VC
* back on the end of the list and return
* -FI_EAGAIN */
- __gnix_vc_work_schedule(vc);
- fastlock_acquire(&vc->work_queue_lock);
- slist_insert_tail(item, &vc->work_queue);
- fastlock_release(&vc->work_queue_lock);
+ COND_ACQUIRE(vc->ep->requires_lock,
+ &vc->work_queue_lock);
+ dlist_insert_tail(&req->dlist, &vc->work_queue);
+ COND_RELEASE(vc->ep->requires_lock,
+ &vc->work_queue_lock);
+
+ /* __gnix_vc_work_schedule() must come after the
+ * request is inserted into the VC's work_queue. */
+ __gnix_vc_work_schedule(vc);
fi_rc = -FI_EAGAIN;
@@ -1749,9 +1963,9 @@ static int __gnix_vc_push_work_reqs(struct gnix_vc *vc)
if ((ret != -FI_ENOSPC) &&
(ret != -FI_EAGAIN)) {
/* TODO report error? */
- GNIX_WARN(FI_LOG_EP_DATA,
- "Failed to push request %p: %s\n",
- req, fi_strerror(-ret));
+ GNIX_FATAL(FI_LOG_EP_DATA,
+ "Failed to push request %p: %s\n",
+ req, fi_strerror(-ret));
}
} else {
break; /* nothing left in the queue */
@@ -1765,11 +1979,11 @@ static struct gnix_vc *__gnix_nic_next_pending_work_vc(struct gnix_nic *nic)
{
struct gnix_vc *vc = NULL;
- fastlock_acquire(&nic->work_vc_lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->work_vc_lock);
vc = dlist_first_entry(&nic->work_vcs, struct gnix_vc, work_list);
if (vc)
dlist_remove_init(&vc->work_list);
- fastlock_release(&nic->work_vc_lock);
+ COND_RELEASE(nic->requires_lock, &nic->work_vc_lock);
if (vc) {
GNIX_INFO(FI_LOG_EP_CTRL, "Dequeued work VC (%p)\n", vc);
@@ -1805,13 +2019,13 @@ int _gnix_vc_tx_schedule(struct gnix_vc *vc)
struct gnix_nic *nic = vc->ep->nic;
/* Don't bother scheduling if there's no work to do. */
- if (slist_empty(&vc->tx_queue))
+ if (dlist_empty(&vc->tx_queue))
return FI_SUCCESS;
if (!_gnix_test_and_set_bit(&vc->flags, GNIX_VC_FLAG_TX_SCHEDULED)) {
- fastlock_acquire(&nic->tx_vc_lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->tx_vc_lock);
dlist_insert_tail(&vc->tx_list, &nic->tx_vcs);
- fastlock_release(&nic->tx_vc_lock);
+ COND_RELEASE(nic->requires_lock, &nic->tx_vc_lock);
GNIX_INFO(FI_LOG_EP_CTRL, "Scheduled TX VC (%p)\n", vc);
}
@@ -1826,37 +2040,46 @@ int _gnix_vc_queue_tx_req(struct gnix_fab_req *req)
struct gnix_vc *vc = req->vc;
int connected, injecting;
- connected = !__gnix_vc_connected(vc); /* 0 on success */
+ if (req->flags & FI_TRIGGER) {
+ rc = _gnix_trigger_queue_req(req);
- injecting = (req->flags & FI_INJECT) ? 1 : 0;
+ /* FI_SUCCESS means the request was queued to wait for the
+ * trigger condition. */
+ if (rc == FI_SUCCESS)
+ return FI_SUCCESS;
+ }
- fastlock_acquire(&vc->tx_queue_lock);
+ connected = !__gnix_vc_connected(vc); /* 0 on success */
+ injecting = (req->flags & FI_INJECT) ? 1 : 0;
+ COND_ACQUIRE(vc->ep->requires_lock, &vc->tx_queue_lock);
if ((req->flags & FI_FENCE) && atomic_get(&vc->outstanding_tx_reqs)) {
/* Fence request must be queued until all outstanding TX
* requests are completed. Subsequent requests will be queued
* due to non-empty tx_queue. */
queue_tx = 1;
- GNIX_INFO(FI_LOG_EP_DATA,
+ GNIX_DEBUG(FI_LOG_EP_DATA,
"Queued FI_FENCE request (%p) on VC\n",
req);
- } else if (connected && slist_empty(&vc->tx_queue)) {
+ } else if (connected && dlist_empty(&vc->tx_queue)) {
+ atomic_inc(&vc->outstanding_tx_reqs);
+
/* try to initiate request */
rc = req->work_fn(req);
- if (rc != FI_SUCCESS) {
+ if (rc == FI_SUCCESS) {
+ GNIX_DEBUG(FI_LOG_EP_DATA,
+ "TX request processed: %p (OTX: %d)\n",
+ req, atomic_get(&vc->outstanding_tx_reqs));
+ } else if (rc != -FI_ECANCELED) {
+ atomic_dec(&vc->outstanding_tx_reqs);
queue_tx = 1;
- GNIX_INFO(FI_LOG_EP_DATA,
+ GNIX_DEBUG(FI_LOG_EP_DATA,
"Queued request (%p) on full VC\n",
req);
- } else {
- atomic_inc(&vc->outstanding_tx_reqs);
- GNIX_INFO(FI_LOG_EP_DATA,
- "TX request processed: %p (OTX: %d)\n",
- req, atomic_get(&vc->outstanding_tx_reqs));
}
} else {
queue_tx = 1;
- GNIX_INFO(FI_LOG_EP_DATA,
+ GNIX_DEBUG(FI_LOG_EP_DATA,
"Queued request (%p) on busy VC\n",
req);
}
@@ -1888,11 +2111,11 @@ int _gnix_vc_queue_tx_req(struct gnix_fab_req *req)
/*
* TODO: for auto progress do something here
*/
- slist_insert_tail(&req->slist, &vc->tx_queue);
+ dlist_insert_tail(&req->dlist, &vc->tx_queue);
_gnix_vc_tx_schedule(vc);
}
- fastlock_release(&vc->tx_queue_lock);
+ COND_RELEASE(vc->ep->requires_lock, &vc->tx_queue_lock);
/*
* if injecting and queuing, push the tx queue for this vc
@@ -1907,22 +2130,13 @@ int _gnix_vc_queue_tx_req(struct gnix_fab_req *req)
}
}
- /*
- * don't treat -FI_EAGAIN as an error
- */
-
- if (rc == -FI_EAGAIN)
- rc = FI_SUCCESS;
-
- return rc;
+ return FI_SUCCESS;
}
/* Push TX requests queued on the VC. */
static int __gnix_vc_push_tx_reqs(struct gnix_vc *vc)
{
int ret, fi_rc = FI_SUCCESS;
- struct slist *list;
- struct slist_entry *item;
struct gnix_fab_req *req;
ret = __gnix_vc_connected(vc); /* 0 on success */
@@ -1933,18 +2147,12 @@ static int __gnix_vc_push_tx_reqs(struct gnix_vc *vc)
return -FI_EAGAIN;
}
- fastlock_acquire(&vc->tx_queue_lock);
-
- list = &vc->tx_queue;
- item = list->head;
- while (item != NULL) {
- req = (struct gnix_fab_req *)container_of(item,
- struct gnix_fab_req,
- slist);
-
+ COND_ACQUIRE(vc->ep->requires_lock, &vc->tx_queue_lock);
+ req = dlist_first_entry(&vc->tx_queue, struct gnix_fab_req, dlist);
+ while (req) {
if ((req->flags & FI_FENCE) &&
atomic_get(&vc->outstanding_tx_reqs)) {
- GNIX_INFO(FI_LOG_EP_DATA,
+ GNIX_DEBUG(FI_LOG_EP_DATA,
"TX request queue stalled on FI_FENCE request: %p (%d)\n",
req, atomic_get(&vc->outstanding_tx_reqs));
/* Success is returned to allow processing of more VCs.
@@ -1953,19 +2161,20 @@ static int __gnix_vc_push_tx_reqs(struct gnix_vc *vc)
break;
}
+ atomic_inc(&vc->outstanding_tx_reqs);
+ dlist_remove_init(&req->dlist);
+
ret = req->work_fn(req);
if (ret == FI_SUCCESS) {
- atomic_inc(&vc->outstanding_tx_reqs);
- GNIX_INFO(FI_LOG_EP_DATA,
+ GNIX_DEBUG(FI_LOG_EP_DATA,
"TX request processed: %p (OTX: %d)\n",
req, atomic_get(&vc->outstanding_tx_reqs));
- } else {
+ } else if (ret != -FI_ECANCELED) {
/* Work failed. Reschedule to put this VC
* back on the end of the list and return
* -FI_EAGAIN. */
- _gnix_vc_tx_schedule(vc);
- GNIX_INFO(FI_LOG_EP_DATA,
+ GNIX_DEBUG(FI_LOG_EP_DATA,
"Failed to push TX request %p: %s\n",
req, fi_strerror(-ret));
fi_rc = -FI_EAGAIN;
@@ -1981,16 +2190,24 @@ static int __gnix_vc_push_tx_reqs(struct gnix_vc *vc)
"Failed to push TX request %p: %s\n",
req, fi_strerror(-ret));
}
+
+ dlist_insert_head(&req->dlist, &vc->tx_queue);
+ atomic_dec(&vc->outstanding_tx_reqs);
+
+ /* _gnix_vc_tx_schedule() must come after the request
+ * is inserted into the VC's tx_queue. */
+ _gnix_vc_tx_schedule(vc);
+
break;
}
- slist_remove_head(&vc->tx_queue);
- item = list->head;
-
+ req = dlist_first_entry(&vc->tx_queue,
+ struct gnix_fab_req,
+ dlist);
/* Return success if the queue is emptied. */
}
- fastlock_release(&vc->tx_queue_lock);
+ COND_RELEASE(vc->ep->requires_lock, &vc->tx_queue_lock);
return fi_rc;
}
@@ -1999,11 +2216,11 @@ static struct gnix_vc *__gnix_nic_next_pending_tx_vc(struct gnix_nic *nic)
{
struct gnix_vc *vc = NULL;
- fastlock_acquire(&nic->tx_vc_lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->tx_vc_lock);
vc = dlist_first_entry(&nic->tx_vcs, struct gnix_vc, tx_list);
if (vc)
dlist_remove_init(&vc->tx_list);
- fastlock_release(&nic->tx_vc_lock);
+ COND_RELEASE(nic->requires_lock, &nic->tx_vc_lock);
if (vc) {
GNIX_INFO(FI_LOG_EP_CTRL, "Dequeued TX VC (%p)\n", vc);
@@ -2049,6 +2266,8 @@ int _gnix_vc_nic_progress(struct gnix_nic *nic)
*/
int _gnix_vc_schedule(struct gnix_vc *vc)
{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+
_gnix_vc_rx_schedule(vc);
__gnix_vc_work_schedule(vc);
_gnix_vc_tx_schedule(vc);
@@ -2056,99 +2275,18 @@ int _gnix_vc_schedule(struct gnix_vc *vc)
return FI_SUCCESS;
}
-static int __gnix_vc_ep_rdm_get_vc(struct gnix_fid_ep *ep, fi_addr_t dest_addr,
- struct gnix_vc **vc_ptr)
-{
- int ret = FI_SUCCESS;
- struct gnix_vc *vc = NULL, *vc_tmp;
- struct gnix_fid_av *av;
- struct gnix_av_addr_entry *av_entry;
- gnix_ht_key_t key;
-
- GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
-
- av = ep->av;
- assert(av != NULL);
-
- ret = _gnix_av_lookup(av, dest_addr, &av_entry);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_DATA,
- "_gnix_av_lookup for addr 0x%llx returned %s \n",
- dest_addr, fi_strerror(-ret));
- goto err;
- }
-
- GNIX_INFO(FI_LOG_EP_CTRL, "fi_addr_t: 0x%llx gnix_addr: 0x%llx\n",
- dest_addr, av_entry->gnix_addr);
-
- memcpy(&key, &av_entry->gnix_addr, sizeof(gnix_ht_key_t));
-
- fastlock_acquire(&ep->vc_ht_lock);
- vc = (struct gnix_vc *)_gnix_ht_lookup(ep->vc_ht,
- key);
- if (vc == NULL) {
- ret = _gnix_vc_alloc(ep,
- av_entry,
- &vc_tmp);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_DATA,
- "_gnix_vc_alloc returned %s\n",
- fi_strerror(-ret));
- goto err_w_lock;
- }
- ret = _gnix_ht_insert(ep->vc_ht, key,
- vc_tmp);
- if (likely(ret == FI_SUCCESS)) {
- vc = vc_tmp;
- vc->modes |= GNIX_VC_MODE_IN_HT;
- fastlock_release(&ep->vc_ht_lock);
- ret = _gnix_vc_connect(vc);
- if (ret != FI_SUCCESS) {
- GNIX_WARN(FI_LOG_EP_DATA,
- "_gnix_vc_connect returned %s\n",
- fi_strerror(-ret));
- goto err;
- }
- } else if (ret == -FI_ENOSPC) {
- _gnix_vc_destroy(vc_tmp);
- vc = _gnix_ht_lookup(ep->vc_ht, key);
- fastlock_release(&ep->vc_ht_lock);
- assert(vc != NULL);
- assert(vc->modes & GNIX_VC_MODE_IN_HT);
- ret = FI_SUCCESS;
- } else {
- GNIX_WARN(FI_LOG_EP_DATA,
- "_gnix_ht_insert returned %s\n",
- fi_strerror(-ret));
- goto err_w_lock;
- }
- } else {
- fastlock_release(&ep->vc_ht_lock);
- }
-
- *vc_ptr = vc;
- return ret;
-
-err_w_lock:
- fastlock_release(&ep->vc_ht_lock);
-err:
- if (vc != NULL)
- _gnix_vc_destroy(vc);
- return ret;
-}
-
int _gnix_vc_ep_get_vc(struct gnix_fid_ep *ep, fi_addr_t dest_addr,
- struct gnix_vc **vc_ptr)
+ struct gnix_vc **vc_ptr)
{
int ret;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
if (GNIX_EP_RDM_DGM(ep->type)) {
- ret = __gnix_vc_ep_rdm_get_vc(ep, dest_addr, vc_ptr);
+ ret = __gnix_vc_get_vc_by_fi_addr(ep, dest_addr, vc_ptr);
if (unlikely(ret != FI_SUCCESS)) {
GNIX_WARN(FI_LOG_EP_DATA,
- "__gnix_vc_ep_get_vc returned %s\n",
+ "__gnix_vc_get_vc_by_fi_addr returned %s\n",
fi_strerror(-ret));
return ret;
}
@@ -2195,7 +2333,7 @@ int _gnix_vc_cm_init(struct gnix_cm_nic *cm_nic)
nic = cm_nic->nic;
assert(nic != NULL);
- fastlock_acquire(&nic->lock);
+ COND_ACQUIRE(nic->requires_lock, &nic->lock);
ret = _gnix_cm_nic_reg_recv_fn(cm_nic,
__gnix_vc_recv_fn,
&ofunc);
@@ -2205,8 +2343,7 @@ int _gnix_vc_cm_init(struct gnix_cm_nic *cm_nic)
fi_strerror(-ret));
}
- fastlock_release(&nic->lock);
+ COND_RELEASE(nic->requires_lock, &nic->lock);
return ret;
}
-
diff --git a/prov/gni/src/gnix_vector.c b/prov/gni/src/gnix_vector.c
index 2fa58a8..e85ae76 100644
--- a/prov/gni/src/gnix_vector.c
+++ b/prov/gni/src/gnix_vector.c
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2015-2016 Los Alamos National Security, LLC.
* All rights reserved.
+ * Copyright (c) 2016 Cray Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -36,13 +37,12 @@
static gnix_vector_ops_t __gnix_vec_lockless_ops;
static gnix_vector_ops_t __gnix_vec_locked_ops;
-
/*******************************************************************************
* INTERNAL HELPER FNS
******************************************************************************/
-static inline uint64_t __gnix_vec_get_new_size(gnix_vector_t *vec, uint64_t index)
+static inline uint32_t __gnix_vec_get_new_size(gnix_vector_t *vec, uint32_t index)
{
- uint64_t new_size = vec->attr.cur_size;
+ uint32_t new_size = vec->attr.cur_size;
if (vec->attr.vec_increase_type == GNIX_VEC_INCREASE_ADD) {
do {
@@ -68,9 +68,24 @@ static inline uint64_t __gnix_vec_get_new_size(gnix_vector_t *vec, uint64_t inde
return new_size;
}
-static inline int __gnix_vec_resize(gnix_vector_t *vec, uint64_t new_size)
+static inline void __gnix_vec_close_entries(gnix_vector_t *vec)
{
- void *tmp = realloc(vec->vector, new_size * sizeof(gnix_vec_entry_t));
+ memset(vec->vector, 0, (sizeof(gnix_vec_entry_t) * vec->attr.cur_size));
+}
+
+/*******************************************************************************
+ * INTERNAL WORKER FNS
+ ******************************************************************************/
+static inline int __gnix_vec_resize(gnix_vector_t *vec, uint32_t new_size)
+{
+ void *tmp;
+
+ if (new_size <= vec->attr.cur_size) {
+ GNIX_WARN(FI_LOG_EP_DATA, "In __gnix_vec_resize, the new vector"
+ "size is less than or equal to the current size.\n");
+ }
+
+ tmp = realloc(vec->vector, new_size * sizeof(gnix_vec_entry_t));
if (!tmp) {
GNIX_WARN(FI_LOG_EP_CTRL, "Insufficient memory in "
@@ -92,6 +107,12 @@ static inline int __gnix_vec_resize(gnix_vector_t *vec, uint64_t new_size)
static inline int __gnix_vec_create(gnix_vector_t *vec, gnix_vec_attr_t *attr)
{
+ if (unlikely(vec->state == GNIX_VEC_STATE_READY)) {
+ GNIX_DEBUG(FI_LOG_EP_DATA, "The vector (%p) is already ready.\n",
+ vec);
+ return -FI_EINVAL;
+ }
+
vec->vector = calloc(attr->vec_initial_size, sizeof(gnix_vec_entry_t));
if (unlikely(!vec->vector)) {
@@ -107,39 +128,29 @@ static inline int __gnix_vec_create(gnix_vector_t *vec, gnix_vec_attr_t *attr)
return FI_SUCCESS;
}
-static inline void __gnix_vec_close_entries(gnix_vector_t *vec)
+static inline int __gnix_vec_close(gnix_vector_t *vec)
{
- memset(vec->vector, 0, (sizeof(gnix_vec_entry_t) * vec->attr.cur_size));
-}
+ if (unlikely(vec->state == GNIX_VEC_STATE_DEAD)) {
+ GNIX_DEBUG(FI_LOG_EP_DATA, "The vector (%p) is already dead.\n",
+ vec);
+ return -FI_EINVAL;
+ }
-static inline void __gnix_vec_close(gnix_vector_t *vec)
-{
free(vec->vector);
vec->ops = NULL;
vec->attr.cur_size = 0;
vec->state = GNIX_VEC_STATE_DEAD;
-}
-
-/*******************************************************************************
- * INLINE OPS FNS
- ******************************************************************************/
-inline int _gnix_vec_insert_first(gnix_vector_t *vec, gnix_vec_entry_t *entry)
-{
- return _gnix_vec_insert_at(vec, entry, 0);
-}
-inline int _gnix_vec_insert_last(gnix_vector_t *vec, gnix_vec_entry_t *entry)
-{
- return _gnix_vec_insert_at(vec, entry, vec->attr.cur_size-1);
+ return FI_SUCCESS;
}
-inline int _gnix_vec_insert_at(gnix_vector_t *vec, gnix_vec_entry_t *entry,
- gnix_vec_index_t index)
+static inline int __gnix_vec_insert_at(gnix_vector_t *vec,
+ gnix_vec_entry_t *entry,
+ gnix_vec_index_t index)
{
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
- if (unlikely(!vec || !entry ||
- index >= vec->attr.vec_maximum_size)) {
+ if (unlikely(index >= vec->attr.vec_maximum_size)) {
GNIX_WARN(FI_LOG_EP_CTRL, "Invalid parameter to "
"__gnix_vec_insert_at\n");
return -FI_EINVAL;
@@ -147,11 +158,11 @@ inline int _gnix_vec_insert_at(gnix_vector_t *vec, gnix_vec_entry_t *entry,
if (unlikely(vec->state == GNIX_VEC_STATE_DEAD)) {
GNIX_FATAL(FI_LOG_EP_CTRL, "gnix_vector_t is in state "
- "GNIX_VEC_STATE_DEAD in _gnix_vec_insert_at.\n");
+ "GNIX_VEC_STATE_DEAD in __gnix_vec_insert_at.\n");
}
if (index >= vec->attr.cur_size) {
- uint64_t new_size = __gnix_vec_get_new_size(vec, index);
+ uint32_t new_size = __gnix_vec_get_new_size(vec, index);
int ret = __gnix_vec_resize(vec, new_size);
if (unlikely(ret))
@@ -160,7 +171,7 @@ inline int _gnix_vec_insert_at(gnix_vector_t *vec, gnix_vec_entry_t *entry,
if (vec->vector[index]) {
GNIX_WARN(FI_LOG_EP_CTRL, "Existing element found in "
- "__gnix_vec_insert_last\n");
+ "__gnix_vec_insert_at\n");
return -FI_ECANCELED;
} else {
vec->vector[index] = entry;
@@ -168,27 +179,16 @@ inline int _gnix_vec_insert_at(gnix_vector_t *vec, gnix_vec_entry_t *entry,
}
}
-inline int _gnix_vec_remove_first(gnix_vector_t *vec)
-{
- return _gnix_vec_remove_at(vec, 0);
-}
-
-inline int _gnix_vec_remove_last(gnix_vector_t *vec)
+static inline int __gnix_vec_remove_at(gnix_vector_t *vec,
+ gnix_vec_index_t index)
{
- return _gnix_vec_remove_at(vec, vec->attr.cur_size-1);
-}
-
-inline int _gnix_vec_remove_at(gnix_vector_t *vec, gnix_vec_index_t index)
-{
- GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
-
- if (unlikely(!vec || index >= vec->attr.cur_size)) {
- GNIX_WARN(FI_LOG_EP_CTRL, "Invalid parameter to "
- "__gnix_vec_remove_at\n");
- return -FI_EINVAL;
- } else if (unlikely(vec->state == GNIX_VEC_STATE_DEAD)) {
+ if (unlikely(vec->state == GNIX_VEC_STATE_DEAD)) {
GNIX_FATAL(FI_LOG_EP_CTRL, "gnix_vector_t is in state "
- "GNIX_VEC_STATE_DEAD in _gnix_vec_remove_at.\n");
+ "GNIX_VEC_STATE_DEAD in __gnix_vec_remove_at.\n");
+ } else if (index >= vec->attr.cur_size) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "Index (%lu) too large in "
+ "__gnix_vec_remove_at\n", index);
+ return -FI_EINVAL;
} else {
if (!vec->vector[index]) {
GNIX_WARN(FI_LOG_EP_CTRL, "No entry exists in "
@@ -201,36 +201,24 @@ inline int _gnix_vec_remove_at(gnix_vector_t *vec, gnix_vec_index_t index)
return FI_SUCCESS;
}
-inline int _gnix_vec_first(gnix_vector_t *vec, void **element)
+static inline int __gnix_vec_at(gnix_vector_t *vec, void **element,
+ gnix_vec_index_t index)
{
- return _gnix_vec_at(vec, element, 0);
-}
-
-inline int _gnix_vec_last(gnix_vector_t *vec, void **element)
-{
- return _gnix_vec_at(vec, element, vec->attr.cur_size-1);
-}
-
-inline int _gnix_vec_at(gnix_vector_t *vec, void **element, gnix_vec_index_t index)
-{
- GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
-
- if (unlikely(!vec || index > vec->attr.cur_size || !element)) {
- GNIX_WARN(FI_LOG_EP_CTRL, "Invalid parameter to "
- "_gnix_vec_at\n");
- return -FI_EINVAL;
- } else if (unlikely(vec->state == GNIX_VEC_STATE_DEAD)) {
+ if (unlikely(vec->state == GNIX_VEC_STATE_DEAD)) {
GNIX_FATAL(FI_LOG_EP_CTRL, "gnix_vector_t is in state "
- "GNIX_VEC_STATE_DEAD in _gnix_vec_at.\n");
+ "GNIX_VEC_STATE_DEAD in __gnix_vec_at.\n");
+ } else if (index >= vec->attr.cur_size) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "Index (%lu) too large in "
+ "__gnix_vec_at\n", index);
+ return -FI_EINVAL;
} else {
if (likely((uint64_t) vec->vector[index])) {
*element = vec->vector[index];
} else {
- GNIX_WARN(FI_LOG_EP_CTRL, "There is no element at index "
- "%lu in _gnix_vec_at\n", index);
+ GNIX_DEBUG(FI_LOG_EP_CTRL, "There is no element at index "
+ "(%lu) in __gnix_vec_at\n", index);
return -FI_ECANCELED;
}
-
}
return FI_SUCCESS;
}
@@ -251,65 +239,66 @@ static int __gnix_vec_lf_init(gnix_vector_t *vec, gnix_vec_attr_t *attr)
static int __gnix_vec_lf_close(gnix_vector_t *vec)
{
+ int ret;
+
__gnix_vec_close_entries(vec);
- __gnix_vec_close(vec);
+ ret = __gnix_vec_close(vec);
- return FI_SUCCESS;
+ return ret;
}
-static int __gnix_vec_lf_resize(gnix_vector_t *vec, uint64_t size)
+static int __gnix_vec_lf_resize(gnix_vector_t *vec, uint32_t size)
{
return __gnix_vec_resize(vec, size);
}
-static int __gnix_vec_lf_insert_first(gnix_vector_t *vec,
- gnix_vec_entry_t *entry)
-{
- return _gnix_vec_insert_first(vec, entry);
-}
-
static int __gnix_vec_lf_insert_last(gnix_vector_t *vec,
gnix_vec_entry_t *entry)
{
- return _gnix_vec_insert_last(vec, entry);
+ return __gnix_vec_insert_at(vec, entry, vec->attr.cur_size - 1);
}
static int __gnix_vec_lf_insert_at(gnix_vector_t *vec,
gnix_vec_entry_t *entry,
gnix_vec_index_t index)
{
- return _gnix_vec_insert_at(vec, entry, index);
-}
-
-static int __gnix_vec_lf_remove_first(gnix_vector_t *vec)
-{
- return _gnix_vec_remove_first(vec);
+ return __gnix_vec_insert_at(vec, entry, index);
}
static int __gnix_vec_lf_remove_last(gnix_vector_t *vec)
{
- return _gnix_vec_remove_last(vec);
+ return __gnix_vec_remove_at(vec, vec->attr.cur_size - 1);
}
static int __gnix_vec_lf_remove_at(gnix_vector_t *vec,
gnix_vec_index_t index)
{
- return _gnix_vec_remove_at(vec, index);
+ return __gnix_vec_remove_at(vec, index);
}
-static int __gnix_vec_lf_first(gnix_vector_t *vec, void **element)
+static int __gnix_vec_lf_last(gnix_vector_t *vec, void **element)
{
- return _gnix_vec_first(vec, element);
+ return __gnix_vec_at(vec, element, vec->attr.cur_size - 1);
}
-static int __gnix_vec_lf_last(gnix_vector_t *vec, void **element)
+static int __gnix_vec_lf_at(gnix_vector_t *vec, void **element, gnix_vec_index_t index)
{
- return _gnix_vec_last(vec, element);
+ return __gnix_vec_at(vec, element, index);
}
-static int __gnix_vec_lf_at(gnix_vector_t *vec, void **element, gnix_vec_index_t index)
+gnix_vec_entry_t *__gnix_vec_lf_iter_next(struct gnix_vector_iter *iter)
{
- return _gnix_vec_at(vec, element, index);
+ uint32_t i;
+
+ for (i = iter->cur_idx; i < iter->vec->attr.cur_size; i++) {
+ if (iter->vec->vector[i]) {
+ iter->cur_idx = i + 1;
+ return iter->vec->vector[i];
+ }
+ }
+
+ iter->cur_idx = iter->vec->attr.cur_size;
+ return NULL;
}
/*******************************************************************************
@@ -329,39 +318,27 @@ static int __gnix_vec_lk_init(gnix_vector_t *vec, gnix_vec_attr_t *attr)
static int __gnix_vec_lk_close(gnix_vector_t *vec)
{
+ int ret;
+
rwlock_wrlock(&vec->lock);
__gnix_vec_close_entries(vec);
- __gnix_vec_close(vec);
+ ret = __gnix_vec_close(vec);
rwlock_unlock(&vec->lock);
rwlock_destroy(&vec->lock);
- return FI_SUCCESS;
-}
-
-static int __gnix_vec_lk_resize(gnix_vector_t *vec, uint64_t size)
-{
- int ret;
-
- rwlock_wrlock(&vec->lock);
-
- ret = __gnix_vec_resize(vec, size);
-
- rwlock_unlock(&vec->lock);
-
return ret;
}
-static int __gnix_vec_lk_insert_first(gnix_vector_t *vec,
- gnix_vec_entry_t *entry)
+static int __gnix_vec_lk_resize(gnix_vector_t *vec, uint32_t size)
{
int ret;
rwlock_wrlock(&vec->lock);
- ret = _gnix_vec_insert_first(vec, entry);
+ ret = __gnix_vec_resize(vec, size);
rwlock_unlock(&vec->lock);
@@ -375,7 +352,7 @@ static int __gnix_vec_lk_insert_last(gnix_vector_t *vec,
rwlock_wrlock(&vec->lock);
- ret = _gnix_vec_insert_last(vec, entry);
+ ret = __gnix_vec_insert_at(vec, entry, vec->attr.cur_size - 1);
rwlock_unlock(&vec->lock);
@@ -390,20 +367,7 @@ static int __gnix_vec_lk_insert_at(gnix_vector_t *vec,
rwlock_wrlock(&vec->lock);
- ret = _gnix_vec_insert_at(vec, entry, index);
-
- rwlock_unlock(&vec->lock);
-
- return ret;
-}
-
-static int __gnix_vec_lk_remove_first(gnix_vector_t *vec)
-{
- int ret;
-
- rwlock_wrlock(&vec->lock);
-
- ret = _gnix_vec_remove_first(vec);
+ ret = __gnix_vec_insert_at(vec, entry, index);
rwlock_unlock(&vec->lock);
@@ -416,7 +380,7 @@ static int __gnix_vec_lk_remove_last(gnix_vector_t *vec)
rwlock_wrlock(&vec->lock);
- ret = _gnix_vec_remove_last(vec);
+ ret = __gnix_vec_remove_at(vec, vec->attr.cur_size - 1);
rwlock_unlock(&vec->lock);
@@ -430,52 +394,66 @@ static int __gnix_vec_lk_remove_at(gnix_vector_t *vec,
rwlock_wrlock(&vec->lock);
- ret = _gnix_vec_remove_at(vec, index);
+ ret = __gnix_vec_remove_at(vec, index);
rwlock_unlock(&vec->lock);
return ret;
}
-static int __gnix_vec_lk_first(gnix_vector_t *vec, void **element)
+static int __gnix_vec_lk_last(gnix_vector_t *vec, void **element)
{
int ret;
rwlock_rdlock(&vec->lock);
- ret = _gnix_vec_first(vec, element);
+ ret = __gnix_vec_at(vec, element, vec->attr.cur_size - 1);
rwlock_unlock(&vec->lock);
return ret;
}
-static int __gnix_vec_lk_last(gnix_vector_t *vec, void **element)
+static int __gnix_vec_lk_at(gnix_vector_t *vec, void **element, gnix_vec_index_t index)
{
int ret;
rwlock_rdlock(&vec->lock);
- ret = _gnix_vec_last(vec, element);
+ ret = __gnix_vec_at(vec, element, index);
rwlock_unlock(&vec->lock);
return ret;
}
-static int __gnix_vec_lk_at(gnix_vector_t *vec, void **element, gnix_vec_index_t index)
+gnix_vec_entry_t *__gnix_vec_lk_iter_next(struct gnix_vector_iter *iter)
{
- int ret;
+ uint32_t i;
+ gnix_vec_entry_t *entry;
- rwlock_rdlock(&vec->lock);
+ rwlock_rdlock(&iter->vec->lock);
- ret = _gnix_vec_at(vec, element, index);
+ for (i = iter->cur_idx; i < iter->vec->attr.cur_size; i++) {
+ if (iter->vec->vector[i]) {
+ iter->cur_idx = i + 1;
+ entry = iter->vec->vector[i];
+ rwlock_unlock(&iter->vec->lock);
- rwlock_unlock(&vec->lock);
+ return entry;
+ }
+ }
- return ret;
+ iter->cur_idx = iter->vec->attr.cur_size;
+
+ rwlock_unlock(&iter->vec->lock);
+
+ return NULL;
}
+/*******************************************************************************
+ * API FNS
+ ******************************************************************************/
/**
* Create the initial vector. The user is responsible for initializing the
* "attr" parameter prior to calling this function.
@@ -496,9 +474,6 @@ int _gnix_vec_init(struct gnix_vector *vec, gnix_vec_attr_t *attr)
GNIX_WARN(FI_LOG_EP_CTRL, "Invalid parameter to _gnix_vec_init."
"\n");
return -FI_EINVAL;
- } else if (unlikely(vec->state == GNIX_VEC_STATE_READY)) {
- GNIX_FATAL(FI_LOG_EP_CTRL, "gnix_vector_t is in state "
- "GNIX_VEC_STATE_READY in _gnix_vec_init.\n");
}
if (attr->vec_internal_locking == GNIX_VEC_LOCKED) {
@@ -524,9 +499,6 @@ int _gnix_vec_close(gnix_vector_t *vec)
GNIX_WARN(FI_LOG_EP_CTRL, "Invalid parameter to _gnix_vec_close."
"\n");
return -FI_EINVAL;
- } else if (unlikely(vec->state == GNIX_VEC_STATE_DEAD)) {
- GNIX_FATAL(FI_LOG_EP_CTRL, "gnix_vector_t is in state "
- "GNIX_VEC_STATE_DEAD in _gnix_vec_close.\n");
} else {
if (vec->attr.vec_internal_locking == GNIX_VEC_LOCKED) {
return __gnix_vec_lk_close(vec);
@@ -536,47 +508,32 @@ int _gnix_vec_close(gnix_vector_t *vec)
}
}
-int _gnix_vec_resize(gnix_vector_t *vec, uint64_t size)
-{
- GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
-
- if (unlikely(!vec || !size)) {
- GNIX_WARN(FI_LOG_EP_CTRL, "Invalid parameter to _gnix_vec_resize."
- "\n");
- return -FI_EINVAL;
- } else {
- if (vec->attr.vec_internal_locking == GNIX_VEC_LOCKED) {
- return __gnix_vec_lk_resize(vec, size);
- } else {
- return __gnix_vec_lf_resize(vec, size);
- }
- }
-}
-
static gnix_vector_ops_t __gnix_vec_lockless_ops = {
- .insert_first = __gnix_vec_lf_insert_first,
+ .resize = __gnix_vec_lf_resize,
+
.insert_last = __gnix_vec_lf_insert_last,
.insert_at = __gnix_vec_lf_insert_at,
- .remove_first = __gnix_vec_lf_remove_first,
.remove_last = __gnix_vec_lf_remove_last,
.remove_at = __gnix_vec_lf_remove_at,
- .first = __gnix_vec_lf_first,
.last = __gnix_vec_lf_last,
.at = __gnix_vec_lf_at,
+
+ .iter_next = __gnix_vec_lf_iter_next,
};
static gnix_vector_ops_t __gnix_vec_locked_ops = {
- .insert_first = __gnix_vec_lk_insert_first,
+ .resize = __gnix_vec_lk_resize,
+
.insert_last = __gnix_vec_lk_insert_last,
.insert_at = __gnix_vec_lk_insert_at,
- .remove_first = __gnix_vec_lk_remove_first,
.remove_last = __gnix_vec_lk_remove_last,
.remove_at = __gnix_vec_lk_remove_at,
- .first = __gnix_vec_lk_first,
.last = __gnix_vec_lk_last,
.at = __gnix_vec_lk_at,
+
+ .iter_next = __gnix_vec_lk_iter_next,
};
diff --git a/prov/gni/src/gnix_wait.c b/prov/gni/src/gnix_wait.c
index 5f92463..232c5e6 100644
--- a/prov/gni/src/gnix_wait.c
+++ b/prov/gni/src/gnix_wait.c
@@ -190,8 +190,6 @@ static int gnix_verify_wait_attr(struct fi_wait_attr *attr)
static int gnix_init_wait_obj(struct gnix_fid_wait *wait, enum fi_wait_obj type)
{
- long flags = 0;
-
GNIX_TRACE(WAIT_SUB, "\n");
wait->type = type;
@@ -201,8 +199,7 @@ static int gnix_init_wait_obj(struct gnix_fid_wait *wait, enum fi_wait_obj type)
if (socketpair(AF_LOCAL, SOCK_STREAM, 0, wait->fd))
goto err;
- fcntl(wait->fd[WAIT_READ], F_GETFL, &flags);
- if (fcntl(wait->fd[WAIT_READ], F_SETFL, flags | O_NONBLOCK))
+ if (fi_fd_nonblock(wait->fd[WAIT_READ]))
goto cleanup;
break;
case FI_WAIT_MUTEX_COND:
diff --git a/prov/gni/src/gnix_xpmem.c b/prov/gni/src/gnix_xpmem.c
new file mode 100644
index 0000000..5206bd4
--- /dev/null
+++ b/prov/gni/src/gnix_xpmem.c
@@ -0,0 +1,611 @@
+/*
+ * Copyright (c) 2016 Los Alamos National Security, LLC.
+ * All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * opyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include "gnix.h"
+#include "gnix_mr.h"
+#include "gnix_hashtable.h"
+#include "gnix_xpmem.h"
+
+
+#if HAVE_XPMEM
+
+bool gnix_xpmem_disabled = false;
+
+#define XPMEM_PAGE_SIZE 4096
+
+static pthread_mutex_t gnix_xpmem_lock = PTHREAD_MUTEX_INITIALIZER;
+static xpmem_segid_t gnix_my_segid;
+static int gnix_xpmem_ref_cnt;
+
+static void *__gnix_xpmem_attach_seg(void *handle,
+ void *address,
+ size_t length,
+ struct _gnix_fi_reg_context *,
+ void *context);
+
+static int __gnix_xpmem_detach_seg(void *handle,
+ void *context);
+
+static int __gnix_xpmem_destroy_mr_cache(void *context);
+
+struct gnix_xpmem_ht_entry {
+ struct gnix_mr_cache *mr_cache;
+ struct gnix_xpmem_handle *xp_hndl;
+ xpmem_apid_t apid;
+};
+
+/*
+ * TODO: should be adjustable from domain params
+ * Note notifier is set to NULL since xpmem device driver
+ * handles mmu notifiers internally so we don't need to use
+ * KDREG.
+ */
+static gnix_mr_cache_attr_t _gnix_xpmem_default_mr_cache_attr = {
+ .soft_reg_limit = 128,
+ .hard_reg_limit = 16384,
+ .hard_stale_limit = 128,
+ .lazy_deregistration = 1,
+ .reg_callback = __gnix_xpmem_attach_seg,
+ .dereg_callback = __gnix_xpmem_detach_seg,
+ .destruct_callback = __gnix_xpmem_destroy_mr_cache,
+ .elem_size = sizeof(struct gnix_xpmem_access_handle),
+ .notifier = NULL,
+};
+
+/*******************************************************************************
+ * INTERNAL HELPER FNS
+ ******************************************************************************/
+
+static void __xpmem_hndl_destruct(void *obj)
+{
+ int __attribute__((unused)) ret;
+ struct gnix_xpmem_handle *hndl = (struct gnix_xpmem_handle *) obj;
+
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+
+ ret = _gnix_ht_destroy(hndl->apid_ht);
+ if (ret == FI_SUCCESS) {
+ free(hndl->apid_ht);
+ hndl->apid_ht = NULL;
+ } else {
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gnix_ht_destroy returned %s\n",
+ fi_strerror(-ret));
+ }
+
+ pthread_mutex_lock(&gnix_xpmem_lock);
+
+ gnix_xpmem_ref_cnt--;
+ /*
+ * if refcnt drops to zero for entire xpmem use, remove
+ * this process' segment from xpmem.
+ */
+ if (gnix_xpmem_ref_cnt == 0) {
+ ret = xpmem_remove(gnix_my_segid);
+ if (ret)
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "xpmem_remove returned error %s\n",
+ strerror(errno));
+ }
+
+ pthread_mutex_unlock(&gnix_xpmem_lock);
+
+ free(hndl);
+}
+
+static void __gnix_xpmem_destroy_ht_entry(void *val)
+{
+ int __attribute__((unused)) ret;
+ struct gnix_xpmem_ht_entry *entry = (struct gnix_xpmem_ht_entry *)val;
+
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
+ ret = _gnix_mr_cache_destroy(entry->mr_cache);
+ if (ret != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_CTRL,
+ "_gnix_mr_cache_destroy returned error %s\n",
+ fi_strerror(-ret));
+
+ xpmem_release(entry->apid);
+ free(entry);
+}
+
+static void *__gnix_xpmem_attach_seg(void *handle,
+ void *address,
+ size_t length,
+ struct _gnix_fi_reg_context *reg_context,
+ void *context)
+{
+ struct gnix_xpmem_access_handle *access_hndl =
+ (struct gnix_xpmem_access_handle *)handle;
+ struct gnix_xpmem_ht_entry *entry = context;
+ struct xpmem_addr xpmem_addr;
+ size_t top, attach_len;
+
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
+ xpmem_addr.apid = entry->apid;
+
+ /*
+ * xpmem requires page aligned addresses for attach operation
+ */
+ xpmem_addr.offset = (off_t) FLOOR((uint64_t)address, XPMEM_PAGE_SIZE);
+ top = CEILING(((uint64_t)address + length), XPMEM_PAGE_SIZE);
+ attach_len = top - FLOOR((uint64_t)address, XPMEM_PAGE_SIZE);
+
+ access_hndl->attach_addr = xpmem_attach(xpmem_addr,
+ attach_len,
+ NULL);
+ if (access_hndl->attach_addr != (void *)-1L) {
+ access_hndl->xp_hndl = entry->xp_hndl;
+ _gnix_ref_get(entry->xp_hndl);
+ access_hndl->remote_base_addr = (void *)xpmem_addr.offset;
+ access_hndl->access_len = attach_len;
+ access_hndl->entry = entry;
+ return handle;
+ } else {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "xpmem_attach returned %s xpmem_addr %ld:0x%016lx len %d\n",
+ strerror(errno), xpmem_addr.apid, xpmem_addr.offset,
+ attach_len);
+ /* TODO: dump /proc/self/maps ? */
+ exit(-1);
+ return NULL;
+ }
+}
+
+static int __gnix_xpmem_detach_seg(void *handle, void *context)
+{
+ int ret;
+ struct gnix_xpmem_access_handle *access_hndl;
+
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
+ access_hndl = (struct gnix_xpmem_access_handle *)handle;
+ assert(access_hndl);
+
+ ret = xpmem_detach(access_hndl->attach_addr);
+ if (ret)
+ GNIX_WARN(FI_LOG_EP_DATA, "xpmem_detach returned %s\n",
+ strerror(errno));
+ _gnix_ref_put(access_hndl->xp_hndl);
+ return ret;
+}
+
+/*
+ * TODO: do we need a destructor callback for mr cache?
+ */
+static int __gnix_xpmem_destroy_mr_cache(void *context)
+{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+ /*
+ * use iterator over mr cache entries and invoke
+ * xpmem_detach on each
+ */
+ return FI_SUCCESS;
+}
+
+/*******************************************************************************
+ * the stuff
+ ******************************************************************************/
+
+int _gnix_xpmem_handle_create(struct gnix_fid_domain *dom,
+ struct gnix_xpmem_handle **handle)
+{
+ int ret = FI_SUCCESS;
+ struct gnix_xpmem_handle *hndl = NULL;
+ struct gnix_hashtable_attr ht_attr = {0};
+
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+
+ hndl = calloc(1, sizeof *hndl);
+ if (!hndl)
+ return -FI_ENOMEM;
+
+ pthread_mutex_lock(&gnix_xpmem_lock);
+
+ if (gnix_xpmem_ref_cnt == 0) {
+ gnix_my_segid = xpmem_make(0, XPMEM_MAXADDR_SIZE,
+ XPMEM_PERMIT_MODE,
+ (void *)0666);
+ if (gnix_my_segid == -1L) {
+ GNIX_WARN(FI_LOG_DOMAIN, "xpmem make failed - %s\n",
+ strerror(errno));
+ ret = -errno;
+ pthread_mutex_unlock(&gnix_xpmem_lock);
+ goto exit;
+ }
+
+ gnix_xpmem_ref_cnt++;
+ }
+
+ pthread_mutex_unlock(&gnix_xpmem_lock);
+
+ _gnix_ref_init(&hndl->ref_cnt, 1,
+ __xpmem_hndl_destruct);
+ fastlock_init(&hndl->lock);
+
+ /*
+ * initialize xpmem_apid_t key'd hash table for
+ * retrieving r/b tree for that apid
+ */
+
+ hndl->apid_ht = calloc(1, sizeof(struct gnix_hashtable));
+ if (hndl->apid_ht == NULL)
+ goto exit;
+
+ /*
+ * TODO: use domain parameters to adjust these
+ */
+
+ ht_attr.ht_initial_size = 1024; /* will we ever have more than
+ this many local processes? */
+ ht_attr.ht_maximum_size = 1024 * 1024;
+ ht_attr.ht_increase_step = 1024;
+ ht_attr.ht_increase_type = GNIX_HT_INCREASE_MULT;
+ ht_attr.ht_collision_thresh = 500;
+ ht_attr.ht_hash_seed = 0xdeadbeefbeefdead;
+ ht_attr.ht_internal_locking = 0;
+ ht_attr.destructor = __gnix_xpmem_destroy_ht_entry;
+
+ ret = _gnix_ht_init(hndl->apid_ht,
+ &ht_attr);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_CTRL, "_gnix_ht_init returned %s\n",
+ fi_strerror(-ret));
+ goto exit;
+ }
+
+ *handle = hndl;
+ return ret;
+
+exit:
+ if (hndl != NULL) {
+ if (hndl->apid_ht != NULL)
+ free(hndl->apid_ht);
+ free(hndl);
+ }
+
+ return ret;
+}
+
+int _gnix_xpmem_handle_destroy(struct gnix_xpmem_handle *hndl)
+{
+ int ret = FI_SUCCESS;
+
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+
+ _gnix_ref_put(hndl);
+
+ return ret;
+}
+
+
+int _gnix_xpmem_access_hndl_get(struct gnix_xpmem_handle *xp_hndl,
+ xpmem_apid_t peer_apid,
+ uint64_t remote_vaddr,
+ size_t len,
+ struct gnix_xpmem_access_handle **access_hndl)
+{
+ int ret = FI_SUCCESS;
+ struct gnix_xpmem_ht_entry *entry;
+ gnix_mr_cache_attr_t mr_cache_attr = {0};
+
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
+ /*
+ * use peer_apid to look up the reg cache
+ * - if not in the hash, create and insert
+ */
+
+ fastlock_acquire(&xp_hndl->lock);
+
+ entry = _gnix_ht_lookup(xp_hndl->apid_ht,
+ (gnix_ht_key_t)peer_apid);
+
+ /*
+ * okay need to create an mr_cache for this apid
+ */
+ if (unlikely(entry == NULL)) {
+
+ entry = calloc(1, sizeof *entry);
+ if (entry == NULL) {
+ ret = -FI_ENOMEM;
+ goto exit_w_lock;
+ }
+
+ entry->apid = peer_apid;
+ entry->xp_hndl = xp_hndl;
+
+ memcpy(&mr_cache_attr, &_gnix_xpmem_default_mr_cache_attr,
+ sizeof(gnix_mr_cache_attr_t));
+ mr_cache_attr.reg_context = entry;
+ mr_cache_attr.dereg_context = entry;
+ mr_cache_attr.destruct_context = entry;
+ ret = _gnix_mr_cache_init(&entry->mr_cache,
+ &mr_cache_attr);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_mr_cache_init returned %s\n",
+ fi_strerror(-ret));
+ goto exit_w_lock;
+ }
+ ret = _gnix_ht_insert(xp_hndl->apid_ht,
+ (gnix_ht_key_t)peer_apid,
+ entry);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_ht_insert returned %s\n",
+ fi_strerror(-ret));
+ goto exit_w_lock;
+ }
+ }
+
+ ret = _gnix_mr_cache_register(entry->mr_cache,
+ remote_vaddr,
+ len,
+ NULL,
+ (void **)access_hndl);
+ if (ret != FI_SUCCESS) {
+ GNIX_WARN(FI_LOG_EP_DATA,
+ "_gnix_mr_cache_register returned %s\n",
+ fi_strerror(-ret));
+ goto exit_w_lock;
+ }
+
+exit_w_lock:
+ fastlock_release(&xp_hndl->lock);
+ return ret;
+
+}
+
+int _gnix_xpmem_access_hndl_put(struct gnix_xpmem_access_handle *access_hndl)
+{
+ int ret = FI_SUCCESS;
+ struct gnix_xpmem_ht_entry *entry;
+ struct gnix_xpmem_handle *xp_hndl;
+
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
+ entry = access_hndl->entry;
+ if (!entry) {
+ GNIX_WARN(FI_LOG_EP_DATA, "entry is null\n");
+ return -FI_EINVAL;
+ }
+
+ xp_hndl = entry->xp_hndl;
+ if (!xp_hndl) {
+ GNIX_WARN(FI_LOG_EP_DATA, "entry->xp_hndl is null\n");
+ return -FI_EINVAL;
+ }
+
+ fastlock_acquire(&xp_hndl->lock);
+
+ ret = _gnix_mr_cache_deregister(entry->mr_cache,
+ access_hndl);
+ if (ret != FI_SUCCESS)
+ GNIX_WARN(FI_LOG_EP_DATA, "_gnix_mr_cache_deregister returned %s\n",
+ fi_strerror(-ret));
+
+ fastlock_release(&xp_hndl->lock);
+
+ return ret;
+}
+
+int _gnix_xpmem_accessible(struct gnix_fid_ep *ep,
+ struct gnix_address addr,
+ bool *accessible)
+{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+
+ if (!ep || !accessible)
+ return -FI_EINVAL;
+
+ if (gnix_xpmem_disabled == true) {
+ *accessible = false;
+ return FI_SUCCESS;
+ }
+
+ if (ep->domain->params.xpmem_enabled == false) {
+ *accessible = false;
+ return FI_SUCCESS;
+ }
+
+ /*
+ * if the endpoint's device_addr is the same as tht
+ * of the supplied address, return true, else false
+ */
+
+ *accessible = (ep->my_name.gnix_addr.device_addr ==
+ addr.device_addr) ? true : false;
+
+ return FI_SUCCESS;
+}
+
+int _gnix_xpmem_copy(struct gnix_xpmem_access_handle *access_hndl,
+ void *dst_addr,
+ void *remote_start_addr,
+ size_t len)
+{
+ void *local_start_addr, *remote_base_addr;
+ uint64_t attach_addr, reg_len;
+
+ GNIX_TRACE(FI_LOG_EP_DATA, "\n");
+
+ if (!access_hndl)
+ return -FI_EINVAL;
+
+ attach_addr = (uint64_t)access_hndl->attach_addr;
+ remote_base_addr = access_hndl->remote_base_addr;
+ reg_len = access_hndl->access_len;
+
+ /*
+ * check that the access handle limits and the
+ * copy request are consistent
+ */
+
+ if (((uint64_t)remote_start_addr < (uint64_t)remote_base_addr) ||
+ ((uint64_t)remote_start_addr >=
+ ((uint64_t)remote_base_addr + reg_len)))
+ return -FI_EINVAL;
+
+ if (((uint64_t)remote_start_addr + len) >
+ ((uint64_t)remote_base_addr + reg_len))
+ return -FI_EINVAL;
+
+ local_start_addr = (void *)((uint8_t *)attach_addr +
+ ((uint8_t *)remote_start_addr -
+ (uint8_t *)remote_base_addr));
+ GNIX_DEBUG(FI_LOG_EP_DATA,
+ "xpmem copy dst addr 0x%016lx start addr 0x%016lx, len %ld\n",
+ (uint64_t)dst_addr, (uint64_t)local_start_addr, len);
+ memcpy(dst_addr, local_start_addr, len);
+
+ return FI_SUCCESS;
+}
+
+int _gnix_xpmem_get_my_segid(struct gnix_xpmem_handle *xp_hndl,
+ xpmem_segid_t *seg_id)
+{
+ int ret = FI_SUCCESS;
+
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+
+ *seg_id = gnix_my_segid;
+ return ret;
+
+}
+
+int _gnix_xpmem_get_apid(struct gnix_xpmem_handle *xp_hndl,
+ xpmem_segid_t segid,
+ xpmem_apid_t *peer_apid)
+{
+ int ret = FI_SUCCESS;
+ xpmem_apid_t apid;
+
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+
+ apid = xpmem_get(segid, XPMEM_RDWR, XPMEM_PERMIT_MODE,
+ (void *)0666);
+ if (apid == -1L) {
+ GNIX_WARN(FI_LOG_DOMAIN, "xpmem_get returned %s\n",
+ strerror(errno));
+ ret = -errno;
+ } else {
+ *peer_apid = apid;
+ }
+
+ return ret;
+}
+
+#else
+
+bool gnix_xpmem_disabled = true;
+
+/*******************************************************************************
+ * almost stub functions when xpmem configuration is disabled
+ ******************************************************************************/
+
+int _gnix_xpmem_handle_create(struct gnix_fid_domain *dom,
+ struct gnix_xpmem_handle **handle)
+{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+ return FI_SUCCESS;
+}
+
+int _gnix_xpmem_handle_destroy(struct gnix_xpmem_handle *hndl)
+{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+ return FI_SUCCESS;
+}
+
+
+int _gnix_xpmem_access_hndl_get(struct gnix_xpmem_handle *xp_hndl,
+ xpmem_apid_t peer_apid,
+ uint64_t remote_vaddr,
+ size_t len,
+ struct gnix_xpmem_access_handle **access_hndl)
+{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+ return FI_SUCCESS;
+}
+
+int _gnix_xpmem_access_hndl_put(struct gnix_xpmem_access_handle *access_hndl)
+{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+ return FI_SUCCESS;
+}
+
+int _gnix_xpmem_accessible(struct gnix_fid_ep *ep,
+ struct gnix_address addr,
+ bool *accessible)
+{
+
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+
+ if (accessible == NULL)
+ return -FI_EINVAL;
+
+ *accessible = false;
+
+ return FI_SUCCESS;
+}
+
+int _gnix_xpmem_copy(struct gnix_xpmem_access_handle *access_hndl,
+ void *dst_addr,
+ void *remote_start_addr,
+ size_t len)
+{
+ return -FI_ENOSYS;
+}
+
+int _gnix_xpmem_get_my_segid(struct gnix_xpmem_handle *xp_hndl,
+ xpmem_segid_t *seg_id)
+{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+ return FI_SUCCESS;
+}
+
+int _gnix_xpmem_get_apid(struct gnix_xpmem_handle *xp_hndl,
+ xpmem_segid_t segid,
+ xpmem_apid_t *peer_apid)
+{
+ GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
+ return FI_SUCCESS;
+}
+
+#endif /* HAVE_XPMEM */
diff --git a/prov/gni/test/allocator.c b/prov/gni/test/allocator.c
deleted file mode 100644
index 007d10f..0000000
--- a/prov/gni/test/allocator.c
+++ /dev/null
@@ -1,517 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stddef.h>
-
-#include "gnix.h"
-#include "gnix_mbox_allocator.h"
-
-#include <criterion/criterion.h>
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom;
-static struct fid_ep *ep;
-static struct fi_info *hints;
-static struct fi_info *fi;
-static struct gnix_fid_ep *ep_priv;
-static struct gnix_mbox_alloc_handle *allocator;
-
-void allocator_setup(void)
-{
- int ret = 0;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = 4;
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert_eq(ret, FI_SUCCESS, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert_eq(ret, FI_SUCCESS, "fi_fabric");
-
- ret = fi_domain(fab, fi, &dom, NULL);
- cr_assert_eq(ret, FI_SUCCESS, "fi_domain");
-
- ret = fi_endpoint(dom, fi, &ep, NULL);
- cr_assert_eq(ret, FI_SUCCESS, "fi_endpoint");
-
- ep_priv = container_of(ep, struct gnix_fid_ep, ep_fid);
-}
-
-void allocator_teardown(void)
-{
- int ret = 0;
-
- ret = fi_close(&ep->fid);
- cr_assert_eq(ret, FI_SUCCESS, "failure in closing ep.");
- ret = fi_close(&dom->fid);
- cr_assert_eq(ret, FI_SUCCESS, "failure in closing domain.");
- ret = fi_close(&fab->fid);
- cr_assert_eq(ret, FI_SUCCESS, "failure in closing fabric.");
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-}
-
-/*
- * Count how many slabs are present in an allocator.
- */
-static size_t count_slabs(struct gnix_mbox_alloc_handle *handle)
-{
- size_t count = 0;
-
- for (struct slist_entry *entry = handle->slab_list.head; entry;
- entry = entry->next) {
- count++;
- }
-
- return count;
-}
-
-/*
- * Absolute value function that returns a ptrdiff_t.
- */
-static ptrdiff_t abs_value(ptrdiff_t x)
-{
- return x * ((x > 0) - (x < 0));
-}
-
-/*
- * Open /proc/self/maps and count the number of times the hugetlbfs
- * string is present. Return value is the count;
- */
-static int verify_hugepages(void)
-{
- int ret = 0;
- FILE *fd;
- char *line;
- size_t size = 1024;
-
- fd = fopen("/proc/self/maps", "r");
- if (!fd) {
- fprintf(stderr, "error opening /proc/self/maps.\n");
- return ret;
- }
-
- line = malloc(size);
- if (!line) {
- fprintf(stderr, "error mallocing space for line.\n");
- return ret;
- }
-
- while (getline(&line, &size, fd) != -1) {
- if (strstr(line, "hugetlbfs")) {
- ret++;
- }
- }
-
- free(line);
- fclose(fd);
-
- return ret;
-}
-
-/*
- * Open an allocator with the given parameters and immediately close it. Verify
- * that everything returned a successful error code.
- */
-static void open_close_allocator(enum gnix_page_size page_size,
- size_t mbox_size,
- size_t mpmmap)
-{
- int ret;
-
- ret = _gnix_mbox_allocator_create(ep_priv->nic, NULL, page_size,
- mbox_size, mpmmap, &allocator);
- cr_assert_eq(ret, FI_SUCCESS, "_gnix_mbox_allocator_create failed.");
- cr_expect_eq(verify_hugepages(), 4,
- "memory not found in /proc/self/maps.");
-
- ret = _gnix_mbox_allocator_destroy(allocator);
- cr_assert_eq(ret, FI_SUCCESS, "_gnix_mbox_allocator_destroy failed.");
- cr_expect_eq(verify_hugepages(), 3,
- "memory not released in /proc/self/maps.");
-}
-
-
-TestSuite(mbox_creation, .init = allocator_setup, .fini = allocator_teardown);
-
-Test(mbox_creation, alloc_single_page)
-{
- /*
- * Test creation of all predefined page sizes.
- */
- open_close_allocator(GNIX_PAGE_2MB, 100, 100);
- open_close_allocator(GNIX_PAGE_4MB, 100, 100);
- open_close_allocator(GNIX_PAGE_8MB, 100, 100);
- open_close_allocator(GNIX_PAGE_16MB, 100, 100);
- open_close_allocator(GNIX_PAGE_32MB, 100, 100);
- open_close_allocator(GNIX_PAGE_64MB, 100, 100);
- open_close_allocator(GNIX_PAGE_128MB, 100, 100);
- open_close_allocator(GNIX_PAGE_256MB, 100, 100);
- open_close_allocator(GNIX_PAGE_512MB, 100, 100);
-}
-
-Test(mbox_creation, alloc_three_pages)
-{
- /*
- * This should allocate a single slab that's 3 pages in size.
- */
- open_close_allocator(GNIX_PAGE_4MB, 1000, 12000);
-}
-
-Test(mbox_creation, alloc_mbox)
-{
- int ret;
-
- struct gnix_mbox *mail_box;
- struct slist_entry *entry;
- struct gnix_slab *slab;
-
- char test_string[] = "hello allocator.";
-
- ret = _gnix_mbox_allocator_create(ep_priv->nic, NULL, GNIX_PAGE_4MB,
- 1000, 12000, &allocator);
- cr_assert_eq(ret, FI_SUCCESS, "_gnix_mbox_allocator_create failed.");
-
- /*
- *value is 4 because the provider has internally already opened
- * an mbox allocator and 2 rdma slabs at this point.
- */
- cr_expect_eq(verify_hugepages(), 4,
- "memory not found in /proc/self/maps.");
-
- ret = _gnix_mbox_alloc(allocator, &mail_box);
- cr_expect_eq(ret, FI_SUCCESS, "_gnix_mbox_alloc failed.");
-
- cr_expect(mail_box);
-
- entry = allocator->slab_list.head;
- cr_assert(entry);
-
- slab = container_of(entry, struct gnix_slab, list_entry);
-
- cr_expect_eq(mail_box->slab, slab,
- "slab list head and mail box slab pointer are not equal.");
- cr_expect_eq(mail_box->memory_handle, &mail_box->slab->memory_handle,
- "mail_box memory handle not equal to slab memory handle.");
- cr_expect_eq(mail_box->offset, 0, "offset is not 0.");
- cr_expect_eq(mail_box->base, mail_box->slab->base,
- "mail_box base not equal to slab base.");
-
- /*
- * Write our test strings and make sure they're equal.
- */
- memcpy(mail_box->base, test_string, sizeof(test_string));
- cr_expect_str_eq((char *) mail_box->base, test_string);
-
- /*
- * Mailboxes haven't been returned so destroy will return -FI_EBUSY.
- */
- ret = _gnix_mbox_allocator_destroy(allocator);
- cr_assert_eq(ret, -FI_EBUSY,
- "_gnix_mbox_allocator_destroy should have returned -FI_EBUSY.");
-
- /*
- * Free allocated mailboxes so we can destroy cleanly.
- */
- ret = _gnix_mbox_free(mail_box);
- cr_expect_eq(ret, FI_SUCCESS, "_gnix_mbox_free failed.");
-
- ret = _gnix_mbox_allocator_destroy(allocator);
- cr_assert_eq(ret, FI_SUCCESS, "_gnix_mbox_allocator_destroy failed.");
-
- cr_expect_eq(verify_hugepages(), 3,
- "memory not released in /proc/self/maps.");
-}
-
-/*
- * Page size needs to be one of the predefined enums. 2200 is not a valid page
- * size. This actually gets expanded to 2200 * 1024 * 1024.
- */
-Test(mbox_creation, page_size_fail)
-{
- int ret;
-
- ret = _gnix_mbox_allocator_create(ep_priv->nic, NULL, 2200,
- 1000, 12000, &allocator);
- cr_assert_eq(ret, -FI_EINVAL,
- "Creating allocator with bogus page size succeeded.");
- cr_assert_eq(allocator, NULL);
- /*
- *value is 3 because the provider has internally already opened
- * an mbox allocator and two other slabs at this point.
- */
- cr_expect_eq(verify_hugepages(), 3,
- "Huge page open, but shouldn't be");
-
- ret = _gnix_mbox_allocator_destroy(allocator);
- cr_assert_eq(ret, -FI_EINVAL,
- "_gnix_mbox_allocator_destroy succeeded on NULL handle.");
-}
-
-Test(mbox_creation, mbox_size_fail)
-{
- int ret;
-
- /*
- * mbox_size can't be zero.
- */
- ret = _gnix_mbox_allocator_create(ep_priv->nic, NULL, GNIX_PAGE_4MB,
- 0, 12000, &allocator);
- cr_assert_eq(ret, -FI_EINVAL,
- "Creating allocator with zero mbox size succeeded.");
-
- cr_assert_eq(allocator, NULL);
- cr_expect_eq(verify_hugepages(), 3,
- "Huge page open, but shouldn't be");
-
- ret = _gnix_mbox_allocator_destroy(allocator);
- cr_assert_eq(ret, -FI_EINVAL,
- "_gnix_mbox_allocator_destroy succeeded on NULL handle.");
-}
-
-Test(mbox_creation, mpmmap_size_fail)
-{
- int ret;
-
- /*
- * Can't have zero mailboxes per mmap.
- */
- ret = _gnix_mbox_allocator_create(ep_priv->nic, NULL, GNIX_PAGE_4MB,
- 1000, 0, &allocator);
- cr_assert_eq(ret, -FI_EINVAL,
- "Creating allocator with zero mailboxes per mmap succeeded.");
- cr_assert_eq(allocator, NULL);
- cr_expect_eq(verify_hugepages(), 3,
- "Huge page open, but shouldn't be");
-
- ret = _gnix_mbox_allocator_destroy(allocator);
- cr_assert_eq(ret, -FI_EINVAL,
- "_gnix_mbox_allocator_destroy succeeded on NULL handle.");
-}
-
-Test(mbox_creation, null_allocator_fail)
-{
- int ret;
-
- /*
- * Can't have a NULL allocator.
- */
- ret = _gnix_mbox_allocator_create(ep_priv->nic, NULL, GNIX_PAGE_4MB,
- 1000, 100, NULL);
- cr_assert_eq(ret, -FI_EINVAL,
- "Creating allocator with null allocator succeeded.");
- cr_expect_eq(verify_hugepages(), 3,
- "Huge page open, but shouldn't be");
-
- ret = _gnix_mbox_allocator_destroy(allocator);
- cr_assert_eq(ret, -FI_EINVAL,
- "_gnix_mbox_allocator_destroy succeeded on NULL handle.");
-}
-
-Test(mbox_creation, multi_allocation)
-{
- int ret;
-
- size_t array_size = 5;
- size_t mbox_size = 1000;
-
- ptrdiff_t expected;
- ptrdiff_t actual;
-
- struct gnix_mbox *mbox_arr[array_size];
-
- ret = _gnix_mbox_allocator_create(ep_priv->nic, NULL, GNIX_PAGE_4MB,
- mbox_size, array_size, &allocator);
- cr_assert_eq(ret, FI_SUCCESS, "_gnix_mbox_allocator_create failed.");
- cr_expect_eq(verify_hugepages(), 4,
- "memory not found in /proc/self/maps.");
-
- /*
- * Create an array of mailboxes of size array_size.
- */
- for (int i = 0; i < array_size; i++) {
- ret = _gnix_mbox_alloc(allocator, &(mbox_arr[i]));
- cr_expect_eq(ret, FI_SUCCESS, "_gnix_mbox_alloc failed.");
- cr_expect(mbox_arr[i]);
- }
-
- /*
- * Compare each mailbox to each other mailbox excluding the diagonal.
- * The expected base should be a function of the mbox_size and the
- * difference between their positions in the array. We can verify this
- * against the offset inside the mailbox object.
- */
- for (int i = 0; i < array_size; i++) {
- for (int j = 0; j < array_size; j++) {
- if (i == j)
- continue;
-
- actual = abs_value(mbox_arr[i]->offset -
- mbox_arr[j]->offset);
- expected = abs_value(i - j) * mbox_size;
-
- cr_expect_eq(actual, expected,
- "Expected offsets and actual base offsets are not equal.");
- }
- }
-
- for (int i = 0; i < array_size; i++) {
- ret = _gnix_mbox_free(mbox_arr[i]);
- cr_expect_eq(ret, FI_SUCCESS, "_gnix_mbox_free failed.");
- }
-
- ret = _gnix_mbox_allocator_destroy(allocator);
- cr_assert_eq(ret, FI_SUCCESS, "_gnix_mbox_allocator_destroy failed.");
-
- cr_expect_eq(verify_hugepages(), 3,
- "memory not released in /proc/self/maps.");
-}
-
-Test(mbox_creation, check_errors)
-{
- int ret;
-
- struct gnix_mbox_alloc_handle *allocator;
- struct gnix_slab *slab;
- struct gnix_mbox *mail_box;
- size_t position;
-
- ret = _gnix_mbox_allocator_create(ep_priv->nic, NULL, GNIX_PAGE_4MB,
- 1000, 12000, &allocator);
- cr_assert_eq(ret, FI_SUCCESS, "_gnix_mbox_allocator_create failed.");
- cr_expect_eq(verify_hugepages(), 4,
- "memory not found in /proc/self/maps.");
-
- ret = _gnix_mbox_alloc(allocator, &mail_box);
- cr_expect_eq(ret, FI_SUCCESS, "_gnix_mbox_alloc failed.");
-
- cr_expect(mail_box);
-
- /* Force various error paths */
- slab = mail_box->slab;
- mail_box->slab = NULL;
- ret = _gnix_mbox_free(mail_box);
- cr_expect_eq(ret, -FI_EINVAL, "_gnix_mbox_free did not fail.");
- mail_box->slab = slab;
-
- allocator = mail_box->slab->allocator;
- mail_box->slab->allocator = NULL;
- ret = _gnix_mbox_free(mail_box);
- cr_expect_eq(ret, -FI_EINVAL, "_gnix_mbox_free did not fail.");
- mail_box->slab->allocator = allocator;
-
- position = mail_box->offset / mail_box->slab->allocator->mbox_size;
- ret = _gnix_test_and_clear_bit(mail_box->slab->used, position);
- cr_expect_eq(ret, 1, "bitmap clear failed.");
- ret = _gnix_mbox_free(mail_box);
- cr_expect_eq(ret, -FI_EINVAL, "_gnix_mbox_free did not fail.");
- ret = _gnix_test_and_set_bit(mail_box->slab->used, position);
- cr_expect_eq(ret, 0, "bitmap set failed.");
-
- /*
- * Free allocated mailboxes so we can destroy cleanly.
- */
- ret = _gnix_mbox_free(mail_box);
- cr_expect_eq(ret, FI_SUCCESS, "_gnix_mbox_free failed.");
-
- ret = _gnix_mbox_allocator_destroy(allocator);
- cr_assert_eq(ret, FI_SUCCESS, "_gnix_mbox_allocator_destroy failed.");
-
- cr_expect_eq(verify_hugepages(), 3,
- "memory not released in /proc/self/maps.");
-}
-
-/*
- * Force the creation of two slabs by setting mpmmap to 1 and making a mailbox
- * the size of the entire page.
- */
-Test(mbox_creation, two_slabs)
-{
- int ret;
-
- /*
- * Only have one mail box per slab.
- */
- size_t mbox_size = GNIX_PAGE_4MB * 1024 * 1024;
- size_t mpmmap = 1;
-
- struct gnix_mbox *mbox_arr[2];
-
- ret = _gnix_mbox_allocator_create(ep_priv->nic, NULL, GNIX_PAGE_4MB,
- mbox_size, mpmmap, &allocator);
- cr_assert_eq(ret, FI_SUCCESS, "_gnix_mbox_allocator_create failed.");
- cr_expect_eq(verify_hugepages(), 4,
- "memory not found in /proc/self/maps.");
-
- /*
- * Should use previously allocated slab
- */
- ret = _gnix_mbox_alloc(allocator, &(mbox_arr[0]));
- cr_expect_eq(ret, FI_SUCCESS, "_gnix_mbox_alloc failed.");
-
- /*
- * Will need another slab. Allocation will occur.
- */
- ret = _gnix_mbox_alloc(allocator, &(mbox_arr[1]));
- cr_expect_eq(ret, FI_SUCCESS, "_gnix_mbox_alloc failed.");
-
- /*
- * The bases should be different. The base is a per slab concept.
- */
- cr_expect_neq(mbox_arr[0]->base, mbox_arr[1]->base,
- "Bases are the same.");
-
- /*
- * The linked list should contain two slabs.
- */
- cr_expect_eq(2, count_slabs(allocator));
-
- ret = _gnix_mbox_free(mbox_arr[0]);
- cr_expect_eq(ret, FI_SUCCESS, "_gnix_mbox_free failed.");
-
- ret = _gnix_mbox_free(mbox_arr[1]);
- cr_expect_eq(ret, FI_SUCCESS, "_gnix_mbox_free failed.");
-
- ret = _gnix_mbox_allocator_destroy(allocator);
- cr_assert_eq(ret, FI_SUCCESS, "_gnix_mbox_allocator_destroy failed.");
-
- cr_expect_eq(verify_hugepages(), 3,
- "memory not released in /proc/self/maps.");
-}
diff --git a/prov/gni/test/api.c b/prov/gni/test/api.c
deleted file mode 100644
index 68572a5..0000000
--- a/prov/gni/test/api.c
+++ /dev/null
@@ -1,817 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <pthread.h>
-#include <unistd.h>
-#include <limits.h>
-#include <assert.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-#include <rdma/fi_atomic.h>
-#include <rdma/fi_tagged.h>
-#include "fi_ext_gni.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include <criterion/criterion.h>
-
-#if 1
-#define dbg_printf(...)
-#else
-#define dbg_printf(...) \
- do { \
- printf(__VA_ARGS__); \
- fflush(stdout); \
- } while (0)
-#endif
-
-#define NUMEPS 2
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom[NUMEPS];
-struct fi_gni_ops_domain *gni_domain_ops[NUMEPS];
-static struct fid_ep *ep[NUMEPS];
-static struct fid_av *av[NUMEPS];
-void *ep_name[NUMEPS];
-fi_addr_t gni_addr[NUMEPS];
-static struct fid_cq *msg_cq[NUMEPS];
-static struct fi_info *fi[NUMEPS];
-static struct fi_cq_attr cq_attr;
-const char *api_cdm_id[NUMEPS] = { "5000", "5001" };
-struct fi_info *hints[NUMEPS];
-
-#define BUF_SZ (1<<20)
-char *target;
-char *source;
-char *uc_target;
-char *uc_source;
-struct fid_mr *rem_mr[NUMEPS], *loc_mr[NUMEPS];
-uint64_t mr_key[NUMEPS];
-
-static struct fid_cntr *send_cntr[NUMEPS], *recv_cntr[NUMEPS];
-static struct fi_cntr_attr cntr_attr = {
- .events = FI_CNTR_EVENTS_COMP,
- .flags = 0
-};
-static uint64_t sends[NUMEPS] = {0}, recvs[NUMEPS] = {0},
- send_errs[NUMEPS] = {0}, recv_errs[NUMEPS] = {0};
-
-void rdm_api_setup_ep(void)
-{
- int ret, i, j;
- struct fi_av_attr attr;
- size_t addrlen = 0;
-
- /* Get info about fabric services with the provided hints */
- for (i = 0; i < NUMEPS; i++) {
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints[i],
- &fi[i]);
- cr_assert(!ret, "fi_getinfo");
- }
-
- attr.type = FI_AV_MAP;
- attr.count = NUMEPS;
-
- cq_attr.format = FI_CQ_FORMAT_TAGGED;
- cq_attr.size = 1024;
- cq_attr.wait_obj = 0;
-
- target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */
- assert(target);
-
- source = malloc(BUF_SZ);
- assert(source);
-
- uc_target = malloc(BUF_SZ);
- assert(uc_target);
-
- uc_source = malloc(BUF_SZ);
- assert(uc_source);
-
- ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- for (i = 0; i < NUMEPS; i++) {
- ret = fi_domain(fab, fi[i], dom + i, NULL);
- cr_assert(!ret, "fi_domain");
-
- ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
- 0, (void **) (gni_domain_ops + i), NULL);
-
- ret = fi_av_open(dom[i], &attr, av + i, NULL);
- cr_assert(!ret, "fi_av_open");
-
- ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
- cr_assert(!ret, "fi_endpoint");
-
- ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
- cr_assert(addrlen > 0);
-
- ep_name[i] = malloc(addrlen);
- cr_assert(ep_name[i] != NULL);
-
- ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
- cr_assert(ret == FI_SUCCESS);
- }
-
- for (i = 0; i < NUMEPS; i++) {
- /* Insert all gni addresses into each av */
- for (j = 0; j < NUMEPS; j++) {
- ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
- 0, NULL);
- cr_assert(ret == 1);
- }
-
- ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_enable(ep[i]);
- cr_assert(!ret, "fi_ep_enable");
-
- ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
- }
-
- for (i = 0; i < NUMEPS; i++) {
- ret = fi_mr_reg(dom[i], target, 3 * BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, rem_mr + i, &target);
- cr_assert_eq(ret, 0);
-
- ret = fi_mr_reg(dom[i], source, BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, loc_mr + i, &source);
- cr_assert_eq(ret, 0);
-
- mr_key[i] = fi_mr_key(rem_mr[i]);
- }
-}
-
-void rdm_api_setup(void)
-{
- int i;
-
- for (i = 0; i < NUMEPS; i++) {
- hints[i] = fi_allocinfo();
- cr_assert(hints[i], "fi_allocinfo");
-
- hints[i]->domain_attr->cq_data_size = NUMEPS * 2;
- hints[i]->domain_attr->data_progress = FI_PROGRESS_AUTO;
- hints[i]->mode = ~0;
- hints[i]->domain_attr->mr_mode = FI_MR_BASIC;
- hints[i]->fabric_attr->name = strdup("gni");
- }
-}
-
-static void rdm_api_teardown_common(bool unreg)
-{
- int ret = 0, i = 0;
-
- free(uc_source);
- free(uc_target);
- free(target);
- free(source);
-
- for (; i < NUMEPS; i++) {
- fi_close(&recv_cntr[i]->fid);
- fi_close(&send_cntr[i]->fid);
-
- if (unreg) {
- fi_close(&loc_mr[i]->fid);
- fi_close(&rem_mr[i]->fid);
- }
-
- ret = fi_close(&ep[i]->fid);
- cr_assert(!ret, "failure in closing ep.");
-
- ret = fi_close(&msg_cq[i]->fid);
- cr_assert(!ret, "failure in send cq.");
-
- ret = fi_close(&av[i]->fid);
- cr_assert(!ret, "failure in closing av.");
-
- ret = fi_close(&dom[i]->fid);
- cr_assert(!ret, "failure in closing domain.");
-
- fi_freeinfo(fi[i]);
- free(ep_name[i]);
- fi_freeinfo(hints[i]);
- }
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
-}
-
-static void rdm_api_teardown(void)
-{
- rdm_api_teardown_common(true);
-}
-
-void rdm_api_init_data(char *buf, int len, char seed)
-{
- int i;
-
- for (i = 0; i < len; i++)
- buf[i] = seed++;
-}
-
-int rdm_api_check_data(char *buf1, char *buf2, int len)
-{
- int i;
-
- for (i = 0; i < len; i++) {
- if (buf1[i] != buf2[i]) {
- printf("data mismatch, elem: %d, exp: %hhx, act: %hhx\n"
- , i, buf1[i], buf2[i]);
- return 0;
- }
- }
-
- return 1;
-}
-
-void rdm_api_check_cqe(struct fi_cq_tagged_entry *cqe, void *ctx,
- uint64_t flags, void *addr, size_t len,
- uint64_t data)
-{
- cr_assert(cqe->op_context == ctx, "CQE Context mismatch");
- cr_assert(cqe->flags == flags, "CQE flags mismatch");
-
- if (flags & FI_RECV) {
- cr_assert(cqe->len == len, "CQE length mismatch");
- cr_assert(cqe->buf == addr, "CQE address mismatch");
-
- if (flags & FI_REMOTE_CQ_DATA)
- cr_assert(cqe->data == data, "CQE data mismatch");
- } else {
- cr_assert(cqe->len == 0, "Invalid CQE length");
- cr_assert(cqe->buf == 0, "Invalid CQE address");
- cr_assert(cqe->data == 0, "Invalid CQE data");
- }
-
- cr_assert(cqe->tag == 0, "Invalid CQE tag");
-}
-
-void rdm_api_check_cntrs(uint64_t s[], uint64_t r[], uint64_t s_e[],
- uint64_t r_e[])
-{
- int i = 0;
-
- for (; i < NUMEPS; i++) {
- sends[i] += s[i];
- recvs[i] += r[i];
- send_errs[i] += s_e[i];
- recv_errs[i] += r_e[i];
-
- cr_assert(fi_cntr_read(send_cntr[i]) == sends[i],
- "Bad send count");
- cr_assert(fi_cntr_read(recv_cntr[i]) == recvs[i],
- "Bad recv count");
- cr_assert(fi_cntr_readerr(send_cntr[i]) == send_errs[i],
- "Bad send err count");
- cr_assert(fi_cntr_readerr(recv_cntr[i]) == recv_errs[i],
- "Bad recv err count");
- }
-}
-
-/*******************************************************************************
- * Test MSG functions
- ******************************************************************************/
-
-#define MSG_SEND_ALLOWED(caps) \
- ((caps & FI_MSG) && ((caps & FI_SEND) || !(caps & FI_RECV)))
-#define MSG_RECV_ALLOWED(caps) \
- ((caps & FI_MSG) && ((caps & FI_RECV) || !(caps & FI_SEND)))
-#define TAG_SEND_ALLOWED(caps) \
- ((caps & FI_TAGGED) && ((caps & FI_SEND) || !(caps & FI_RECV)))
-#define TAG_RECV_ALLOWED(caps) \
- ((caps & FI_TAGGED) && ((caps & FI_RECV) || !(caps & FI_SEND)))
-#define WRITE_ALLOWED(caps, rcaps) \
- ((caps & FI_RMA) && \
- ((caps & FI_WRITE) || !(caps & FI_READ)) && \
- ((rcaps & FI_RMA) || (rcaps & FI_REMOTE_WRITE)) \
- )
-#define READ_ALLOWED(caps, rcaps) \
- ((caps & FI_RMA) && \
- ((caps & FI_READ) || !(caps & FI_WRITE)) && \
- (((rcaps & FI_RMA) && \
- !(rcaps & (FI_READ | FI_WRITE | FI_REMOTE_WRITE))) || \
- (rcaps & FI_REMOTE_READ) \
- ) \
- )
-static int write_allowed(uint64_t rma_amo, uint64_t caps, uint64_t rcaps)
-{
- dbg_printf("write %s caps:%s, rcaps:%s\n",
- fi_tostr(&rma_amo, FI_TYPE_CAPS),
- fi_tostr(&caps, FI_TYPE_CAPS),
- fi_tostr(&rcaps, FI_TYPE_CAPS));
- if ((caps & rma_amo) &&
- ((caps & FI_WRITE) || !(caps & FI_READ))) {
- if ((rcaps & rma_amo) &&
- ((rcaps & FI_REMOTE_WRITE) ||
- (!(rcaps & (FI_READ | FI_WRITE | FI_REMOTE_READ)))
- )
- ) {
- return 1;
- }
- }
- return 0;
-}
-
-static int read_allowed(uint64_t rma_amo, uint64_t caps, uint64_t rcaps)
-{
- dbg_printf("read %s caps:%s, rcaps:%s\n",
- fi_tostr(&rma_amo, FI_TYPE_CAPS),
- fi_tostr(&caps, FI_TYPE_CAPS),
- fi_tostr(&rcaps, FI_TYPE_CAPS));
- if ((caps & rma_amo) &&
- ((caps & FI_READ) || !(caps & FI_WRITE))) {
- if ((rcaps & rma_amo) &&
- ((rcaps & FI_REMOTE_READ) ||
- (!(rcaps & (FI_READ | FI_WRITE | FI_REMOTE_WRITE)))
- )
- ) {
- return 1;
- }
- }
- return 0;
-}
-
-TestSuite(rdm_api, .init = rdm_api_setup, .fini = rdm_api_teardown,
- .disabled = false);
-
-/*
- * ssize_t fi_send(struct fid_ep *ep, void *buf, size_t len,
- * void *desc, fi_addr_t dest_addr, void *context);
- *
- * ssize_t fi_recv(struct fid_ep *ep, void * buf, size_t len,
- * void *desc, fi_addr_t src_addr, void *context);
- */
-void api_send_recv(int len)
-{
- ssize_t sz;
- uint64_t caps = fi[0]->caps;
-
- rdm_api_init_data(source, len, 0xab);
- rdm_api_init_data(target, len, 0);
-
- sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
- if (MSG_SEND_ALLOWED(caps)) {
- cr_assert(sz == 0, "fi_send failed caps:0x%lx err:%ld",
- caps, sz);
- } else {
- cr_assert(sz < 0, "fi_send should fail caps:0x%lx err:%ld",
- caps, sz);
- }
-
- sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source);
- if (MSG_RECV_ALLOWED(caps)) {
- cr_assert(sz == 0, "fi_recv failed caps:0x%lx err:%ld",
- caps, sz);
- } else {
- cr_assert(sz < 0, "fi_recv should fail caps:0x%lx err:%ld",
- caps, sz);
- }
-}
-
-Test(rdm_api, msg_no_caps)
-{
- hints[0]->caps = 0;
- hints[1]->caps = 0;
- rdm_api_setup_ep();
- api_send_recv(BUF_SZ);
-}
-
-Test(rdm_api, msg_send_rcv)
-{
- hints[0]->caps = FI_MSG;
- hints[1]->caps = FI_MSG;
- rdm_api_setup_ep();
- api_send_recv(BUF_SZ);
-}
-
-Test(rdm_api, msg_send_only)
-{
- hints[0]->caps = FI_MSG | FI_SEND;
- hints[1]->caps = FI_MSG | FI_SEND;
- rdm_api_setup_ep();
- api_send_recv(BUF_SZ);
-}
-
-Test(rdm_api, msg_recv_only)
-{
- hints[0]->caps = FI_MSG | FI_RECV;
- hints[1]->caps = FI_MSG | FI_RECV;
- rdm_api_setup_ep();
- api_send_recv(BUF_SZ);
-}
-
-Test(rdm_api, msg_send_rcv_w_tagged)
-{
- hints[0]->caps = FI_TAGGED;
- hints[1]->caps = FI_TAGGED;
- rdm_api_setup_ep();
- api_send_recv(BUF_SZ);
-}
-
-void api_tagged_send_recv(int len)
-{
- ssize_t sz;
- uint64_t caps = fi[0]->caps;
-
- rdm_api_init_data(source, len, 0xab);
- rdm_api_init_data(target, len, 0);
-
- sz = fi_tsend(ep[0], source, len, loc_mr, gni_addr[1], len, target);
- if (TAG_SEND_ALLOWED(caps)) {
- cr_assert(sz == 0, "fi_tsend failed caps:0x%lx err:%ld",
- caps, sz);
- } else {
- cr_assert(sz < 0, "fi_tsend should fail caps:0x%lx err:%ld",
- caps, sz);
- }
-
- sz = fi_trecv(ep[1], target, len, rem_mr, gni_addr[0], len, 0, source);
- if (TAG_RECV_ALLOWED(caps)) {
- cr_assert(sz == 0, "fi_trecv failed caps:0x%lx err:%ld",
- caps, sz);
- } else {
- cr_assert(sz < 0, "fi_trecv should fail caps:0x%lx err:%ld",
- caps, sz);
- }
-}
-
-Test(rdm_api, tsend)
-{
- hints[0]->caps = FI_TAGGED;
- hints[1]->caps = FI_TAGGED;
- rdm_api_setup_ep();
- api_tagged_send_recv(BUF_SZ);
-}
-
-Test(rdm_api, tsend_only)
-{
- hints[0]->caps = FI_TAGGED | FI_SEND;
- hints[1]->caps = FI_TAGGED | FI_SEND;
- rdm_api_setup_ep();
- api_tagged_send_recv(BUF_SZ);
-}
-
-Test(rdm_api, trecv_only)
-{
- hints[0]->caps = FI_TAGGED | FI_RECV;
- hints[1]->caps = FI_TAGGED | FI_RECV;
- rdm_api_setup_ep();
- api_tagged_send_recv(BUF_SZ);
-}
-
-Test(rdm_api, tsend_rcv_w_msg)
-{
- hints[0]->caps = FI_MSG;
- hints[1]->caps = FI_MSG;
- rdm_api_setup_ep();
- api_tagged_send_recv(BUF_SZ);
-}
-
-#define READ_CTX 0x4e3dda1aULL
-void api_write_read(int len)
-{
- int ret;
- struct fi_cq_tagged_entry cqe;
- struct fi_cq_err_entry err_cqe = {0};
-
- rdm_api_init_data(source, len, 0xab);
- rdm_api_init_data(target, len, 0);
-
- fi_write(ep[0], source, len,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- target);
-
- while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN)
- pthread_yield();
-
- if (ret == -FI_EAVAIL) {
- fi_cq_readerr(msg_cq[0], &err_cqe, 0);
- dbg_printf("fi_cq_readerr err:%d\n", err_cqe.err);
- }
-
- if (write_allowed(FI_RMA, fi[0]->caps, fi[1]->caps)) {
- cr_assert(ret == 1,
- "fi_write failed caps:0x%lx ret:%d",
- fi[0]->caps, ret);
- } else {
- cr_assert(err_cqe.err == FI_EOPNOTSUPP,
- "fi_write should fail caps:0x%lx err:%d",
- fi[0]->caps, err_cqe.err);
- }
-
- fi_read(ep[0], source, len,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- (void *)READ_CTX);
-
- while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN)
- pthread_yield();
-
- if (ret == -FI_EAVAIL) {
- fi_cq_readerr(msg_cq[0], &err_cqe, 0);
- dbg_printf("fi_cq_readerr err:%d\n", err_cqe.err);
- }
-
- if (read_allowed(FI_RMA, fi[0]->caps, fi[1]->caps)) {
- cr_assert(ret == 1,
- "fi_read failed caps:0x%lx rcaps:0x%lx",
- fi[0]->caps, fi[1]->caps);
- } else {
- cr_assert(err_cqe.err == FI_EOPNOTSUPP,
- "fi_read should fail caps:0x%lx rcaps:0x%lx",
- fi[0]->caps, fi[1]->caps);
- }
-}
-
-Test(rdm_api, rma_only)
-{
- hints[0]->caps = FI_RMA;
- hints[1]->caps = FI_RMA;
- rdm_api_setup_ep();
- api_write_read(BUF_SZ);
-}
-
-Test(rdm_api, rma_write_only)
-{
- hints[0]->caps = FI_RMA | FI_WRITE;
- hints[1]->caps = FI_RMA | FI_REMOTE_WRITE;
- rdm_api_setup_ep();
- api_write_read(BUF_SZ);
-}
-
-Test(rdm_api, rma_write_no_remote)
-{
- hints[0]->caps = FI_RMA | FI_WRITE;
- hints[1]->caps = FI_RMA | FI_WRITE;
- rdm_api_setup_ep();
- api_write_read(BUF_SZ);
-}
-
-Test(rdm_api, rma_read_only)
-{
- hints[0]->caps = FI_RMA | FI_READ;
- hints[1]->caps = FI_RMA | FI_REMOTE_READ;
- rdm_api_setup_ep();
- api_write_read(BUF_SZ);
-}
-
-Test(rdm_api, rma_read_no_remote)
-{
- hints[0]->caps = FI_RMA | FI_READ;
- hints[1]->caps = FI_RMA | FI_READ;
- rdm_api_setup_ep();
- api_write_read(BUF_SZ);
-}
-
-Test(rdm_api, rma_write_read_w_msg)
-{
- hints[0]->caps = FI_MSG;
- hints[1]->caps = FI_MSG;
- rdm_api_setup_ep();
- api_write_read(BUF_SZ);
-}
-
-
-void api_do_read_buf(void)
-{
- int ret;
- int len = 8*1024;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- struct fi_cq_err_entry err_cqe;
-
- rdm_api_init_data(source, BUF_SZ, 0);
- rdm_api_init_data(target, BUF_SZ, 0xad);
-
- /* cause a chained transaction */
- sz = fi_read(ep[0], source+6, len,
- loc_mr[0], gni_addr[1], (uint64_t)target+6, mr_key[1],
- (void *)READ_CTX);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN)
- pthread_yield();
-
- if (ret == -FI_EAVAIL) {
- fi_cq_readerr(msg_cq[0], &err_cqe, 0);
- dbg_printf("fi_cq_readerr err:%d\n", err_cqe.err);
- }
-
- if (read_allowed(FI_RMA, fi[0]->caps, fi[1]->caps)) {
- cr_assert(ret == 1,
- "fi_read failed caps:0x%lx rcaps:0x%lx",
- fi[0]->caps, fi[1]->caps);
- } else {
- cr_assert(err_cqe.err == FI_EOPNOTSUPP,
- "fi_read should fail caps:0x%lx rcaps:0x%lx",
- fi[0]->caps, fi[1]->caps);
- }
-}
-
-Test(rdm_api, read_chained)
-{
- hints[0]->caps = FI_RMA;
- hints[1]->caps = FI_RMA;
- rdm_api_setup_ep();
- api_do_read_buf();
-}
-
-Test(rdm_api, read_chained_remote)
-{
- hints[0]->caps = FI_RMA | FI_READ;
- hints[1]->caps = FI_RMA | FI_REMOTE_READ;
- rdm_api_setup_ep();
- api_do_read_buf();
-}
-
-Test(rdm_api, read_chained_w_write)
-{
- hints[0]->caps = FI_RMA | FI_WRITE;
- hints[1]->caps = FI_RMA | FI_REMOTE_READ;
- rdm_api_setup_ep();
- api_do_read_buf();
-}
-
-Test(rdm_api, read_chained_no_remote)
-{
- hints[0]->caps = FI_RMA | FI_READ;
- hints[1]->caps = FI_RMA | FI_READ;
- rdm_api_setup_ep();
- api_do_read_buf();
-}
-
-#define SOURCE_DATA 0xBBBB0000CCCCULL
-#define TARGET_DATA 0xAAAA0000DDDDULL
-#define FETCH_SOURCE_DATA 0xACEDACEDULL
-
-void do_atomic_write_fetch(void)
-{
- int ret;
- ssize_t sz;
- uint64_t operand;
- struct fi_cq_tagged_entry cqe;
- struct fi_cq_err_entry err_cqe;
-
- /* u64 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT64, FI_ATOMIC_WRITE, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN)
- pthread_yield();
-
- if (ret == -FI_EAVAIL) {
- fi_cq_readerr(msg_cq[0], &err_cqe, 0);
- dbg_printf("fi_cq_readerr err:%d\n", err_cqe.err);
- }
-
- if (write_allowed(FI_ATOMIC, fi[0]->caps, fi[1]->caps)) {
- cr_assert(ret == 1,
- "fi_atomic failed caps:0x%lx rcaps:0x%lx",
- fi[0]->caps, fi[1]->caps);
- } else {
- cr_assert(err_cqe.err == FI_EOPNOTSUPP,
- "fi_atomic should fail caps:0x%lx rcaps:0x%lx",
- fi[0]->caps, fi[1]->caps);
- }
-
- /* u64 */
- operand = SOURCE_DATA;
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT64, FI_ATOMIC_READ, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN)
- pthread_yield();
-
- if (ret == -FI_EAVAIL) {
- fi_cq_readerr(msg_cq[0], &err_cqe, 0);
- dbg_printf("fi_cq_readerr err:%d\n", err_cqe.err);
- }
-
- if (read_allowed(FI_ATOMIC, fi[0]->caps, fi[1]->caps)) {
- cr_assert(ret == 1,
- "fi_fetch_atomic failed caps:0x%lx rcaps:0x%lx",
- fi[0]->caps, fi[1]->caps);
- } else {
- cr_assert(err_cqe.err == FI_EOPNOTSUPP,
- "fi_fetch_atomic should fail caps:0x%lx rcaps:0x%lx",
- fi[0]->caps, fi[1]->caps);
- }
-}
-
-Test(rdm_api, amo_write_read)
-{
- hints[0]->caps = FI_ATOMIC;
- hints[1]->caps = FI_ATOMIC;
- rdm_api_setup_ep();
- do_atomic_write_fetch();
-}
-
-Test(rdm_api, amo_write_only)
-{
- hints[0]->caps = FI_ATOMIC | FI_WRITE;
- hints[1]->caps = FI_ATOMIC | FI_REMOTE_WRITE;
- rdm_api_setup_ep();
- do_atomic_write_fetch();
-}
-
-Test(rdm_api, amo_write_no_remote)
-{
- hints[0]->caps = FI_ATOMIC | FI_WRITE;
- hints[1]->caps = FI_ATOMIC | FI_WRITE;
- rdm_api_setup_ep();
- do_atomic_write_fetch();
-}
-
-Test(rdm_api, amo_read_only)
-{
- hints[0]->caps = FI_ATOMIC | FI_READ;
- hints[1]->caps = FI_ATOMIC | FI_REMOTE_READ;
- rdm_api_setup_ep();
- do_atomic_write_fetch();
-}
-
-Test(rdm_api, amo_read_no_remote)
-{
- hints[0]->caps = FI_ATOMIC | FI_READ;
- hints[1]->caps = FI_ATOMIC | FI_READ;
- rdm_api_setup_ep();
- do_atomic_write_fetch();
-}
-
-Test(rdm_api, amo_write_read_w_msg)
-{
- hints[0]->caps = FI_MSG;
- hints[1]->caps = FI_MSG;
- rdm_api_setup_ep();
- api_write_read(BUF_SZ);
-}
-
diff --git a/prov/gni/test/api_cq.c b/prov/gni/test/api_cq.c
deleted file mode 100644
index 4093792..0000000
--- a/prov/gni/test/api_cq.c
+++ /dev/null
@@ -1,460 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <pthread.h>
-#include <unistd.h>
-#include <limits.h>
-#include <assert.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-#include <rdma/fi_atomic.h>
-#include <rdma/fi_tagged.h>
-#include "fi_ext_gni.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include <criterion/criterion.h>
-
-#if 1
-#define dbg_printf(...)
-#else
-#define dbg_printf(...) \
- do { \
- printf(__VA_ARGS__); \
- fflush(stdout); \
- } while (0)
-#endif
-
-#define NUMEPS 2
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom[NUMEPS];
-struct fi_gni_ops_domain *gni_domain_ops[NUMEPS];
-static struct fid_ep *ep[NUMEPS];
-static struct fid_av *av[NUMEPS];
-void *ep_name[NUMEPS];
-fi_addr_t gni_addr[NUMEPS];
-static struct fid_cq *msg_cq[NUMEPS];
-static struct fi_info *fi[NUMEPS];
-static struct fi_cq_attr cq_attr;
-struct fi_info *hints[NUMEPS];
-
-#define BUF_SZ (1<<20)
-char *target;
-char *source;
-char *uc_target;
-char *uc_source;
-struct fid_mr *rem_mr[NUMEPS], *loc_mr[NUMEPS];
-uint64_t mr_key[NUMEPS];
-uint64_t cq_bind_flags;
-
-static struct fid_cntr *send_cntr[NUMEPS], *recv_cntr[NUMEPS];
-static struct fi_cntr_attr cntr_attr = {
- .events = FI_CNTR_EVENTS_COMP,
- .flags = 0
-};
-
-void api_cq_bind(uint64_t flags)
-{
- int ret, i;
-
- for (i = 0; i < NUMEPS; i++) {
- ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, flags);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_enable(ep[i]);
- cr_assert(!ret, "fi_enable");
- }
-}
-
-void api_cq_setup(void)
-{
- int ret, i, j;
- struct fi_av_attr attr;
- size_t addrlen = 0;
-
- for (i = 0; i < NUMEPS; i++) {
- hints[i] = fi_allocinfo();
- cr_assert(hints[i], "fi_allocinfo");
-
- hints[i]->domain_attr->cq_data_size = NUMEPS * 2;
- hints[i]->domain_attr->data_progress = FI_PROGRESS_AUTO;
- hints[i]->mode = ~0;
- hints[i]->fabric_attr->name = strdup("gni");
- }
-
- /* Get info about fabric services with the provided hints */
- for (i = 0; i < NUMEPS; i++) {
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints[i],
- &fi[i]);
- cr_assert(!ret, "fi_getinfo");
- }
-
- attr.type = FI_AV_MAP;
- attr.count = NUMEPS;
-
- cq_attr.format = FI_CQ_FORMAT_TAGGED;
- cq_attr.size = 1024;
- cq_attr.wait_obj = 0;
-
- target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */
- assert(target);
-
- source = malloc(BUF_SZ);
- assert(source);
-
- uc_target = malloc(BUF_SZ);
- assert(uc_target);
-
- uc_source = malloc(BUF_SZ);
- assert(uc_source);
-
- ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- for (i = 0; i < NUMEPS; i++) {
- ret = fi_domain(fab, fi[i], dom + i, NULL);
- cr_assert(!ret, "fi_domain");
-
- ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
- 0, (void **) (gni_domain_ops + i), NULL);
-
- ret = fi_av_open(dom[i], &attr, av + i, NULL);
- cr_assert(!ret, "fi_av_open");
-
- ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
- cr_assert(!ret, "fi_endpoint");
-
- ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
- cr_assert(addrlen > 0);
-
- ep_name[i] = malloc(addrlen);
- cr_assert(ep_name[i] != NULL);
-
- ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
- cr_assert(ret == FI_SUCCESS);
- }
-
- for (i = 0; i < NUMEPS; i++) {
- /* Insert all gni addresses into each av */
- for (j = 0; j < NUMEPS; j++) {
- ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
- 0, NULL);
- cr_assert(ret == 1);
- }
-
- ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
-
- ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
- }
-
- for (i = 0; i < NUMEPS; i++) {
- ret = fi_mr_reg(dom[i], target, 3 * BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, rem_mr + i, &target);
- cr_assert_eq(ret, 0);
-
- ret = fi_mr_reg(dom[i], source, BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, loc_mr + i, &source);
- cr_assert_eq(ret, 0);
-
- mr_key[i] = fi_mr_key(rem_mr[i]);
- }
-}
-
-static void api_cq_teardown_common(bool unreg)
-{
- int ret = 0, i = 0;
-
- free(uc_source);
- free(uc_target);
- free(target);
- free(source);
-
- for (; i < NUMEPS; i++) {
- fi_close(&recv_cntr[i]->fid);
- fi_close(&send_cntr[i]->fid);
-
- if (unreg) {
- fi_close(&loc_mr[i]->fid);
- fi_close(&rem_mr[i]->fid);
- }
-
- ret = fi_close(&ep[i]->fid);
- cr_assert(!ret, "failure in closing ep.");
-
- ret = fi_close(&msg_cq[i]->fid);
- cr_assert(!ret, "failure in send cq.");
-
- ret = fi_close(&av[i]->fid);
- cr_assert(!ret, "failure in closing av.");
-
- ret = fi_close(&dom[i]->fid);
- cr_assert(!ret, "failure in closing domain.");
-
- fi_freeinfo(fi[i]);
- free(ep_name[i]);
- fi_freeinfo(hints[i]);
- }
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
-}
-
-static void api_cq_teardown(void)
-{
- api_cq_teardown_common(true);
-}
-
-void api_cq_init_data(char *buf, int len, char seed)
-{
- int i;
-
- for (i = 0; i < len; i++)
- buf[i] = seed++;
-}
-
-int api_cq_check_data(char *buf1, char *buf2, int len)
-{
- int i;
-
- for (i = 0; i < len; i++) {
- if (buf1[i] != buf2[i]) {
- printf("data mismatch, elem: %d, exp: %hhx, act: %hhx\n"
- , i, buf1[i], buf2[i]);
- return 0;
- }
- }
-
- return 1;
-}
-
-#define MSG_SEND_ALLOWED(flags) \
- (flags & FI_SEND)
-#define MSG_RECV_ALLOWED(flags) \
- (flags & FI_RECV)
-
-void api_cq_send_allowed(ssize_t sz, uint64_t flags, char *fn)
-{
- if (MSG_SEND_ALLOWED(flags)) {
- cr_assert(sz == 0, "%s failed flags:0x%lx sz:%ld",
- fn, flags, sz);
- } else {
- cr_assert(sz < 0, "%s should fail flags:0x%lx sz:%ld",
- fn, flags, sz);
- }
-}
-
-void api_cq_recv_allowed(ssize_t sz, uint64_t flags, char *fn)
-{
- if (MSG_RECV_ALLOWED(cq_bind_flags)) {
- cr_assert(sz == 0, "%s failed flags:0x%lx sz:%ld",
- fn, flags, sz);
- } else {
- cr_assert(sz < 0, "%s should fail flags:0x%lx sz:%ld",
- fn, flags, sz);
- }
-}
-
-TestSuite(api_cq, .init = api_cq_setup, .fini = api_cq_teardown,
- .disabled = false);
-
-void api_cq_send_recv(int len)
-{
- ssize_t sz;
- struct iovec iov;
- struct fi_msg_rma rma_msg;
- struct fi_rma_iov rma_iov;
-
- iov.iov_base = NULL;
- iov.iov_len = 0;
-
- api_cq_init_data(source, len, 0xab);
- api_cq_init_data(target, len, 0);
-
- sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
- api_cq_send_allowed(sz, cq_bind_flags, "fi_send");
-
- sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source);
- api_cq_recv_allowed(sz, cq_bind_flags, "fi_recv");
-
- sz = fi_write(ep[0], source, len,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- target);
- api_cq_send_allowed(sz, cq_bind_flags, "fi_write");
-
- sz = fi_writev(ep[0], &iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- target);
- api_cq_send_allowed(sz, cq_bind_flags, "fi_writev");
-
- iov.iov_len = len;
- iov.iov_base = source;
-
- rma_iov.addr = (uint64_t)target;
- rma_iov.len = len;
- rma_iov.key = mr_key[1];
- rma_msg.msg_iov = &iov;
- rma_msg.desc = (void **)loc_mr;
- rma_msg.iov_count = 1;
- rma_msg.addr = gni_addr[1];
- rma_msg.rma_iov = &rma_iov;
- rma_msg.rma_iov_count = 1;
- rma_msg.context = target;
- rma_msg.data = (uint64_t)target;
-
- sz = fi_writemsg(ep[0], &rma_msg, 0);
- api_cq_send_allowed(sz, cq_bind_flags, "fi_writemsg");
-
-#define WRITE_DATA 0x5123da1a145
- sz = fi_writedata(ep[0], source, len, loc_mr[0], WRITE_DATA,
- gni_addr[1], (uint64_t)target, mr_key[1],
- target);
- api_cq_send_allowed(sz, cq_bind_flags, "fi_writedata");
-
-#define READ_CTX 0x4e3dda1aULL
- sz = fi_read(ep[0], source, len,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- (void *)READ_CTX);
- api_cq_send_allowed(sz, cq_bind_flags, "fi_read");
-
- sz = fi_readv(ep[0], &iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- target);
- api_cq_send_allowed(sz, cq_bind_flags, "fi_readv");
-
- sz = fi_readmsg(ep[0], &rma_msg, 0);
- api_cq_send_allowed(sz, cq_bind_flags, "fi_readmsg");
-
- sz = fi_inject_write(ep[0], source, 64,
- gni_addr[1], (uint64_t)target, mr_key[1]);
- cr_assert_eq(sz, 0);
-}
-
-Test(api_cq, msg)
-{
- cq_bind_flags = FI_SEND | FI_RECV;
- api_cq_bind(cq_bind_flags);
- api_cq_send_recv(BUF_SZ);
-}
-
-Test(api_cq, msg_send_only)
-{
- cq_bind_flags = FI_SEND;
- api_cq_bind(cq_bind_flags);
- api_cq_send_recv(BUF_SZ);
-}
-
-Test(api_cq, msg_recv_only)
-{
- cq_bind_flags = FI_RECV;
- api_cq_bind(cq_bind_flags);
- api_cq_send_recv(BUF_SZ);
-}
-
-Test(api_cq, msg_no_cq)
-{
- cq_bind_flags = 0;
- api_cq_bind(cq_bind_flags);
- api_cq_send_recv(BUF_SZ);
-}
-
-#define SOURCE_DATA 0xBBBB0000CCCCULL
-#define TARGET_DATA 0xAAAA0000DDDDULL
-#define FETCH_SOURCE_DATA 0xACEDACEDULL
-
-void api_cq_atomic(void)
-{
- ssize_t sz;
-
- /* u64 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT64, FI_ATOMIC_WRITE, target);
- api_cq_send_allowed(sz, cq_bind_flags, "fi_atomic");
-
- sz = fi_inject_atomic(ep[0], source, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT64, FI_MIN);
- cr_assert_eq(sz, 0);
-}
-
-Test(api_cq, atomic)
-{
- cq_bind_flags = FI_SEND | FI_RECV;
- api_cq_bind(cq_bind_flags);
- api_cq_atomic();
-}
-
-Test(api_cq, atomic_send_only)
-{
- cq_bind_flags = FI_SEND;
- api_cq_bind(cq_bind_flags);
- api_cq_atomic();
-}
-
-Test(api_cq, atomic_recv_only)
-{
- cq_bind_flags = FI_RECV;
- api_cq_bind(cq_bind_flags);
- api_cq_atomic();
-}
diff --git a/prov/gni/test/av.c b/prov/gni/test/av.c
deleted file mode 100644
index 1ae6ca0..0000000
--- a/prov/gni/test/av.c
+++ /dev/null
@@ -1,486 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stddef.h>
-#include <stdlib.h>
-
-#include "fi.h"
-#include "rdma/fi_domain.h"
-#include "rdma/fi_prov.h"
-
-#include "gnix.h"
-#include "gnix_av.h"
-
-#include <criterion/criterion.h>
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom;
-static struct fi_info *hints;
-static struct fi_info *fi;
-struct gnix_ep_name *fake_names;
-static struct fid_av *av;
-static struct gnix_fid_av *gnix_av;
-
-
-#define SIMPLE_EP_ENTRY(id) \
-{ \
- .gnix_addr = { \
- .device_addr = id, \
- .cdm_id = id, \
- }, \
- .name_type = id, \
- .cm_nic_cdm_id = id, \
- .cookie = id, \
-}
-
-#define SIMPLE_ADDR_COUNT 16
-static struct gnix_ep_name simple_ep_names[SIMPLE_ADDR_COUNT] = {
- SIMPLE_EP_ENTRY(1),
- SIMPLE_EP_ENTRY(2),
- SIMPLE_EP_ENTRY(3),
- SIMPLE_EP_ENTRY(4),
- SIMPLE_EP_ENTRY(5),
- SIMPLE_EP_ENTRY(6),
- SIMPLE_EP_ENTRY(7),
- SIMPLE_EP_ENTRY(8),
- SIMPLE_EP_ENTRY(9),
- SIMPLE_EP_ENTRY(10),
- SIMPLE_EP_ENTRY(11),
- SIMPLE_EP_ENTRY(12),
- SIMPLE_EP_ENTRY(13),
- SIMPLE_EP_ENTRY(14),
- SIMPLE_EP_ENTRY(15),
- SIMPLE_EP_ENTRY(16),
-};
-
-typedef char ep_name_string[GNIX_AV_MAX_STR_ADDR_LEN];
-static ep_name_string simple_ep_names_strs[SIMPLE_ADDR_COUNT];
-
-static void av_setup(void)
-{
- int ret = 0;
- int i;
- size_t len;
-
- memset(&simple_ep_names_strs, 0,
- (GNIX_AV_MAX_STR_ADDR_LEN * SIMPLE_ADDR_COUNT));
-
- for (i = 0; i < SIMPLE_ADDR_COUNT; i++) {
- len = GNIX_AV_MAX_STR_ADDR_LEN;
- gnix_av_straddr(NULL,
- (void *) &simple_ep_names[i],
- simple_ep_names_strs[i], &len);
- }
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = 4;
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert_eq(ret, FI_SUCCESS, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert_eq(ret, FI_SUCCESS, "fi_fabric");
-
- ret = fi_domain(fab, fi, &dom, NULL);
- cr_assert_eq(ret, FI_SUCCESS, "fi_domain");
-
-}
-
-static void av_teardown(void)
-{
- int ret = 0;
-
- ret = fi_close(&dom->fid);
- cr_assert_eq(ret, FI_SUCCESS, "failure in closing domain.");
- ret = fi_close(&fab->fid);
- cr_assert_eq(ret, FI_SUCCESS, "failure in closing fabric.");
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-}
-
-
-static void av_full_map_setup(void)
-{
- struct fi_av_attr av_table_attr = {
- .type = FI_AV_MAP,
- .count = 16,
- };
- int ret;
-
- av_setup();
-
- ret = fi_av_open(dom, &av_table_attr, &av, NULL);
- cr_assert_eq(ret, FI_SUCCESS, "failed to open av");
-
- gnix_av = container_of(av, struct gnix_fid_av, av_fid);
-}
-
-static void av_full_map_teardown(void)
-{
- int ret;
-
- ret = fi_close(&av->fid);
- cr_assert_eq(ret, FI_SUCCESS, "failed to close av");
-
- av_teardown();
-}
-
-static void av_full_table_setup(void)
-{
- struct fi_av_attr av_table_attr = {
- .type = FI_AV_TABLE,
- .count = 16,
- };
- int ret;
-
- av_setup();
-
- ret = fi_av_open(dom, &av_table_attr, &av, NULL);
- cr_assert_eq(ret, FI_SUCCESS, "failed to open av");
-
- gnix_av = container_of(av, struct gnix_fid_av, av_fid);
-}
-
-static void av_full_table_teardown(void)
-{
- int ret;
-
- ret = fi_close(&av->fid);
- cr_assert_eq(ret, FI_SUCCESS, "failed to close av");
-
- av_teardown();
-}
-
-TestSuite(av_bare, .init = av_setup, .fini = av_teardown, .disabled = false);
-
-TestSuite(av_full_map, .init = av_full_map_setup,
- .fini = av_full_map_teardown, .disabled = false);
-
-TestSuite(av_full_table, .init = av_full_table_setup,
- .fini = av_full_table_teardown, .disabled = false);
-
-static void invalid_addrlen_pointer_test(void)
-{
- int ret;
- fi_addr_t address = 0xdeadbeef;
- void *addr = (void *) 0xb00fbabe;
-
- /* while the pointers to address and addr aren't valid, they are
- * acceptable as stated by the manpage. This will only test for a
- * proper return code from fi_av_lookup()
- */
- ret = fi_av_lookup(av, address, addr, NULL);
- cr_assert_eq(ret, -FI_EINVAL);
-}
-
-Test(av_full_map, invalid_addrlen_pointer)
-{
- invalid_addrlen_pointer_test();
-}
-
-Test(av_full_table, invalid_addrlen_pointer)
-{
- invalid_addrlen_pointer_test();
-}
-
-static void remove_addr_test(void)
-{
- int ret;
- int i;
- fi_addr_t addresses[SIMPLE_ADDR_COUNT];
- fi_addr_t *compare;
-
- /* insert addresses */
- ret = fi_av_insert(av, (void *) simple_ep_names_strs, SIMPLE_ADDR_COUNT,
- addresses, 0, NULL);
- cr_assert_eq(ret, SIMPLE_ADDR_COUNT);
-
- /* check address contents */
- for (i = 0; i < SIMPLE_ADDR_COUNT; i++) {
- if (gnix_av->type == FI_AV_MAP) {
- compare = (fi_addr_t *) &simple_ep_names[i].gnix_addr;
- cr_assert_eq(*compare, addresses[i]);
- } else {
- cr_assert_eq(i, addresses[i]);
- }
- }
-
- /* remove addresses */
- ret = fi_av_remove(av, addresses, SIMPLE_ADDR_COUNT, 0);
- cr_assert_eq(ret, FI_SUCCESS);
-}
-
-Test(av_full_map, remove_addr)
-{
- remove_addr_test();
-}
-
-Test(av_full_table, remove_addr)
-{
- remove_addr_test();
-}
-
-static void lookup_invalid_test(void)
-{
- int ret;
- ep_name_string addresses[SIMPLE_ADDR_COUNT];
- size_t addrlen = GNIX_AV_MAX_STR_ADDR_LEN;
-
- /* test null addrlen */
- ret = fi_av_lookup(av, 0xdeadbeef, (void *) 0xdeadbeef, NULL);
- cr_assert_eq(ret, -FI_EINVAL);
-
- /* test null addr */
- ret = fi_av_lookup(av, 0xdeadbeef, NULL, &addrlen);
- cr_assert_eq(ret, -FI_EINVAL);
-
- /* test invalid lookup */
- if (gnix_av->type == FI_AV_TABLE) {
- ret = fi_av_lookup(av, 2000, &addresses[0], &addrlen);
- cr_assert_eq(ret, -FI_EINVAL);
-
- /* test within range, but not inserted case */
- ret = fi_av_lookup(av, 1, &addresses[1], &addrlen);
- cr_assert_eq(ret, -FI_EINVAL);
- } else {
- ret = fi_av_lookup(av, 0xdeadbeef, &addresses[0], &addrlen);
- cr_assert_eq(ret, -FI_ENOENT);
- }
-}
-
-Test(av_full_map, lookup_invalid)
-{
- lookup_invalid_test();
-}
-
-Test(av_full_table, lookup_invalid)
-{
- lookup_invalid_test();
-}
-
-static void lookup_test(void)
-{
- int ret;
- int i;
- fi_addr_t addresses[SIMPLE_ADDR_COUNT];
- fi_addr_t *compare;
- ep_name_string found;
- size_t addrlen = GNIX_AV_MAX_STR_ADDR_LEN;
-
- /* insert addresses */
- ret = fi_av_insert(av, (void *) simple_ep_names_strs, SIMPLE_ADDR_COUNT,
- addresses, 0, NULL);
- cr_assert_eq(ret, SIMPLE_ADDR_COUNT);
-
- /* check address contents */
- for (i = 0; i < SIMPLE_ADDR_COUNT; i++) {
- if (gnix_av->type == FI_AV_MAP) {
- compare = (fi_addr_t *) &simple_ep_names[i].gnix_addr;
- cr_assert_eq(*compare, addresses[i]);
- } else {
- cr_assert_eq(i, addresses[i]);
- }
- }
-
- ret = fi_av_lookup(av, addresses[1], &found, &addrlen);
- cr_assert_eq(ret, FI_SUCCESS);
-}
-
-Test(av_full_map, lookup)
-{
- lookup_test();
-}
-
-Test(av_full_table, lookup)
-{
- lookup_test();
-}
-
-static void straddr_test(void)
-{
- int ret;
- int i;
- const char *buf;
- char address[GNIX_AV_MAX_STR_ADDR_LEN];
- fi_addr_t addresses[SIMPLE_ADDR_COUNT];
- fi_addr_t *compare;
- size_t addrlen = GNIX_AV_MAX_STR_ADDR_LEN;
- char *pend;
- long int value;
-
- /* insert addresses */
- ret = fi_av_insert(av, (void *) simple_ep_names_strs, SIMPLE_ADDR_COUNT,
- addresses, 0, NULL);
- cr_assert_eq(ret, SIMPLE_ADDR_COUNT);
-
- /* check address contents */
- for (i = 0; i < SIMPLE_ADDR_COUNT; i++) {
- if (gnix_av->type == FI_AV_MAP) {
- compare = (fi_addr_t *) &simple_ep_names[i].gnix_addr;
- cr_assert_eq(*compare, addresses[i]);
- } else {
- cr_assert_eq(i, addresses[i]);
- }
- }
-
- buf = fi_av_straddr(av, &simple_ep_names[0], address, &addrlen);
-
- /* verify that a full address has been returned. */
- cr_assert_eq(addrlen, GNIX_AV_MAX_STR_ADDR_LEN);
-
- /* extract the first component */
- buf = strtok(address, ":");
- cr_assert_not_null(buf, "version not found");
-
- value = strtol(buf, &pend, 16);
-
- /* verify the version has been returned. */
- cr_assert_eq(GNIX_AV_STR_ADDR_VERSION, value, "Invalid version");
-
- /* extract the second component */
- buf = strtok(NULL, ":");
- cr_assert_not_null(buf, "device_addr not found");
-
- value = strtol(buf, &pend, 16);
-
- /* verify the device address has been returned. */
- cr_assert_eq(simple_ep_names[0].gnix_addr.device_addr, value,
- "Invalid device_addr");
-
- /* extract the third component */
- buf = strtok(NULL, ":");
- cr_assert_not_null(buf, "cdm_id not found");
-
- value = strtol(buf, &pend, 16);
-
- /* verify the cdm_id has been returned. */
- cr_assert_eq(simple_ep_names[0].gnix_addr.cdm_id, value,
- "Invalid cdm_id");
-
- /* extract the fourth component */
- buf = strtok(NULL, ":");
- cr_assert_not_null(buf, "name_type not found");
-
- value = strtol(buf, &pend, 10);
-
- /* verify the name_type has been returned. */
- cr_assert_eq(simple_ep_names[0].name_type, value, "Invalid name_type");
-
- /* extract the fifth component */
- buf = strtok(NULL, ":");
- cr_assert_not_null(buf, "cm_nic_cdm_id not found");
-
- value = strtol(buf, &pend, 16);
-
- /* verify the cm_nic_cdm_id has been returned. */
- cr_assert_eq(simple_ep_names[0].cm_nic_cdm_id, value,
- "Invalid cm_nic_cdm_id");
-
- /* extract the sixth component */
- buf = strtok(NULL, ":");
- cr_assert_not_null(buf, "cookie not found");
-
- value = strtol(buf, &pend, 16);
-
- /* verify the cookie has been returned. */
- cr_assert_eq(simple_ep_names[0].cookie, value, "Invalid cookie");
-
- /* check to see if additional component are specified */
- buf = strtok(NULL, ":");
- cr_assert_null(buf, "extra values specified");
-}
-
-Test(av_full_map, straddr)
-{
- straddr_test();
-}
-
-Test(av_full_table, straddr)
-{
- straddr_test();
-}
-
-#define TABLE_SIZE_INIT 16
-#define TABLE_SIZE_FINAL 1024
-
-Test(av_bare, test_capacity)
-{
- int ret, i;
- fi_addr_t addresses[TABLE_SIZE_FINAL];
- struct fi_av_attr av_table_attr = {
- .type = FI_AV_TABLE,
- .count = TABLE_SIZE_INIT,
- };
-
- ret = fi_av_open(dom, &av_table_attr, &av, NULL);
- cr_assert_eq(ret, FI_SUCCESS, "failed to open av");
-
- fake_names = (struct gnix_ep_name *)calloc(TABLE_SIZE_FINAL,
- sizeof(*fake_names));
- cr_assert_neq(fake_names, NULL);
-
- for (i = 0; i < TABLE_SIZE_INIT; i++) {
- fake_names[i].gnix_addr.device_addr = i + 100;
- fake_names[i].gnix_addr.cdm_id = i;
- fake_names[i].cm_nic_cdm_id = 0xbeef;
- fake_names[i].cookie = 0xdeadbeef;
- }
-
- ret = fi_av_insert(av, fake_names, TABLE_SIZE_INIT,
- addresses, 0, NULL);
- cr_assert_eq(ret, TABLE_SIZE_INIT, "av insert failed");
-
- /*
- * now add some more
- */
-
- for (i = TABLE_SIZE_INIT; i < TABLE_SIZE_FINAL; i++) {
- fake_names[i].gnix_addr.device_addr = i + 100;
- fake_names[i].gnix_addr.cdm_id = i;
- fake_names[i].cm_nic_cdm_id = 0xbeef;
- fake_names[i].cookie = 0xdeadbeef;
- }
-
- ret = fi_av_insert(av, &fake_names[TABLE_SIZE_INIT],
- TABLE_SIZE_FINAL - TABLE_SIZE_INIT,
- &addresses[TABLE_SIZE_INIT], 0, NULL);
- cr_assert_eq(ret, TABLE_SIZE_FINAL - TABLE_SIZE_INIT,
- "av insert failed");
-
-}
diff --git a/prov/gni/test/bitmap.c b/prov/gni/test/bitmap.c
deleted file mode 100644
index 0b5fc6e..0000000
--- a/prov/gni/test/bitmap.c
+++ /dev/null
@@ -1,576 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-/*
- * Created on: Apr 23, 2015
- * Author: jswaro
- */
-#include <stdlib.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <sys/time.h>
-
-#include <rdma/fi_errno.h>
-#include <gnix_bitmap.h>
-#include "common.h"
-
-#include <criterion/criterion.h>
-
-gnix_bitmap_t *test_bitmap = NULL;
-int call_free_bitmap = 0;
-
-#if HAVE_ATOMICS
-
-#define __gnix_set_block(bitmap, index, value) \
- atomic_store(&(bitmap)->arr[(index)], (value))
-#define __gnix_load_block(bitmap, index) atomic_load(&(bitmap->arr[(index)]))
-#else
-static inline void __gnix_set_block(gnix_bitmap_t *bitmap, int index,
- uint64_t value)
-{
- gnix_bitmap_block_t *block = &bitmap->arr[index];
-
- fastlock_acquire(&block->lock);
- block->val = value;
- fastlock_release(&block->lock);
-}
-
-static inline uint64_t __gnix_load_block(gnix_bitmap_t *bitmap, int index)
-{
- gnix_bitmap_block_t *block = &bitmap->arr[index];
- uint64_t ret;
-
- fastlock_acquire(&block->lock);
- ret = block->val;
- fastlock_release(&block->lock);
-
- return ret;
-}
-#endif
-
-void __gnix_bitmap_test_setup(void)
-{
- cr_assert(test_bitmap == NULL);
- test_bitmap = calloc(1, sizeof(*test_bitmap));
- cr_assert(test_bitmap != NULL);
-
- call_free_bitmap = 1;
-}
-
-void __gnix_bitmap_test_teardown(void)
-{
- if (call_free_bitmap) {
- _gnix_free_bitmap(test_bitmap);
- } else if (test_bitmap && test_bitmap->arr) {
- free(test_bitmap->arr);
- }
-
- cr_assert(test_bitmap != NULL);
- free(test_bitmap);
- test_bitmap = NULL;
-}
-
-
-static void __test_clean_bitmap_state(gnix_bitmap_t *bitmap,
- int _length, gnix_bitmap_state_e _state)
-{
- cr_assert(bitmap->arr != NULL);
- cr_assert(bitmap->length == _length);
- cr_assert(bitmap->state == _state);
-}
-
-static void __test_initialize_bitmap(gnix_bitmap_t *bitmap, int bits)
-{
- int ret = _gnix_alloc_bitmap(bitmap, bits);
-
- cr_assert(ret == 0);
- __test_clean_bitmap_state(bitmap, bits, GNIX_BITMAP_STATE_READY);
-}
-
-static void __test_initialize_bitmap_clean(gnix_bitmap_t *bitmap, int bits)
-{
- __test_initialize_bitmap(bitmap, bits);
- cr_assert(_gnix_bitmap_empty(bitmap));
-}
-
-static void __test_realloc_bitmap(gnix_bitmap_t *bitmap, int bits)
-{
- int ret = _gnix_realloc_bitmap(bitmap, bits);
-
- cr_assert(ret == 0);
- __test_clean_bitmap_state(bitmap, bits, GNIX_BITMAP_STATE_READY);
-}
-
-static void __test_realloc_bitmap_clean(gnix_bitmap_t *bitmap, int initial,
- int next)
-{
- __test_initialize_bitmap(bitmap, initial);
- __test_realloc_bitmap(bitmap, next);
- cr_assert(_gnix_bitmap_empty(bitmap));
-}
-
-static void __test_free_bitmap_clean(gnix_bitmap_t *bitmap)
-{
- int ret = _gnix_free_bitmap(bitmap);
-
- cr_assert(ret == 0);
- cr_assert(bitmap->arr == NULL);
- cr_assert(bitmap->length == 0);
- cr_assert(bitmap->state == GNIX_BITMAP_STATE_FREE);
-}
-
-/*
- * Basic functionality tests for the gnix_bitmap_t object
- */
-
-TestSuite(gnix_bitmap,
- .init = __gnix_bitmap_test_setup,
- .fini = __gnix_bitmap_test_teardown);
-
-Test(gnix_bitmap, uninitialized)
-{
- cr_assert(test_bitmap->arr == NULL);
- cr_assert(test_bitmap->length == 0);
- cr_assert(test_bitmap->state == GNIX_BITMAP_STATE_UNINITIALIZED);
-
- call_free_bitmap = 0;
-}
-
-Test(gnix_bitmap, initialize_128)
-{
- __test_initialize_bitmap(test_bitmap, 128);
-
- call_free_bitmap = 0;
-}
-
-Test(gnix_bitmap, initialize_1)
-{
- __test_initialize_bitmap(test_bitmap, 1);
-
- call_free_bitmap = 0;
-}
-
-Test(gnix_bitmap, initialize_0)
-{
- int ret;
-
- ret = _gnix_alloc_bitmap(test_bitmap, 0);
- cr_assert(ret == -FI_EINVAL);
-
- call_free_bitmap = 0;
-}
-
-Test(gnix_bitmap, already_initialized)
-{
- int ret;
-
- __test_initialize_bitmap(test_bitmap, 128);
-
- ret = _gnix_alloc_bitmap(test_bitmap, 128);
- cr_assert(ret == -FI_EINVAL);
-
- call_free_bitmap = 0;
-}
-
-Test(gnix_bitmap, destroy_bitmap)
-{
- __test_initialize_bitmap(test_bitmap, 128);
-
- __test_free_bitmap_clean(test_bitmap);
-}
-
-Test(gnix_bitmap, destroy_bitmap_uninitialized)
-{
- int ret;
-
- ret = _gnix_free_bitmap(test_bitmap);
- cr_assert(ret == -FI_EINVAL);
- cr_expect(test_bitmap->arr == NULL);
- cr_expect(test_bitmap->length == 0);
- cr_expect(test_bitmap->state == GNIX_BITMAP_STATE_UNINITIALIZED);
-}
-
-Test(gnix_bitmap, destroy_bitmap_already_freed)
-{
- int ret;
-
- __test_initialize_bitmap(test_bitmap, 128);
-
- __test_free_bitmap_clean(test_bitmap);
-
- ret = _gnix_free_bitmap(test_bitmap);
- cr_assert(ret == -FI_EINVAL);
- cr_expect(test_bitmap->arr == NULL);
- cr_expect(test_bitmap->length == 0);
- cr_expect(test_bitmap->state == GNIX_BITMAP_STATE_FREE);
-}
-
-Test(gnix_bitmap, realloc_63)
-{
- __test_realloc_bitmap_clean(test_bitmap, 128, 63);
-}
-
-Test(gnix_bitmap, realloc_64)
-{
- __test_realloc_bitmap_clean(test_bitmap, 128, 64);
-}
-
-Test(gnix_bitmap, realloc_65)
-{
- __test_realloc_bitmap_clean(test_bitmap, 128, 65);
-}
-
-Test(gnix_bitmap, realloc_255)
-{
- __test_realloc_bitmap_clean(test_bitmap, 128, 255);
-}
-
-Test(gnix_bitmap, realloc_256)
-{
- __test_realloc_bitmap_clean(test_bitmap, 128, 256);
-}
-
-Test(gnix_bitmap, realloc_257)
-{
- __test_realloc_bitmap_clean(test_bitmap, 128, 257);
-}
-
-Test(gnix_bitmap, realloc_63_check_bits)
-{
- __test_realloc_bitmap_clean(test_bitmap, 128, 63);
-}
-
-Test(gnix_bitmap, realloc_64_check_bits)
-{
- __test_realloc_bitmap_clean(test_bitmap, 128, 64);
-}
-
-Test(gnix_bitmap, realloc_65_check_bits)
-{
- __test_realloc_bitmap_clean(test_bitmap, 128, 65);
-}
-
-Test(gnix_bitmap, realloc_255_check_bits)
-{
- __test_realloc_bitmap_clean(test_bitmap, 128, 255);
-}
-
-Test(gnix_bitmap, realloc_256_check_bits)
-{
- __test_realloc_bitmap_clean(test_bitmap, 128, 256);
-}
-
-Test(gnix_bitmap, realloc_257_check_bits)
-{
- __test_realloc_bitmap_clean(test_bitmap, 128, 257);
-}
-
-Test(gnix_bitmap, bit_set_test_pass)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- _gnix_set_bit(test_bitmap, 1);
-
- cr_assert(_gnix_test_bit(test_bitmap, 1));
-}
-
-Test(gnix_bitmap, bit_set_test_fail)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- _gnix_set_bit(test_bitmap, 1);
-
- cr_assert(!_gnix_test_bit(test_bitmap, 0));
-}
-
-Test(gnix_bitmap, bit_set_clear)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- _gnix_set_bit(test_bitmap, 1);
-
- cr_assert(_gnix_test_bit(test_bitmap, 1));
-
- _gnix_clear_bit(test_bitmap, 1);
-
- cr_assert(!_gnix_test_bit(test_bitmap, 1));
-}
-
-Test(gnix_bitmap, bit_clear)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- _gnix_clear_bit(test_bitmap, 1);
-
- cr_assert(!_gnix_test_bit(test_bitmap, 1));
-}
-
-Test(gnix_bitmap, bit_set)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- _gnix_set_bit(test_bitmap, 1);
-}
-
-Test(gnix_bitmap, bit_test_and_set_unset)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- cr_assert(!_gnix_test_and_set_bit(test_bitmap, 1));
-}
-
-Test(gnix_bitmap, bit_test_and_set_already_set)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- _gnix_set_bit(test_bitmap, 1);
- cr_assert(_gnix_test_bit(test_bitmap, 1));
-
- cr_assert(_gnix_test_and_set_bit(test_bitmap, 1));
-}
-
-Test(gnix_bitmap, bit_test_and_clear_unset)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- cr_assert(!_gnix_test_and_clear_bit(test_bitmap, 1));
-}
-
-Test(gnix_bitmap, bit_test_and_clear_already_set)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- _gnix_set_bit(test_bitmap, 1);
- cr_assert(_gnix_test_bit(test_bitmap, 1));
-
- cr_assert(_gnix_test_and_clear_bit(test_bitmap, 1));
-}
-
-Test(gnix_bitmap, ffs_clean_bitmap)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- cr_assert(_gnix_find_first_set_bit(test_bitmap) == -FI_EAGAIN);
-}
-
-Test(gnix_bitmap, ffs_first_bit_set)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- _gnix_set_bit(test_bitmap, 0);
-
- cr_assert(_gnix_find_first_set_bit(test_bitmap) == 0);
-}
-
-Test(gnix_bitmap, ffs_seventeen_set)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- _gnix_set_bit(test_bitmap, 17);
-
- cr_assert(_gnix_find_first_set_bit(test_bitmap) == 17);
-}
-
-Test(gnix_bitmap, ffz_clean_bitmap)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- cr_assert(_gnix_find_first_zero_bit(test_bitmap) == 0);
-}
-
-Test(gnix_bitmap, ffz_full_bitmap)
-{
- int i;
-
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- for (i = 0; i < test_bitmap->length; ++i) {
- _gnix_set_bit(test_bitmap, i);
- cr_assert(_gnix_test_bit(test_bitmap, i));
- }
-
- cr_assert(_gnix_find_first_zero_bit(test_bitmap) == -FI_EAGAIN);
-}
-
-Test(gnix_bitmap, ffz_first_half_set)
-{
- int i;
-
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- for (i = 0; i < 32 ; ++i) {
- _gnix_set_bit(test_bitmap, i);
- cr_assert(_gnix_test_bit(test_bitmap, i));
- }
-
- cr_expect(test_bitmap->length == 64);
- cr_expect(i == 32);
- cr_assert(_gnix_find_first_zero_bit(test_bitmap) == i);
-}
-
-Test(gnix_bitmap, map_fill_0)
-{
- int i;
-
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- for (i = 0; i < test_bitmap->length; ++i) {
- _gnix_set_bit(test_bitmap, i);
- cr_assert(_gnix_test_bit(test_bitmap, i));
- }
-
- cr_assert(_gnix_bitmap_full(test_bitmap));
-
- _gnix_fill_bitmap(test_bitmap, 0);
-
- cr_assert(_gnix_bitmap_empty(test_bitmap));
-}
-
-Test(gnix_bitmap, map_fill_1)
-{
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- _gnix_fill_bitmap(test_bitmap, 1);
-
- cr_assert(_gnix_bitmap_full(test_bitmap));
-}
-
-Test(gnix_bitmap, bitmap_load)
-{
- gnix_bitmap_value_t expected = ~0;
-
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- _gnix_fill_bitmap(test_bitmap, 1);
-
- cr_assert(expected == __gnix_load_block(test_bitmap, 0));
-}
-
-Test(gnix_bitmap, bitmap_set)
-{
- gnix_bitmap_value_t expected = ~0;
-
- __test_initialize_bitmap_clean(test_bitmap, 64);
-
- __gnix_set_block(test_bitmap, 0, expected);
-
- cr_assert(__gnix_load_block(test_bitmap, 0) == expected);
-}
-
-
-Test(gnix_bitmap, fill_bitmap_60_ffz_eagain)
-{
- int i;
-
- __test_initialize_bitmap_clean(test_bitmap, 60);
-
- for (i = 0; i < 60; ++i)
- _gnix_set_bit(test_bitmap, i);
-
- cr_assert(_gnix_find_first_zero_bit(test_bitmap) == -FI_EAGAIN);
-}
-
-Test(gnix_bitmap, fill_bitmap_60_ffs_eagain)
-{
- int i;
-
- __test_initialize_bitmap_clean(test_bitmap, 60);
-
- /* this will succeed because set_bit doesn't account for bounds of the
- * bitmap as the user should be responsible for handling the bitmap
- * properly.
- */
- for (i = 60; i < 64; ++i)
- _gnix_set_bit(test_bitmap, i);
-
- cr_assert(_gnix_find_first_set_bit(test_bitmap) == -FI_EAGAIN);
-}
-
-TestSuite(perf_bitmap,
- .init = __gnix_bitmap_test_setup,
- .fini = __gnix_bitmap_test_teardown,
- .disabled = true);
-
-Test(perf_bitmap, performance_set_test)
-{
- int i, j;
- int secs, usec;
- struct timeval start, end;
-
- __test_initialize_bitmap_clean(test_bitmap, 8192);
-
- gettimeofday(&start, 0);
- for (i = 0; i < 100000; ++i) {
- j = i % 8192;
- _gnix_set_bit(test_bitmap, j);
- cr_assert(_gnix_test_bit(test_bitmap, j));
- _gnix_clear_bit(test_bitmap, j);
- cr_assert(!_gnix_test_bit(test_bitmap, j));
- }
- gettimeofday(&end, 0);
-
- calculate_time_difference(&start, &end, &secs, &usec);
-
- cr_assert(_gnix_bitmap_empty(test_bitmap));
-
- cr_expect(secs < 1);
-}
-
-Test(perf_bitmap, performance_set_test_random)
-{
- int i, j;
- int secs, usec;
- struct timeval start, end;
-
- srand(time(NULL));
-
- __test_initialize_bitmap_clean(test_bitmap, 8192);
-
- gettimeofday(&start, 0);
- for (i = 0; i < 100000; ++i) {
- j = rand() % 8192;
- _gnix_set_bit(test_bitmap, j);
- cr_assert(_gnix_test_bit(test_bitmap, j));
- _gnix_clear_bit(test_bitmap, j);
- cr_assert(!_gnix_test_bit(test_bitmap, j));
- }
- gettimeofday(&end, 0);
-
- calculate_time_difference(&start, &end, &secs, &usec);
-
- cr_assert(_gnix_bitmap_empty(test_bitmap));
-
- cr_expect(secs < 1);
-}
-
diff --git a/prov/gni/test/buddy_allocator.c b/prov/gni/test/buddy_allocator.c
deleted file mode 100644
index 26da442..0000000
--- a/prov/gni/test/buddy_allocator.c
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All
- * rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "gnix_buddy_allocator.h"
-#include <criterion/criterion.h>
-#include <time.h>
-
-#define LEN (1024 * 1024) /* buddy_handle->len */
-#define MAX_LEN (LEN / 2) /* buddy_handle->max */
-#define MIN_LEN MIN_BLOCK_SIZE
-
-long *buf = NULL; /* buddy_handle->base */
-gnix_buddy_alloc_handle_t *buddy_handle;
-
-struct ptrs_t {
- void *ptr; /* ptrs alloc'd by buddy_alloc */
- uint32_t size; /* size of the ptr */
-} *ptrs;
-
-void buddy_allocator_setup(void)
-{
- int ret;
-
- ptrs = calloc(LEN / MIN_LEN, sizeof(struct ptrs_t));
- cr_assert(ptrs, "buddy_allocator_setup");
-
- buf = calloc(LEN, sizeof(long));
- cr_assert(buf, "buddy_allocator_setup");
-
- ret = _gnix_buddy_allocator_create(buf, LEN, MAX_LEN, &buddy_handle);
- cr_assert(!ret, "_gnix_buddy_allocator_create");
-}
-
-void buddy_allocator_teardown(void)
-{
- int ret;
-
- ret = _gnix_buddy_allocator_destroy(buddy_handle);
- cr_assert(!ret, "_gnix_buddy_allocator_destroy");
-
- free(ptrs);
- free(buf);
-}
-
-/* Test invalid parameters for setup */
-void buddy_allocator_setup_error(void)
-{
- int ret;
-
- ret = _gnix_buddy_allocator_create(NULL, LEN, MAX_LEN, &buddy_handle);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_buddy_allocator_create(buf, 0, MAX_LEN, &buddy_handle);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_buddy_allocator_create(buf, LEN, LEN + 1, &buddy_handle);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_buddy_allocator_create(buf, LEN, 0, &buddy_handle);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_buddy_allocator_create(buf, LEN, MAX_LEN, NULL);
- cr_assert_eq(ret, -FI_EINVAL);
-}
-
-/* Test invalid parameters for teardown */
-void buddy_allocator_teardown_error(void)
-{
- int ret;
-
- ret = _gnix_buddy_allocator_destroy(NULL);
- cr_assert_eq(ret, -FI_EINVAL);
-}
-
-/* Sequential alloc */
-void do_alloc(uint32_t len)
-{
- uint32_t i = 0, ret;
-
- /* Allocate all the memory and write to each block */
- for (; i < LEN / len; i++) {
- ptrs[i].size = len;
- ret = _gnix_buddy_alloc(buddy_handle, &ptrs[i].ptr, len);
- cr_assert(!ret, "_gnix_buddy_alloc");
- memset(ptrs[i].ptr, 0, len);
- }
-
- /* Ensure that all free lists are empty */
- for (i = 0; i < buddy_handle->nlists; i++) {
- ret = dlist_empty(buddy_handle->lists + i);
- cr_assert_eq(ret, 1);
- }
-}
-
-/* Sequential free */
-void do_free(uint32_t len)
-{
- int i = 0, ret;
-
- /* Free all allocated blocks */
- for (i = 0; i < LEN / len; i++) {
- ret = _gnix_buddy_free(buddy_handle, ptrs[i].ptr, ptrs[i].size);
- cr_assert(!ret, "_gnix_buddy_free");
- }
-
- /* Ensure that every free list except the last is empty */
- for (i = 0; i < buddy_handle->nlists - 1; i++) {
- ret = dlist_empty(buddy_handle->lists + i);
- cr_assert_eq(ret, 1);
- }
- ret = dlist_empty(buddy_handle->lists + i);
- cr_assert_eq(ret, 0);
-}
-
-TestSuite(buddy_allocator, .init = buddy_allocator_setup,
- .fini = buddy_allocator_teardown, .disabled = false);
-
-/* Sequential alloc and frees */
-Test(buddy_allocator, sequential_alloc_free)
-{
- uint32_t i = MIN_LEN;
-
- for (i = MIN_LEN; i <= MAX_LEN; i *= 2) {
- do_alloc(i);
- do_free(i);
- }
-}
-
-/* Pseudo random allocs and frees */
-Test(buddy_allocator, random_alloc_free)
-{
- int i = 0, j = 0, ret;
-
- srand((unsigned) time(NULL));
-
- for (j = MIN_LEN; j <= MAX_LEN; j *= 2) {
- do {
- ret = rand() % 100;
-
- if (ret <= 49) {
- /* ~50% chance to alloc min size blocks*/
- ptrs[i].size = MIN_BLOCK_SIZE;
- } else if (ret >= 50 &&
- ret <= 87) {
- /* ~37% chance to alloc blocks of size
- * [MIN_BLOCK_SIZE * 2, MAX_BLOCK_SIZE / 2]
- */
- ptrs[i].size = OFFSET(MIN_BLOCK_SIZE,
- (rand() %
- (buddy_handle->nlists -
- 1)) + 1);
- } else {
- /* ~13% chance to alloc max size blocks */
- ptrs[i].size = buddy_handle->max;
- }
-
- ret = _gnix_buddy_alloc(buddy_handle, &ptrs[i].ptr,
- ptrs[i].size);
- cr_assert_neq(ret, -FI_EINVAL);
-
- i++;
- } while (ret != -FI_ENOMEM);
-
- /* Free all allocated blocks */
- for (i -= 2; i >= 0; i--) {
- ret = _gnix_buddy_free(buddy_handle, ptrs[i].ptr,
- ptrs[i].size);
- cr_assert(!ret, "_gnix_buddy_free");
- }
-
- /* Ensure that every free list except the last is empty */
- for (i = 0; i < buddy_handle->nlists - 1; i++) {
- ret = dlist_empty(buddy_handle->lists + i);
- cr_assert_eq(ret, 1);
- }
- ret = dlist_empty(buddy_handle->lists + i);
- cr_assert_eq(ret, 0);
-
- i = 0;
- }
-}
-
-Test(buddy_allocator, alloc_free_error)
-{
- int ret;
- void *tmp;
-
- do_alloc(MIN_LEN);
-
- /* Request one additional block */
- ret = _gnix_buddy_alloc(buddy_handle, &tmp, MIN_LEN);
- cr_assert_eq(ret, -FI_ENOMEM);
-
- do_free(MIN_LEN);
-}
-
-/* Test invalid buddy alloc and free parameters */
-Test(buddy_allocator, parameter_error)
-{
- int ret;
-
- buddy_allocator_setup_error();
- buddy_allocator_teardown_error();
-
- /* BEGIN: Alloc, invalid parameters */
- ret = _gnix_buddy_alloc(NULL, ptrs->ptr, MAX_LEN);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_buddy_alloc(buddy_handle, ptrs->ptr, MAX_LEN + 1);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_buddy_alloc(buddy_handle, ptrs->ptr, 0);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_buddy_alloc(buddy_handle, NULL, MAX_LEN);
- cr_assert_eq(ret, -FI_EINVAL);
- /* END: Alloc, invalid parameters */
-
- /* BEGIN: Free, invalid parameters */
- ret = _gnix_buddy_free(NULL, ptrs->ptr, MAX_LEN);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_buddy_free(buddy_handle, NULL, MAX_LEN);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_buddy_free(buddy_handle, buf - 1, MAX_LEN);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_buddy_free(buddy_handle, buf + LEN, MAX_LEN);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_buddy_free(buddy_handle, buf, MAX_LEN + 1);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_buddy_free(buddy_handle, buf - 1, 0);
- cr_assert_eq(ret, -FI_EINVAL);
- /* END: Free, invalid parameters */
-}
diff --git a/prov/gni/test/cancel.c b/prov/gni/test/cancel.c
deleted file mode 100644
index 42780d7..0000000
--- a/prov/gni/test/cancel.c
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <pthread.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include "gnix_vc.h"
-#include "gnix_cm_nic.h"
-#include "gnix_hashtable.h"
-#include "gnix_rma.h"
-#include "gnix_ep.h"
-
-#include <criterion/criterion.h>
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom;
-static struct fid_ep *ep[2];
-static struct fid_av *av;
-static struct fi_info *hints;
-static struct fi_info *fi;
-void *ep_name[2];
-size_t gni_addr[2];
-static struct fid_cq *msg_cq[2];
-static struct fi_cq_attr cq_attr;
-
-#define BUF_SZ (8*1024)
-char *target;
-char *source;
-struct fid_mr *rem_mr, *loc_mr;
-uint64_t mr_key;
-
-void cancel_setup(void)
-{
- int ret = 0;
- struct fi_av_attr attr;
- size_t addrlen = 0;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = 4;
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- ret = fi_domain(fab, fi, &dom, NULL);
- cr_assert(!ret, "fi_domain");
-
- attr.type = FI_AV_MAP;
- attr.count = 16;
-
- ret = fi_av_open(dom, &attr, &av, NULL);
- cr_assert(!ret, "fi_av_open");
-
- ret = fi_endpoint(dom, fi, &ep[0], NULL);
- cr_assert(!ret, "fi_endpoint");
-
- cq_attr.format = FI_CQ_FORMAT_CONTEXT;
- cq_attr.size = 1024;
- cq_attr.wait_obj = 0;
-
- ret = fi_cq_open(dom, &cq_attr, &msg_cq[0], 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_cq_open(dom, &cq_attr, &msg_cq[1], 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_ep_bind(ep[0], &msg_cq[0]->fid, FI_SEND | FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
- cr_assert(addrlen > 0);
-
- ep_name[0] = malloc(addrlen);
- cr_assert(ep_name[0] != NULL);
-
- ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_endpoint(dom, fi, &ep[1], NULL);
- cr_assert(!ret, "fi_endpoint");
-
- ret = fi_ep_bind(ep[1], &msg_cq[1]->fid, FI_SEND | FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
-
- ep_name[1] = malloc(addrlen);
- cr_assert(ep_name[1] != NULL);
-
- ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_av_insert(av, ep_name[0], 1, &gni_addr[0], 0,
- NULL);
- cr_assert(ret == 1);
-
- ret = fi_av_insert(av, ep_name[1], 1, &gni_addr[1], 0,
- NULL);
- cr_assert(ret == 1);
-
- ret = fi_ep_bind(ep[0], &av->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_ep_bind(ep[1], &av->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_enable(ep[0]);
- cr_assert(!ret, "fi_ep_enable");
-
- ret = fi_enable(ep[1]);
- cr_assert(!ret, "fi_ep_enable");
-
- target = malloc(BUF_SZ);
- assert(target);
-
- source = malloc(BUF_SZ);
- assert(source);
-
- ret = fi_mr_reg(dom, target, BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, &rem_mr, &target);
- cr_assert_eq(ret, 0);
-
- ret = fi_mr_reg(dom, source, BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, &loc_mr, &source);
- cr_assert_eq(ret, 0);
-
- mr_key = fi_mr_key(rem_mr);
-}
-
-void cancel_teardown(void)
-{
- int ret = 0;
-
- fi_close(&loc_mr->fid);
- fi_close(&rem_mr->fid);
-
- free(target);
- free(source);
-
- ret = fi_close(&ep[0]->fid);
- cr_assert(!ret, "failure in closing ep.");
-
- ret = fi_close(&ep[1]->fid);
- cr_assert(!ret, "failure in closing ep.");
-
- ret = fi_close(&msg_cq[0]->fid);
- cr_assert(!ret, "failure in send cq.");
-
- ret = fi_close(&msg_cq[1]->fid);
- cr_assert(!ret, "failure in recv cq.");
-
- ret = fi_close(&av->fid);
- cr_assert(!ret, "failure in closing av.");
-
- ret = fi_close(&dom->fid);
- cr_assert(!ret, "failure in closing domain.");
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
- free(ep_name[0]);
- free(ep_name[1]);
-}
-
-void cancel_init_data(char *buf, int len, char seed)
-{
- int i;
-
- for (i = 0; i < len; i++) {
- buf[i] = seed++;
- }
-}
-
-int cancel_check_data(char *buf1, char *buf2, int len)
-{
- int i;
-
- for (i = 0; i < len; i++) {
- if (buf1[i] != buf2[i]) {
- printf("data mismatch, elem: %d, exp: %x, act: %x\n",
- i, buf1[i], buf2[i]);
- return 0;
- }
- }
-
- return 1;
-}
-
-/*******************************************************************************
- * Test MSG functions
- ******************************************************************************/
-
-TestSuite(gnix_cancel, .init = cancel_setup, .fini = cancel_teardown,
- .disabled = false);
-
-Test(gnix_cancel, cancel_ep_send)
-{
- int ret;
- struct gnix_fid_ep *gnix_ep;
- struct gnix_fab_req *req;
- struct fi_cq_err_entry buf;
- struct gnix_vc *vc;
- void *foobar_ptr = NULL;
- gnix_ht_key_t *key;
-
- /* simulate a posted request */
- gnix_ep = container_of(ep[0], struct gnix_fid_ep, ep_fid);
- req = _gnix_fr_alloc(gnix_ep);
-
- req->msg.send_addr = 0xdeadbeef;
- req->msg.send_len = 128;
- req->user_context = foobar_ptr;
- req->type = GNIX_FAB_RQ_SEND;
-
- /* allocate, store vc */
- ret = _gnix_vc_alloc(gnix_ep, NULL, &vc);
- cr_assert(ret == FI_SUCCESS, "_gnix_vc_alloc failed");
-
- key = (gnix_ht_key_t *)&gnix_ep->my_name.gnix_addr;
- ret = _gnix_ht_insert(gnix_ep->vc_ht, *key, vc);
- cr_assert(!ret);
-
- /* make a dummy request */
- fastlock_acquire(&vc->tx_queue_lock);
- slist_insert_head(&req->slist, &vc->tx_queue);
- fastlock_release(&vc->tx_queue_lock);
-
- /* cancel simulated request */
- ret = fi_cancel(&ep[0]->fid, foobar_ptr);
- cr_assert(ret == FI_SUCCESS, "fi_cancel failed");
-
- /* check for event */
- ret = fi_cq_readerr(msg_cq[0], &buf, FI_SEND);
- cr_assert(ret == 1, "did not find one error event");
-
- cr_assert(buf.buf == (void *) 0xdeadbeef, "buffer mismatch");
- cr_assert(buf.data == 0, "data mismatch");
- cr_assert(buf.err == FI_ECANCELED, "error code mismatch");
- cr_assert(buf.prov_errno == FI_ECANCELED, "prov error code mismatch");
- cr_assert(buf.len == 128, "length mismatch");
-}
-
-Test(gnix_cancel, cancel_ep_recv)
-{
- int ret;
- struct fi_cq_err_entry buf;
-
- /* simulate a posted request */
- ret = fi_recv(ep[0], (void *) 0xdeadbeef, 128, 0, FI_ADDR_UNSPEC,
- (void *) 0xcafebabe);
- cr_assert(ret == FI_SUCCESS, "fi_recv failed");
-
- /* cancel simulated request */
- ret = fi_cancel(&ep[0]->fid, (void *) 0xcafebabe);
- cr_assert(ret == FI_SUCCESS, "fi_cancel failed");
-
- /* check for event */
- ret = fi_cq_readerr(msg_cq[0], &buf, FI_RECV);
- cr_assert(ret == 1, "did not find one error event");
-
- cr_assert(buf.buf == (void *) 0xdeadbeef, "buffer mismatch");
- cr_assert(buf.data == 0, "data mismatch");
- cr_assert(buf.err == FI_ECANCELED, "error code mismatch");
- cr_assert(buf.prov_errno == FI_ECANCELED, "prov error code mismatch");
- cr_assert(buf.len == 128, "length mismatch");
-}
-
-Test(gnix_cancel, cancel_ep_no_event)
-{
- int ret;
-
- ret = fi_cancel(&ep[0]->fid, NULL);
- cr_assert(ret == -FI_ENOENT, "fi_cancel failed");
-}
-
-Test(gnix_cancel, cancel_ep_no_domain)
-{
- int ret;
- struct gnix_fid_ep *gnix_ep;
- struct gnix_fid_domain *gnix_dom;
-
- /* simulate a disconnected endpoint */
- gnix_ep = container_of(ep[0], struct gnix_fid_ep, ep_fid);
- gnix_dom = gnix_ep->domain;
- gnix_ep->domain = NULL;
-
- /* run test */
- ret = fi_cancel(&ep[0]->fid, NULL);
- cr_assert(ret == -FI_EDOMAIN, "fi_cancel failed");
-
- /* reconnect */
- gnix_ep->domain = gnix_dom;
-}
diff --git a/prov/gni/test/cntr.c b/prov/gni/test/cntr.c
deleted file mode 100644
index 5f596d7..0000000
--- a/prov/gni/test/cntr.c
+++ /dev/null
@@ -1,797 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <pthread.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include "gnix_vc.h"
-#include "gnix_cm_nic.h"
-#include "gnix_hashtable.h"
-#include "gnix_rma.h"
-
-#include <criterion/criterion.h>
-
-#if 1
-#define dbg_printf(...)
-#else
-#define dbg_printf(...) \
-do { \
- printf(__VA_ARGS__); \
- fflush(stdout); \
-} while (0)
-#endif
-
-#define NUM_EPS 5
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom;
-static struct fid_ep *ep[NUM_EPS];
-static struct fid_av *av;
-static struct fi_info *hints;
-static struct fi_info *fi;
-static void *ep_name[NUM_EPS];
-static size_t gni_addr[NUM_EPS];
-static struct fid_cq *send_cq;
-static struct fid_cq *recv_cq;
-static struct fi_cq_attr cq_attr;
-static struct fid_cntr *write_cntr, *read_cntr, *rcv_cntr;
-static struct fi_cntr_attr cntr_attr = {.events = FI_CNTR_EVENTS_COMP,
- .flags = 0};
-
-#define BUF_SZ (64*1024)
-char *target;
-char *source;
-struct fid_mr *rem_mr, *loc_mr;
-uint64_t mr_key;
-
-static inline void cntr_setup_eps(void)
-{
- int i, ret;
- struct fi_av_attr attr;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = 4;
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- ret = fi_domain(fab, fi, &dom, NULL);
- cr_assert(!ret, "fi_domain");
-
- attr.type = FI_AV_MAP;
- attr.count = 16;
-
- ret = fi_av_open(dom, &attr, &av, NULL);
- cr_assert(!ret, "fi_av_open");
-
- for (i = 0; i < NUM_EPS; i++) {
- ret = fi_endpoint(dom, fi, &ep[i], NULL);
- cr_assert(!ret, "fi_endpoint");
- }
-}
-
-static inline void cntr_setup_cqs(void)
-{
- int ret;
-
- cq_attr.format = FI_CQ_FORMAT_CONTEXT;
- cq_attr.size = 1024;
- cq_attr.wait_obj = FI_WAIT_NONE;
-
- ret = fi_cq_open(dom, &cq_attr, &send_cq, 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_cq_open(dom, &cq_attr, &recv_cq, 0);
- cr_assert(!ret, "fi_cq_open");
-
-}
-
-static inline void cntr_setup_cntrs(void)
-{
- int ret;
-
- ret = fi_cntr_open(dom, &cntr_attr, &write_cntr, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_cntr_open(dom, &cntr_attr, &read_cntr, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_cntr_open(dom, &cntr_attr, &rcv_cntr, 0);
- cr_assert(!ret, "fi_cntr_open");
-
-}
-
-static inline void cntr_setup_av(void)
-{
- int i, ret;
- size_t addrlen = 0;
-
- ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
- cr_assert(addrlen > 0);
-
- for (i = 0; i < NUM_EPS; i++) {
- ep_name[i] = malloc(addrlen);
- cr_assert(ep_name[i] != NULL);
-
- ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_av_insert(av, ep_name[i], 1, &gni_addr[i], 0,
- NULL);
- cr_assert(ret == 1);
- }
-
- for (i = 0; i < NUM_EPS; i++) {
- ret = fi_ep_bind(ep[i], &av->fid, 0);
- cr_assert(!ret, "fi_ep_bind av");
- }
-}
-
-static inline void cntr_setup_bind_cqs(void)
-{
- int i, ret;
-
- for (i = 0; i < NUM_EPS; i++) {
- ret = fi_ep_bind(ep[i], &send_cq->fid, FI_SEND);
- cr_assert(!ret, "fi_ep_bind cq");
-
- ret = fi_ep_bind(ep[i], &recv_cq->fid, FI_RECV);
- cr_assert(!ret, "fi_ep_bind cq");
- }
-}
-
-static inline void cntr_setup_bind_cntrs(void)
-{
- int i, ret;
-
- for (i = 0; i < NUM_EPS; i++) {
- ret = fi_ep_bind(ep[i], &write_cntr->fid, FI_WRITE | FI_SEND);
- cr_assert(!ret, "fi_ep_bind cntr");
-
- ret = fi_ep_bind(ep[i], &read_cntr->fid, FI_READ);
- cr_assert(!ret, "fi_ep_bind cntr");
-
- ret = fi_ep_bind(ep[i], &rcv_cntr->fid, FI_RECV);
- cr_assert(!ret, "fi_ep_bind cntr");
- }
-}
-
-static inline void cntr_setup_enable_ep(void)
-{
- int i, ret;
-
- for (i = 0; i < NUM_EPS; i++) {
- ret = fi_enable(ep[i]);
- cr_assert(!ret, "fi_ep_enable");
- }
-}
-
-static inline void cntr_setup_mr(void)
-{
- int ret;
-
- target = malloc(BUF_SZ);
- assert(target);
-
- source = malloc(BUF_SZ);
- assert(source);
-
- ret = fi_mr_reg(dom, target, BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, &rem_mr, &target);
- cr_assert_eq(ret, 0);
-
- ret = fi_mr_reg(dom, source, BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, &loc_mr, &source);
- cr_assert_eq(ret, 0);
-
- mr_key = fi_mr_key(rem_mr);
-}
-
-static void cntr_setup(void)
-{
- cntr_setup_eps();
- cntr_setup_av();
- cntr_setup_cqs();
- cntr_setup_bind_cqs();
- cntr_setup_cntrs();
- cntr_setup_bind_cntrs();
- cntr_setup_enable_ep();
- cntr_setup_mr();
-}
-
-static inline void cntr_teardown_mr(void)
-{
- fi_close(&loc_mr->fid);
- fi_close(&rem_mr->fid);
-
- free(target);
- free(source);
-}
-
-static inline void cntr_teardown_eps(void)
-{
- int i, ret;
-
- for (i = 0; i < NUM_EPS; i++) {
- ret = fi_close(&ep[i]->fid);
- cr_assert(!ret, "failure in closing ep.");
-
- }
-}
-
-static inline void cntr_teardown_cqs(void)
-{
- int ret;
- ret = fi_close(&send_cq->fid);
- cr_assert(!ret, "failure in closing send cq.");
-
- ret = fi_close(&recv_cq->fid);
- cr_assert(!ret, "failure in closing recv cq.");
-
-}
-
-static inline void cntr_teardown_cntrs(void)
-{
- int ret;
-
- ret = fi_close(&write_cntr->fid);
- cr_assert(!ret, "failure in closing write_cntr.");
-
- ret = fi_close(&read_cntr->fid);
- cr_assert(!ret, "failure in closing read_cntr.");
-
- ret = fi_close(&rcv_cntr->fid);
- cr_assert(!ret, "failure in closing read_cntr.");
-
-}
-
-static inline void cntr_teardown_fini(void)
-{
- int i, ret;
-
- ret = fi_close(&av->fid);
- cr_assert(!ret, "failure in closing av.");
-
- ret = fi_close(&dom->fid);
- cr_assert(!ret, "failure in closing domain.");
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-
- for (i = 0; i < NUM_EPS; i++) {
- free(ep_name[i]);
- }
-}
-
-static void cntr_teardown(void)
-{
- cntr_teardown_mr();
- cntr_teardown_eps();
- cntr_teardown_cqs();
- cntr_teardown_cntrs();
- cntr_teardown_fini();
-}
-
-static void init_data(char *buf, int len, char seed)
-{
- int i;
-
- for (i = 0; i < len; i++)
- buf[i] = seed++;
-}
-
-static int check_data(char *buf1, char *buf2, int len)
-{
- int i;
-
- for (i = 0; i < len; i++) {
- if (buf1[i] != buf2[i]) {
- printf("data mismatch, elem: %d, exp: %x, act: %x\n",
- i, buf1[i], buf2[i]);
- return 0;
- }
- }
-
- return 1;
-}
-
-static void xfer_for_each_size(void (*xfer)(int len), int slen, int elen)
-{
- int i;
-
- for (i = slen; i <= elen; i *= 2)
- xfer(i);
-}
-
-/*******************************************************************************
- * Test RMA functions
- ******************************************************************************/
-
-TestSuite(cntr, .init = cntr_setup, .fini = cntr_teardown,
- .disabled = false);
-
-static void do_write(int len)
-{
- uint64_t old_w_cnt, new_w_cnt;
- uint64_t old_r_cnt, new_r_cnt;
- ssize_t sz;
-
- init_data(source, len, 0xab);
- init_data(target, len, 0);
-
-
- old_w_cnt = fi_cntr_read(write_cntr);
- old_r_cnt = fi_cntr_read(read_cntr);
-
- sz = fi_write(ep[0], source, len,
- loc_mr, gni_addr[1], (uint64_t)target, mr_key,
- target);
- cr_assert_eq(sz, 0);
-
- do {
- new_w_cnt = fi_cntr_read(write_cntr);
- if (new_w_cnt == (old_w_cnt + 1))
- break;
- pthread_yield();
- } while (1);
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-
- new_r_cnt = fi_cntr_read(read_cntr);
-
- /*
- * no fi_read called so old and new read cnts should be equal
- */
- cr_assert(new_r_cnt == old_r_cnt);
-}
-
-Test(cntr, write)
-{
- xfer_for_each_size(do_write, 8, BUF_SZ);
-}
-
-static void do_write_wait(int len)
-{
- uint64_t old_w_cnt, new_w_cnt;
- uint64_t old_r_cnt, new_r_cnt;
- ssize_t sz;
- const int iters = 100;
- int i;
-
- init_data(source, len, 0xab);
- init_data(target, len, 0);
-
- old_w_cnt = fi_cntr_read(write_cntr);
- old_r_cnt = fi_cntr_read(read_cntr);
-
- for (i = 0; i < iters; i++) {
- sz = fi_write(ep[0], source, len,
- loc_mr, gni_addr[1], (uint64_t)target, mr_key,
- target);
- cr_assert_eq(sz, 0);
- }
-
- fi_cntr_wait(write_cntr, old_w_cnt+iters, -1);
- new_w_cnt = fi_cntr_read(write_cntr);
- cr_assert(old_w_cnt + iters == new_w_cnt);
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-
- new_r_cnt = fi_cntr_read(read_cntr);
-
- /*
- * no fi_read called so old and new read cnts should be equal
- */
- cr_assert(new_r_cnt == old_r_cnt);
-}
-
-Test(cntr, write_wait)
-{
- xfer_for_each_size(do_write_wait, 8, BUF_SZ);
-}
-
-static void do_read(int len)
-{
- ssize_t sz;
- uint64_t old_w_cnt, new_w_cnt;
- uint64_t old_r_cnt, new_r_cnt;
-
-#define READ_CTX 0x4e3dda1aULL
- init_data(source, len, 0);
- init_data(target, len, 0xad);
-
- old_w_cnt = fi_cntr_read(write_cntr);
- old_r_cnt = fi_cntr_read(read_cntr);
-
- sz = fi_read(ep[0], source, len,
- loc_mr, gni_addr[1], (uint64_t)target, mr_key,
- (void *)READ_CTX);
- cr_assert_eq(sz, 0);
-
- do {
- new_r_cnt = fi_cntr_read(read_cntr);
- if (new_r_cnt == (old_r_cnt + 1))
- break;
- pthread_yield();
- } while (1);
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-
- new_w_cnt = fi_cntr_read(write_cntr);
-
- /*
- * no fi_read called so old and new read cnts should be equal
- */
- cr_assert(new_w_cnt == old_w_cnt);
-}
-
-static void do_read_wait(int len)
-{
- int i, iters = 100;
- ssize_t sz;
- uint64_t old_w_cnt, new_w_cnt;
- uint64_t old_r_cnt;
-
-#define READ_CTX 0x4e3dda1aULL
- init_data(source, len, 0);
- init_data(target, len, 0xad);
-
- old_w_cnt = fi_cntr_read(write_cntr);
- old_r_cnt = fi_cntr_read(read_cntr);
-
- for (i = 0; i < iters; i++) {
- sz = fi_read(ep[0], source, len,
- loc_mr, gni_addr[1], (uint64_t)target,
- mr_key, (void *)READ_CTX);
- cr_assert_eq(sz, 0);
- }
-
- fi_cntr_wait(read_cntr, old_r_cnt + iters, -1);
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-
- new_w_cnt = fi_cntr_read(write_cntr);
-
- /*
- * no fi_read called so old and new read cnts should be equal
- */
- cr_assert(new_w_cnt == old_w_cnt);
-}
-
-Test(cntr, read)
-{
- xfer_for_each_size(do_read, 8, BUF_SZ);
-}
-
-Test(cntr, read_wait)
-{
- xfer_for_each_size(do_read_wait, 8, BUF_SZ);
-}
-
-
-Test(cntr, send_recv)
-{
- int ret, i, got_r = 0;
- struct fi_context r_context, s_context;
- struct fi_cq_entry cqe;
- uint64_t old_s_cnt, new_s_cnt;
- uint64_t old_r_cnt, new_r_cnt;
- char s_buffer[128], r_buffer[128];
-
- old_s_cnt = fi_cntr_read(write_cntr);
- old_r_cnt = fi_cntr_read(rcv_cntr);
-
- for (i = 0; i < 16; i++) {
- sprintf(s_buffer, "Hello there iter=%d", i);
- memset(r_buffer, 0, 128);
- ret = fi_recv(ep[1],
- r_buffer,
- sizeof(r_buffer),
- NULL,
- gni_addr[0],
- &r_context);
- cr_assert_eq(ret, FI_SUCCESS, "fi_recv");
- ret = fi_send(ep[0],
- s_buffer,
- strlen(s_buffer),
- NULL,
- gni_addr[1],
- &s_context);
- cr_assert_eq(ret, FI_SUCCESS, "fi_send");
-
- while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN)
- pthread_yield();
-
- cr_assert((cqe.op_context == &r_context) ||
- (cqe.op_context == &s_context), "fi_cq_read");
- got_r = (cqe.op_context == &r_context) ? 1 : 0;
-
- if (got_r) {
- new_r_cnt = fi_cntr_read(rcv_cntr);
- old_r_cnt++;
- cr_assert(new_r_cnt == old_r_cnt);
- } else {
- new_s_cnt = fi_cntr_read(write_cntr);
- old_s_cnt++;
- cr_assert(new_s_cnt == old_s_cnt);
- }
-
- while ((ret = fi_cq_read(recv_cq, &cqe, 1)) == -FI_EAGAIN)
- pthread_yield();
- if (got_r)
- cr_assert((cqe.op_context == &s_context), "fi_cq_read");
- else
- cr_assert((cqe.op_context == &r_context), "fi_cq_read");
-
- if (got_r) {
- new_s_cnt = fi_cntr_read(write_cntr);
- old_s_cnt++;
- cr_assert(new_s_cnt == old_s_cnt);
- } else {
- new_r_cnt = fi_cntr_read(rcv_cntr);
- old_r_cnt++;
- cr_assert(new_r_cnt == old_r_cnt);
- }
-
- cr_assert(strcmp(s_buffer, r_buffer) == 0, "check message");
-
- got_r = 0;
- }
-
-}
-
-/*
- * Multithreaded tests
- */
-
-struct tinfo {
- int msg_size;
- int iters;
-};
-
-#define get_mark(i) ((char) (((i)%255)+0x31))
-
-static atomic_t cntr_test_next_tid;
-static __thread uint32_t cntr_test_tid = ~(uint32_t) 0;
-#define cntr_test_get_tid() \
- ((cntr_test_tid == ~(uint32_t) 0) ? \
- atomic_inc(&cntr_test_next_tid) : \
- cntr_test_tid)
-
-
-static struct fid_cntr *ep_write_cntrs[NUM_EPS];
-static struct fid_cntr *ep_read_cntrs[NUM_EPS];
-
-static void cntr_setup_mt(void)
-{
- int i, ret;
-
- cntr_setup_eps();
- cntr_setup_av();
- cntr_setup_cqs();
- cntr_setup_bind_cqs();
-
- for (i = 0; i < NUM_EPS; i++) {
- ret = fi_cntr_open(dom, &cntr_attr, &ep_write_cntrs[i], 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[i], &ep_write_cntrs[i]->fid,
- FI_WRITE | FI_SEND);
- cr_assert(!ret, "fi_ep_bind cntr");
-
- ret = fi_cntr_open(dom, &cntr_attr, &ep_read_cntrs[i], 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[i], &ep_read_cntrs[i]->fid, FI_READ);
- cr_assert(!ret, "fi_ep_bind cntr");
- }
-
- cntr_setup_enable_ep();
- cntr_setup_mr();
-
- atomic_initialize(&cntr_test_next_tid, 0);
-}
-
-static void cntr_teardown_mt(void)
-{
- int i, ret;
-
- cntr_teardown_mr();
- cntr_teardown_eps();
- cntr_teardown_cqs();
-
- for (i = 0; i < NUM_EPS; i++) {
- ret = fi_close(&ep_write_cntrs[i]->fid);
- cr_assert(!ret, "failure in closing write_cntr.");
-
- ret = fi_close(&ep_read_cntrs[i]->fid);
- cr_assert(!ret, "failure in closing read_cntr.");
- }
-
- cntr_teardown_fini();
-}
-
-TestSuite(cntr_mt, .init = cntr_setup_mt, .fini = cntr_teardown_mt,
- .disabled = false);
-
-static void *do_thread_read_wait(void *data)
-{
- int i, tid, ret;
- ssize_t sz;
- struct tinfo *info = (struct tinfo *) data;
- int msg_size = info->msg_size;
- int iters = info->iters;
-
- tid = cntr_test_get_tid();
-
- dbg_printf("%d: reading\n", tid);
- for (i = 0; i < iters; i++) {
- sz = fi_read(ep[tid], &source[tid*msg_size], msg_size, loc_mr,
- gni_addr[0], (uint64_t)&target[tid*msg_size],
- mr_key, (void *)(READ_CTX+i));
- cr_assert_eq(sz, 0);
- }
-
- dbg_printf("%d: waiting\n", tid);
- ret = fi_cntr_wait(ep_read_cntrs[tid], iters, -1);
- cr_assert(ret == FI_SUCCESS);
-
- dbg_printf("%d: done\n", tid);
- return NULL;
-}
-
-Test(cntr_mt, read_wait)
-{
- int i, j;
- pthread_t threads[NUM_EPS];
- const int msg_size = 128;
- struct tinfo info = { msg_size, 500 /* iters */};
-
- cr_assert(NUM_EPS*msg_size <= BUF_SZ);
-
- memset(source, 0, NUM_EPS*msg_size);
- for (i = 0; i < NUM_EPS; i++) {
- memset(&target[i*msg_size], get_mark(i), msg_size);
- }
-
- dbg_printf("creating threads\n");
- for (i = 1; i < NUM_EPS; i++) {
- pthread_create(&threads[i], NULL, do_thread_read_wait, &info);
- }
-
- dbg_printf("joining\n");
-
- for (i = 1; i < NUM_EPS; i++) {
- pthread_join(threads[i], NULL);
- }
-
- /* Must wait until all threads are done, since we don't know
- * which thread got which id */
- for (i = 1; i < NUM_EPS; i++) {
- for (j = 0; j < msg_size; j++) {
- cr_assert(source[i*msg_size+j] == get_mark(i));
- }
- }
-
- dbg_printf("done\n");
-
-}
-
-static void *do_thread_write_wait(void *data)
-{
- int i, tid, ret;
- ssize_t sz;
- struct tinfo *info = (struct tinfo *) data;
- int msg_size = info->msg_size;
- int iters = info->iters;
-
- tid = cntr_test_get_tid();
-
- dbg_printf("%d: writing\n", tid);
- for (i = 0; i < iters; i++) {
- sz = fi_write(ep[tid], &source[tid*msg_size], msg_size, loc_mr,
- gni_addr[0], (uint64_t)&target[tid*msg_size],
- mr_key, (void *)(READ_CTX+i));
- cr_assert_eq(sz, 0);
- }
-
- dbg_printf("%d: waiting\n", tid);
- ret = fi_cntr_wait(ep_write_cntrs[tid], iters, -1);
- cr_assert(ret == FI_SUCCESS);
-
- dbg_printf("%d: done\n", tid);
- return NULL;
-}
-
-Test(cntr_mt, write_wait)
-{
- int i, j;
- pthread_t threads[NUM_EPS];
- const int msg_size = 128;
- struct tinfo info = { msg_size, 500 /* iters */};
-
- cr_assert(NUM_EPS*msg_size <= BUF_SZ);
-
- memset(target, 0, NUM_EPS*msg_size);
- for (i = 0; i < NUM_EPS; i++) {
- memset(&source[i*msg_size], get_mark(i), msg_size);
- }
-
- dbg_printf("creating threads\n");
- for (i = 1; i < NUM_EPS; i++) {
- pthread_create(&threads[i], NULL, do_thread_write_wait, &info);
- }
-
- dbg_printf("joining\n");
-
- for (i = 1; i < NUM_EPS; i++) {
- pthread_join(threads[i], NULL);
- }
-
- /* Must wait until all threads are done, since we don't know
- * which thread got which id */
- for (i = 1; i < NUM_EPS; i++) {
- for (j = 0; j < msg_size; j++) {
- cr_assert(target[i*msg_size+j] == get_mark(i));
- }
- }
-
- dbg_printf("done\n");
-}
diff --git a/prov/gni/test/common.h b/prov/gni/test/common.h
deleted file mode 100644
index f93d614..0000000
--- a/prov/gni/test/common.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef PROV_GNI_TEST_COMMON_H_
-#define PROV_GNI_TEST_COMMON_H_
-
-#include <time.h>
-#include <stdint.h>
-#include <sys/time.h>
-
-void calculate_time_difference(struct timeval *start, struct timeval *end,
- int *secs_out, int *usec_out);
-
-#endif /* PROV_GNI_TEST_COMMON_H_ */
diff --git a/prov/gni/test/cq.c b/prov/gni/test/cq.c
deleted file mode 100644
index 039a937..0000000
--- a/prov/gni/test/cq.c
+++ /dev/null
@@ -1,662 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include "gnix_cq.h"
-#include "gnix.h"
-
-#include <criterion/criterion.h>
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom;
-static struct fid_cq *rcq;
-static struct fi_info *hints;
-static struct fi_info *fi;
-static struct fi_cq_attr cq_attr;
-static struct gnix_fid_cq *cq_priv;
-
-static struct gnix_fid_wait *wait_priv;
-static struct fid_wait *wait_set;
-static struct fi_wait_attr wait_attr;
-
-void setup(void)
-{
- int ret = 0;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = 4;
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- ret = fi_domain(fab, fi, &dom, NULL);
- cr_assert(!ret, "fi_domain");
-
- cq_attr.wait_obj = FI_WAIT_NONE;
-}
-
-void teardown(void)
-{
- int ret = 0;
-
- ret = fi_close(&dom->fid);
- cr_assert(!ret, "failure in closing domain.");
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-}
-
-void cq_create(enum fi_cq_format format, enum fi_wait_obj wait_obj,
- size_t size)
-{
- int ret = 0;
-
- cq_attr.format = format;
- cq_attr.size = size;
- cq_attr.wait_obj = wait_obj;
-
- ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
- cr_assert(!ret, "fi_cq_open");
-
- cq_priv = container_of(rcq, struct gnix_fid_cq, cq_fid);
-
- if (cq_priv->wait) {
- wait_priv = container_of(cq_priv->wait, struct gnix_fid_wait,
- wait);
- }
-}
-
-void cq_setup(void)
-{
- setup();
- cq_create(FI_CQ_FORMAT_UNSPEC, FI_WAIT_NONE, 0);
-}
-
-void cq_msg_setup(void)
-{
- setup();
- cq_create(FI_CQ_FORMAT_MSG, FI_WAIT_NONE, 8);
-}
-
-void cq_data_setup(void)
-{
- setup();
- cq_create(FI_CQ_FORMAT_DATA, FI_WAIT_NONE, 8);
-}
-
-void cq_tagged_setup(void)
-{
- setup();
- cq_create(FI_CQ_FORMAT_TAGGED, FI_WAIT_NONE, 8);
-}
-
-void cq_wait_none_setup(void)
-{
- setup();
- cq_create(FI_CQ_FORMAT_MSG, FI_WAIT_NONE, 8);
-}
-
-void cq_wait_fd_setup(void)
-{
- setup();
- cq_create(FI_CQ_FORMAT_MSG, FI_WAIT_FD, 8);
-}
-
-void cq_wait_unspec_setup(void)
-{
- setup();
- cq_create(FI_CQ_FORMAT_MSG, FI_WAIT_UNSPEC, 8);
-}
-
-void cq_wait_mutex_cond_setup(void)
-{
- setup();
- cq_create(FI_CQ_FORMAT_MSG, FI_WAIT_MUTEX_COND, 8);
-}
-
-void cq_teardown(void)
-{
- cr_assert(!fi_close(&rcq->fid), "failure in closing cq.");
- teardown();
-}
-
-/*******************************************************************************
- * Creation Tests:
- *
- * Create the CQ with various parameters and make sure the fields are being
- * initialized correctly.
- ******************************************************************************/
-
-TestSuite(creation, .init = setup, .fini = cq_teardown);
-
-Test(creation, format_unspec)
-{
- int ret = 0;
-
- cq_attr.format = FI_CQ_FORMAT_UNSPEC;
-
- ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
- cr_assert(!ret, "fi_cq_open");
-
- cq_priv = container_of(rcq, struct gnix_fid_cq, cq_fid);
- cr_assert(cq_priv->entry_size == sizeof(struct fi_cq_entry));
-}
-
-Test(creation, format_context)
-{
- int ret = 0;
-
- cq_attr.format = FI_CQ_FORMAT_CONTEXT;
-
- ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
- cr_assert(!ret, "fi_cq_open");
-
- cq_priv = container_of(rcq, struct gnix_fid_cq, cq_fid);
- cr_assert(cq_priv->entry_size == sizeof(struct fi_cq_entry));
-}
-
-Test(creation, format_msg)
-{
- int ret = 0;
-
- cq_attr.format = FI_CQ_FORMAT_MSG;
-
- ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
- cr_assert(!ret, "fi_cq_open");
-
- cq_priv = container_of(rcq, struct gnix_fid_cq, cq_fid);
- cr_assert(cq_priv->entry_size == sizeof(struct fi_cq_msg_entry));
-}
-
-Test(creation, format_data)
-{
- int ret = 0;
-
- cq_attr.format = FI_CQ_FORMAT_DATA;
-
- ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
- cr_assert(!ret, "fi_cq_open");
-
- cq_priv = container_of(rcq, struct gnix_fid_cq, cq_fid);
- cr_assert(cq_priv->entry_size == sizeof(struct fi_cq_data_entry));
-}
-
-Test(creation, format_tagged)
-{
- int ret = 0;
-
- cq_attr.format = FI_CQ_FORMAT_TAGGED;
-
- ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
- cr_assert(!ret, "fi_cq_open");
-
- cq_priv = container_of(rcq, struct gnix_fid_cq, cq_fid);
- cr_assert(cq_priv->entry_size == sizeof(struct fi_cq_tagged_entry));
-}
-
-TestSuite(insertion, .init = cq_setup, .fini = cq_teardown);
-
-Test(insertion, single)
-{
- int ret = 0;
- char input_ctx = 'a';
- struct fi_cq_entry entry;
-
- cr_assert(!cq_priv->events->item_list.head);
-
- _gnix_cq_add_event(cq_priv, &input_ctx, 0, 0, 0, 0, 0, 0);
-
- cr_assert(cq_priv->events->item_list.head);
- cr_assert_eq(cq_priv->events->item_list.head,
- cq_priv->events->item_list.tail);
-
- ret = fi_cq_read(rcq, &entry, 1);
- cr_assert(ret == 1);
- cr_assert(!cq_priv->events->item_list.head);
-
- cr_assert_eq(*(char *) entry.op_context, input_ctx,
- "Expected same op_context as inserted.");
-}
-
-Test(insertion, limit)
-{
- int ret = 0;
- char input_ctx = 'a';
- struct fi_cq_entry entry;
- const size_t cq_size = cq_priv->attr.size;
-
- for (size_t i = 0; i < cq_size; i++)
- _gnix_cq_add_event(cq_priv, &input_ctx, 0, 0, 0, 0, 0, 0);
-
- cr_assert(cq_priv->events->item_list.head);
- cr_assert(!cq_priv->events->free_list.head);
-
- _gnix_cq_add_event(cq_priv, &input_ctx, 0, 0, 0, 0, 0, 0);
-
- for (size_t i = 0; i < cq_size + 1; i++) {
- ret = fi_cq_read(rcq, &entry, 1);
- cr_assert_eq(ret, 1);
- }
-
- cr_assert(!cq_priv->events->item_list.head);
- cr_assert(cq_priv->events->free_list.head);
-}
-
-TestSuite(reading, .init = cq_setup, .fini = cq_teardown);
-
-Test(reading, empty)
-{
- int ret = 0;
- struct fi_cq_entry entry;
-
- ret = fi_cq_read(rcq, &entry, 1);
- cr_assert_eq(ret, -FI_EAGAIN);
-}
-
-Test(reading, error)
-{
- int ret = 0;
- struct fi_cq_entry entry;
- struct fi_cq_err_entry err_entry;
-
- char input_ctx = 'a';
- uint64_t flags = 0xb;
- size_t len = sizeof(input_ctx);
- void *buf = &input_ctx;
- uint64_t data = 20;
- uint64_t tag = 40;
- size_t olen = 20;
- int err = 50;
- int prov_errno = 80;
-
- /*
- * By default CQ start out with no error entries and no entries
- * in the error entry free list.
- */
- cr_assert(!cq_priv->errors->item_list.head);
- cr_assert(!cq_priv->errors->free_list.head);
-
- _gnix_cq_add_error(cq_priv, &input_ctx, flags, len, buf, data, tag,
- olen, err, prov_errno, 0);
-
- cr_assert(cq_priv->errors->item_list.head);
-
- ret = fi_cq_read(rcq, &entry, 1);
- cr_assert_eq(ret, -FI_EAVAIL);
-
- cr_assert(!cq_priv->events->item_list.head);
- cr_assert(cq_priv->errors->item_list.head);
-
- ret = fi_cq_readerr(rcq, &err_entry, 0);
- cr_assert_eq(ret, 1);
-
- /*
- * Item should have been removed from error queue and placed on free
- * queue.
- */
- cr_assert(!cq_priv->errors->item_list.head);
- cr_assert(cq_priv->errors->free_list.head);
-
- /*
- * Compare structural items...
- */
- cr_assert_eq(*(char *) err_entry.op_context, input_ctx);
- cr_assert_eq(err_entry.flags, flags);
- cr_assert_eq(err_entry.len, len);
- cr_assert_eq(err_entry.buf, buf);
- cr_assert_eq(err_entry.data, data);
- cr_assert_eq(err_entry.tag, tag);
- cr_assert_eq(err_entry.olen, olen);
- cr_assert_eq(err_entry.err, err);
- cr_assert_eq(err_entry.prov_errno, prov_errno);
- cr_assert_eq(err_entry.err_data, 0);
-}
-
-#define ENTRY_CNT 5
-Test(reading, issue192)
-{
- int ret = 0;
- char input_ctx = 'a';
- struct fi_cq_entry entries[ENTRY_CNT];
-
- _gnix_cq_add_event(cq_priv, &input_ctx, 0, 0, 0, 0, 0, 0);
-
- ret = fi_cq_read(rcq, &entries, ENTRY_CNT);
- cr_assert_eq(ret, 1);
-
- ret = fi_cq_read(rcq, &entries, ENTRY_CNT);
- cr_assert_eq(ret, -FI_EAGAIN);
-}
-
-TestSuite(cq_msg, .init = cq_msg_setup, .fini = cq_teardown);
-
-Test(cq_msg, single)
-{
- int ret = 0;
- char input_ctx = 'a';
- struct fi_cq_msg_entry entry;
-
- cr_assert(!cq_priv->events->item_list.head);
-
- _gnix_cq_add_event(cq_priv, &input_ctx, 2, 4, 0, 0, 0, 0);
-
- cr_assert(cq_priv->events->item_list.head);
-
- ret = fi_cq_read(rcq, &entry, 1);
- cr_assert_eq(ret, 1);
-
- cr_assert_eq(entry.flags, 2);
- cr_assert_eq(*(char *) entry.op_context, input_ctx);
- cr_assert_eq(entry.len, 4);
-
- cr_assert(!cq_priv->events->item_list.head);
-}
-
-/*
- * Create up to the size events to fill it up. Check that all the properties
- * are correct, then add one more that is different. Read size items and then
- * add an error and try reading. Ensure that we get back -FI_EAVAIL. Then read
- * the last item and make sure it's the same values put in originally.
- */
-Test(cq_msg, fill)
-{
- struct fi_cq_msg_entry entry;
- struct fi_cq_err_entry err_entry;
- int ret = 0;
- char input_ctx = 'a';
- uint64_t flags = 2;
- size_t len = 4;
- const size_t cq_size = cq_priv->attr.size;
-
- cr_assert(!cq_priv->events->item_list.head);
- cr_assert(cq_priv->events->free_list.head);
-
- for (size_t i = 0; i < cq_size; i++)
- _gnix_cq_add_event(cq_priv, &input_ctx, flags, len, 0, 0, 0, 0);
-
- cr_assert(cq_priv->events->item_list.head);
- cr_assert(!cq_priv->events->free_list.head);
-
- _gnix_cq_add_event(cq_priv, &input_ctx, flags * 2, len * 2, 0, 0, 0, 0);
-
- for (size_t i = 0; i < cq_size; i++) {
- ret = fi_cq_read(rcq, &entry, 1);
- cr_assert_eq(ret, 1);
-
- cr_assert_eq(*(char *) entry.op_context, input_ctx);
- cr_assert_eq(entry.len, len);
- cr_assert_eq(entry.flags, flags);
- }
-
- /*
- * If we insert an error it should return -FI_EAVAIL despite having
- * something to read.
- */
- _gnix_cq_add_error(cq_priv, &input_ctx, flags, len, 0, 0, 0, 0, 0, 0,
- 0);
- cr_assert(cq_priv->errors->item_list.head);
-
- ret = fi_cq_read(rcq, &entry, 1);
- cr_assert_eq(ret, -FI_EAVAIL);
-
- ret = fi_cq_readerr(rcq, &err_entry, 0);
- cr_assert_eq(ret, 1);
-
- /*
- * Creating an error allocs an error but it is then placed in the free
- * list after reading.
- */
- cr_assert(cq_priv->errors->free_list.head);
- cr_assert(!cq_priv->errors->item_list.head);
-
- ret = fi_cq_read(rcq, &entry, 1);
- cr_assert_eq(ret, 1);
-
- cr_assert(cq_priv->events->free_list.head);
- cr_assert(!cq_priv->events->item_list.head);
-
- cr_assert_eq(*(char *) entry.op_context, input_ctx);
- cr_assert_eq(entry.len, (len * 2));
- cr_assert_eq(entry.flags, (flags * 2));
-}
-
-Test(cq_msg, multi_read)
-{
- int ret = 0;
- size_t count = 3;
- struct fi_cq_msg_entry entry[count];
-
- cr_assert(cq_priv->events->free_list.head);
- cr_assert(!cq_priv->events->item_list.head);
-
- for (size_t i = 0; i < count; i++)
- _gnix_cq_add_event(cq_priv, 0, (uint64_t) i, 0, 0, 0, 0, 0);
-
- cr_assert(cq_priv->events->item_list.head);
-
- ret = fi_cq_read(rcq, &entry, count);
- cr_assert_eq(ret, count);
-
- for (size_t j = 0; j < count; j++)
- cr_assert_eq(entry[j].flags, (uint64_t) j);
-}
-
-Test(cq_msg, multi_sread, .init = cq_wait_unspec_setup)
-{
- int ret = 0;
- size_t count = 3;
- struct fi_cq_msg_entry entry[count];
-
- cr_assert(cq_priv->events->free_list.head);
- cr_assert(!cq_priv->events->item_list.head);
-
- ret = fi_cq_sread(rcq, &entry, count, NULL, 100);
- cr_assert_eq(ret, -FI_EAGAIN);
-
- for (size_t i = 0; i < count; i++)
- _gnix_cq_add_event(cq_priv, 0, (uint64_t) i, 0, 0, 0, 0, 0);
-
- cr_assert(cq_priv->events->item_list.head);
-
- ret = fi_cq_sread(rcq, &entry, count, NULL, -1);
- cr_assert_eq(ret, count);
-
- for (size_t j = 0; j < count; j++)
- cr_assert_eq(entry[j].flags, (uint64_t) j);
-}
-
-TestSuite(cq_wait_obj, .fini = cq_teardown);
-TestSuite(cq_wait_control, .fini = cq_teardown, .disabled = true);
-TestSuite(cq_wait_ops, .fini = cq_teardown);
-
-Test(cq_wait_obj, none, .init = cq_wait_none_setup)
-{
- cr_expect(!wait_priv, "wait_priv is not null.");
-}
-
-Test(cq_wait_obj, unspec, .init = cq_wait_unspec_setup)
-{
- cr_expect_eq(wait_priv->type, FI_WAIT_FD);
- cr_expect_eq(wait_priv->type, cq_priv->attr.wait_obj);
- cr_expect_eq(wait_priv->type, cq_attr.wait_obj);
- cr_expect_eq(&wait_priv->fabric->fab_fid, fab);
- cr_expect_eq(wait_priv->cond_type, FI_CQ_COND_NONE);
-}
-
-Test(cq_wait_obj, fd, .init = cq_wait_fd_setup)
-{
- cr_expect_eq(wait_priv->type, FI_WAIT_FD);
- cr_expect_eq(wait_priv->type, cq_priv->attr.wait_obj);
- cr_expect_eq(wait_priv->type, cq_attr.wait_obj);
- cr_expect_eq(&wait_priv->fabric->fab_fid, fab);
- cr_expect_eq(wait_priv->cond_type, FI_CQ_COND_NONE);
-}
-
-Test(cq_wait_obj, mutex_cond, .init = cq_wait_mutex_cond_setup)
-{
- cr_expect_eq(wait_priv->type, FI_WAIT_MUTEX_COND);
- cr_expect_eq(wait_priv->type, cq_priv->attr.wait_obj);
- cr_expect_eq(wait_priv->type, cq_attr.wait_obj);
- cr_expect_eq(&wait_priv->fabric->fab_fid, fab);
- cr_expect_eq(wait_priv->cond_type, FI_CQ_COND_NONE);
-}
-
-Test(cq_wait_control, none, .init = cq_wait_none_setup)
-{
- int ret;
- int fd;
-
- ret = fi_control(&cq_priv->cq_fid.fid, FI_GETWAIT, &fd);
- cr_expect_eq(-FI_ENOSYS, ret, "fi_control exists for none.");
-}
-
-Test(cq_wait_control, unspec, .init = cq_wait_unspec_setup)
-{
- int ret;
- int fd;
-
- ret = fi_control(&cq_priv->cq_fid.fid, FI_GETWAIT, &fd);
- cr_expect_eq(FI_SUCCESS, ret, "fi_control failed.");
-
- cr_expect_eq(wait_priv->fd[WAIT_READ], fd);
-}
-
-Test(cq_wait_control, fd, .init = cq_wait_fd_setup)
-{
- int ret;
- int fd;
-
- ret = fi_control(&cq_priv->cq_fid.fid, FI_GETWAIT, &fd);
- cr_expect_eq(FI_SUCCESS, ret, "fi_control failed.");
-
- cr_expect_eq(wait_priv->fd[WAIT_READ], fd);
-}
-
-Test(cq_wait_control, mutex_cond, .init = cq_wait_mutex_cond_setup)
-{
- int ret;
- struct fi_mutex_cond mutex_cond;
-
- ret = fi_control(&cq_priv->cq_fid.fid, FI_GETWAIT, &mutex_cond);
- cr_expect_eq(FI_SUCCESS, ret, "fi_control failed.");
-
- ret = memcmp(&wait_priv->mutex, mutex_cond.mutex,
- sizeof(*mutex_cond.mutex));
- cr_expect_eq(0, ret, "mutex compare failed.");
-
- ret = memcmp(&wait_priv->cond, mutex_cond.cond,
- sizeof(*mutex_cond.cond));
- cr_expect_eq(0, ret, "cond compare failed.");
-}
-
-Test(cq_wait_ops, none, .init = cq_wait_none_setup)
-{
- cr_expect_eq(cq_priv->cq_fid.ops->signal, fi_no_cq_signal,
- "signal implementation available.");
- cr_expect_eq(cq_priv->cq_fid.ops->sread, fi_no_cq_sread,
- "sread implementation available.");
- cr_expect_eq(cq_priv->cq_fid.ops->sreadfrom, fi_no_cq_sreadfrom,
- "sreadfrom implementation available.");
- cr_expect_eq(cq_priv->cq_fid.fid.ops->control, fi_no_control,
- "control implementation available.");
-}
-
-Test(cq_wait_ops, fd, .init = cq_wait_fd_setup)
-{
- cr_expect_neq(cq_priv->cq_fid.ops->signal, fi_no_cq_signal,
- "signal implementation not available.");
- cr_expect_neq(cq_priv->cq_fid.fid.ops->control, fi_no_control,
- "control implementation not available.");
-}
-
-Test(cq_wait_set, fd, .init = setup, .disabled = true)
-{
- int ret;
- int fd;
-
- wait_attr.wait_obj = FI_WAIT_FD;
-
- ret = fi_wait_open(fab, &wait_attr, &wait_set);
- cr_expect_eq(FI_SUCCESS, ret, "fi_wait_open failed.");
-
- wait_priv = container_of(wait_set, struct gnix_fid_wait, wait);
-
- cq_attr.format = FI_CQ_FORMAT_MSG;
- cq_attr.size = 8;
- cq_attr.wait_obj = FI_WAIT_SET;
- cq_attr.wait_set = wait_set;
-
- ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
- cr_expect_eq(FI_SUCCESS, ret, "fi_cq_open failed.");
-
- cq_priv = container_of(rcq, struct gnix_fid_cq, cq_fid);
-
- ret = fi_control(&cq_priv->cq_fid.fid, FI_GETWAIT, &fd);
- cr_expect_eq(FI_SUCCESS, ret, "fi_control failed.");
-
- cr_expect_eq(wait_priv->fd[WAIT_READ], fd);
-
- ret = fi_close(&rcq->fid);
- cr_expect_eq(FI_SUCCESS, ret, "failure in closing cq.");
-
- ret = fi_close(&wait_set->fid);
- cr_expect_eq(FI_SUCCESS, ret, "failure in closing waitset.");
-
- teardown();
-}
diff --git a/prov/gni/test/datagram.c b/prov/gni/test/datagram.c
deleted file mode 100644
index d0a28cd..0000000
--- a/prov/gni/test/datagram.c
+++ /dev/null
@@ -1,453 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <pthread.h>
-#include <unistd.h>
-#include <limits.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include "gnix_datagram.h"
-#include "gnix_cm_nic.h"
-
-#include <criterion/criterion.h>
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom;
-static struct fid_ep *ep;
-static struct fi_info *hints;
-static struct fi_info *fi;
-static struct gnix_fid_ep *ep_priv;
-const char my_cdm_id[] = "3000";
-
-void dg_setup(void)
-{
- int ret = 0;
- char my_hostname[HOST_NAME_MAX];
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = 4;
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = gethostname(my_hostname, sizeof(my_hostname));
- cr_assert(!ret, "gethostname");
-
- ret = fi_getinfo(FI_VERSION(1, 0), my_hostname, my_cdm_id, FI_SOURCE,
- hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- ret = fi_domain(fab, fi, &dom, NULL);
- cr_assert(!ret, "fi_domain");
-
- ret = fi_endpoint(dom, fi, &ep, NULL);
- cr_assert(!ret, "fi_endpoint");
-}
-
-void dg_setup_prog_manual(void)
-{
- int ret = 0;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = 4;
- hints->domain_attr->control_progress = FI_PROGRESS_MANUAL;
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- ret = fi_domain(fab, fi, &dom, NULL);
- cr_assert(!ret, "fi_domain");
-
- ret = fi_endpoint(dom, fi, &ep, NULL);
- cr_assert(!ret, "fi_endpoint");
-}
-
-void dg_teardown(void)
-{
- int ret = 0;
-
- ret = fi_close(&ep->fid);
- cr_assert(!ret, "failure in closing ep.");
- ret = fi_close(&dom->fid);
- cr_assert(!ret, "failure in closing domain.");
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-}
-
-/*******************************************************************************
- * Allocation Tests:
- *
- * try different datagram allocation/free patterns and see if something
- * explodes.
- ******************************************************************************/
-
-TestSuite(dg_allocation, .init = dg_setup, .fini = dg_teardown);
-
-Test(dg_allocation, dgram_verify_cdm_id)
-{
- uint32_t correct = atol(my_cdm_id);
- struct gnix_cm_nic *cm_nic;
-
- ep_priv = container_of(ep, struct gnix_fid_ep, ep_fid);
- cm_nic = ep_priv->cm_nic;
- cr_assert((cm_nic != NULL), "cm_nic NULL");
- cr_assert((cm_nic->my_name.gnix_addr.cdm_id == correct), "cm_nic incorrect cdm_id");
-}
-
-
-Test(dg_allocation, dgram_alloc_wc)
-{
- int ret = 0, i;
- struct gnix_cm_nic *cm_nic;
- struct gnix_datagram **dgram_ptr;
- struct gnix_fid_fabric *fab_priv;
-
- ep_priv = container_of(ep, struct gnix_fid_ep, ep_fid);
- cm_nic = ep_priv->cm_nic;
- cr_assert((cm_nic != NULL), "cm_nic NULL");
-
- cr_assert((cm_nic->dgram_hndl != NULL), "cm_nic dgram_hndl NULL");
-
- fab_priv = container_of(fab, struct gnix_fid_fabric, fab_fid);
-
- dgram_ptr = calloc(fab_priv->n_wc_dgrams,
- sizeof(struct gnix_datagram *));
- cr_assert((dgram_ptr != NULL), "calloc failed");
-
- for (i = 0; i < fab_priv->n_wc_dgrams; i++) {
- ret = _gnix_dgram_alloc(cm_nic->dgram_hndl, GNIX_DGRAM_WC,
- &dgram_ptr[i]);
- cr_assert(!ret, "_gnix_dgram_alloc wc");
- }
-
- for (i = 0; i < fab_priv->n_wc_dgrams; i++) {
- ret = _gnix_dgram_free(dgram_ptr[i]);
- cr_assert(!ret, "_gnix_dgram_free wc");
- }
-
- free(dgram_ptr);
-}
-
-Test(dg_allocation, dgram_alloc_wc_alt)
-{
- int ret = 0, i;
- struct gnix_cm_nic *cm_nic;
- struct gnix_datagram *dgram_ptr;
- struct gnix_fid_fabric *fab_priv;
-
- ep_priv = container_of(ep, struct gnix_fid_ep, ep_fid);
- cm_nic = ep_priv->cm_nic;
- cr_assert((cm_nic != NULL), "cm_nic NULL");
-
- cr_assert((cm_nic->dgram_hndl != NULL), "cm_nic dgram_hndl NULL");
-
- fab_priv = container_of(fab, struct gnix_fid_fabric, fab_fid);
-
- for (i = 0; i < fab_priv->n_wc_dgrams; i++) {
- ret = _gnix_dgram_alloc(cm_nic->dgram_hndl, GNIX_DGRAM_WC,
- &dgram_ptr);
- cr_assert(!ret, "_gnix_dgram_alloc wc");
- ret = _gnix_dgram_free(dgram_ptr);
- cr_assert(!ret, "_gnix_dgram_free wc");
- }
-}
-
-Test(dg_allocation, dgram_alloc_bnd)
-{
- int ret = 0, i;
- struct gnix_cm_nic *cm_nic;
- struct gnix_datagram **dgram_ptr;
- struct gnix_fid_fabric *fab_priv;
-
- ep_priv = container_of(ep, struct gnix_fid_ep, ep_fid);
- cm_nic = ep_priv->cm_nic;
- cr_assert((cm_nic != NULL), "cm_nic NULL");
-
- cr_assert((cm_nic->dgram_hndl != NULL), "cm_nic dgram_hndl NULL");
-
- fab_priv = container_of(fab, struct gnix_fid_fabric, fab_fid);
-
- dgram_ptr = calloc(fab_priv->n_bnd_dgrams,
- sizeof(struct gnix_datagram *));
- cr_assert((dgram_ptr != NULL), "calloc failed");
-
- for (i = 0; i < fab_priv->n_bnd_dgrams; i++) {
- ret = _gnix_dgram_alloc(cm_nic->dgram_hndl, GNIX_DGRAM_BND,
- &dgram_ptr[i]);
- cr_assert(!ret, "_gnix_dgram_alloc bnd");
- }
-
- for (i = 0; i < fab_priv->n_wc_dgrams; i++) {
- ret = _gnix_dgram_free(dgram_ptr[i]);
- cr_assert(!ret, "_gnix_dgram_free bnd");
- }
-
- free(dgram_ptr);
-}
-
-Test(dg_allocation, dgram_alloc_wc_bnd)
-{
- int ret = 0, i;
- struct gnix_cm_nic *cm_nic;
- struct gnix_datagram *dgram_ptr;
- struct gnix_fid_fabric *fab_priv;
-
- ep_priv = container_of(ep, struct gnix_fid_ep, ep_fid);
- cm_nic = ep_priv->cm_nic;
- cr_assert((cm_nic != NULL), "cm_nic NULL");
-
- cr_assert((cm_nic->dgram_hndl != NULL), "cm_nic dgram_hndl NULL");
-
- fab_priv = container_of(fab, struct gnix_fid_fabric, fab_fid);
-
- for (i = 0; i < fab_priv->n_bnd_dgrams; i++) {
- ret = _gnix_dgram_alloc(cm_nic->dgram_hndl, GNIX_DGRAM_BND,
- &dgram_ptr);
- cr_assert(!ret, "_gnix_dgram_alloc bnd");
- ret = _gnix_dgram_free(dgram_ptr);
- cr_assert(!ret, "_gnix_dgram_free bnd");
- }
-}
-
-Test(dg_allocation, dgram_pack_unpack)
-{
- int ret = 0;
- ssize_t len;
- struct gnix_cm_nic *cm_nic;
- struct gnix_datagram *dgram_ptr;
- char in_buf[] = "0xdeadbeef";
- char out_buf[GNI_DATAGRAM_MAXSIZE];
-
- ep_priv = container_of(ep, struct gnix_fid_ep, ep_fid);
- cm_nic = ep_priv->cm_nic;
- cr_assert((cm_nic != NULL), "cm_nic NULL");
-
- cr_assert((cm_nic->dgram_hndl != NULL), "cm_nic dgram_hndl NULL");
-
- ret = _gnix_dgram_alloc(cm_nic->dgram_hndl, GNIX_DGRAM_BND,
- &dgram_ptr);
- cr_assert(!ret, "_gnix_dgram_alloc bnd");
-
- /*
- * check pack/unpack for GNIX_DGRAM_IN_BUF
- */
-
- len = _gnix_dgram_pack_buf(dgram_ptr, GNIX_DGRAM_IN_BUF,
- in_buf, sizeof(in_buf));
- cr_assert(len > 0);
- cr_assert_eq(len, (ssize_t)sizeof(in_buf));
-
- len = _gnix_dgram_unpack_buf(dgram_ptr, GNIX_DGRAM_IN_BUF,
- out_buf, sizeof(in_buf));
- cr_assert(len > 0);
- cr_assert_eq(len, (ssize_t)sizeof(in_buf));
-
- cr_assert_eq(0, strcmp(in_buf, out_buf));
-
- /*
- * check pack/unpack for GNIX_DGRAM_OUT_BUF
- */
-
- len = _gnix_dgram_pack_buf(dgram_ptr, GNIX_DGRAM_OUT_BUF,
- in_buf, sizeof(in_buf));
- cr_assert(len > 0);
- cr_assert_eq(len, (ssize_t)sizeof(in_buf));
-
- memset(out_buf, 0, sizeof(out_buf));
-
- len = _gnix_dgram_unpack_buf(dgram_ptr, GNIX_DGRAM_OUT_BUF,
- out_buf, sizeof(in_buf));
- cr_assert(len > 0);
- cr_assert_eq(len, (ssize_t)sizeof(in_buf));
-
- cr_assert_eq(0, strcmp(in_buf, out_buf));
-
- ret = _gnix_dgram_free(dgram_ptr);
- cr_assert(!ret, "_gnix_dgram_free bnd");
-
-}
-
-static struct gnix_address local_address;
-static int dgram_match;
-
-static int dgram_callback_fn(struct gnix_datagram *the_dgram,
- struct gnix_address where_from,
- gni_post_state_t dgram_state)
-{
- if (dgram_state != GNI_POST_COMPLETED) {
- fprintf(stderr, "dgram_state check failed %s %d\n",
- __func__, __LINE__);
- return -FI_EIO;
- }
-
- if ((where_from.device_addr != local_address.device_addr) ||
- (where_from.cdm_id != local_address.cdm_id)) {
- fprintf(stderr, "where from check failed %s %d\n",
- __func__, __LINE__);
- return -FI_EIO;
- }
-
- dgram_match = 1;
- return 0;
-}
-
-Test(dg_allocation, dgram_wc_post_exchg)
-{
- int ret = 0;
- struct gnix_cm_nic *cm_nic;
- struct gnix_datagram *dgram_wc, *dgram_bnd;
-
- ep_priv = container_of(ep, struct gnix_fid_ep, ep_fid);
- cm_nic = ep_priv->cm_nic;
- cr_assert((cm_nic != NULL), "cm_nic NULL");
-
- cr_assert((cm_nic->dgram_hndl != NULL), "cm_nic dgram_hndl NULL");
-
- ret = _gnix_dgram_alloc(cm_nic->dgram_hndl, GNIX_DGRAM_WC,
- &dgram_wc);
- cr_assert(!ret, "_gnix_dgram_alloc wc");
-
- dgram_wc->callback_fn = dgram_callback_fn;
- ret = _gnix_dgram_wc_post(dgram_wc);
- cr_assert((ret == 0), "_gnix_dgram_alloc wc");
-
- ret = _gnix_dgram_alloc(cm_nic->dgram_hndl, GNIX_DGRAM_BND,
- &dgram_bnd);
- cr_assert((ret == 0), "_gnix_dgram_alloc bnd");
-
- dgram_bnd->target_addr = cm_nic->my_name.gnix_addr;
-
- local_address = cm_nic->my_name.gnix_addr;
-
- dgram_bnd->callback_fn = dgram_callback_fn;
- ret = _gnix_dgram_bnd_post(dgram_bnd);
- cr_assert(ret == 0);
-
- /*
- * progress auto, don't need to do anything
- */
- while (dgram_match != 1) {
- ret = _gnix_cm_nic_progress(cm_nic);
- cr_assert(ret == 0);
- pthread_yield();
- }
-
- ret = _gnix_dgram_free(dgram_bnd);
- cr_assert(!ret, "_gnix_dgram_free bnd");
-
- ret = _gnix_dgram_free(dgram_wc);
- cr_assert(!ret, "_gnix_dgram_free wc");
-
-}
-
-Test(dg_allocation, dgram_wc_post_exchg_manual, .init = dg_setup_prog_manual)
-{
- int ret = 0;
- struct gnix_cm_nic *cm_nic;
- struct gnix_datagram *dgram_wc, *dgram_bnd;
-
- ep_priv = container_of(ep, struct gnix_fid_ep, ep_fid);
- cm_nic = ep_priv->cm_nic;
- cr_assert((cm_nic != NULL), "cm_nic NULL");
-
- cr_assert(cm_nic->ctrl_progress == FI_PROGRESS_MANUAL);
-
- cr_assert((cm_nic->dgram_hndl != NULL), "cm_nic dgram_hndl NULL");
-
- ret = _gnix_dgram_alloc(cm_nic->dgram_hndl, GNIX_DGRAM_WC,
- &dgram_wc);
- cr_assert(!ret, "_gnix_dgram_alloc wc");
-
- dgram_wc->callback_fn = dgram_callback_fn;
- ret = _gnix_dgram_wc_post(dgram_wc);
- cr_assert((ret == 0), "_gnix_dgram_alloc wc");
-
- ret = _gnix_dgram_alloc(cm_nic->dgram_hndl, GNIX_DGRAM_BND,
- &dgram_bnd);
- cr_assert((ret == 0), "_gnix_dgram_alloc bnd");
-
- dgram_bnd->target_addr = cm_nic->my_name.gnix_addr;
-
- local_address = cm_nic->my_name.gnix_addr;
-
- dgram_bnd->callback_fn = dgram_callback_fn;
- ret = _gnix_dgram_bnd_post(dgram_bnd);
- cr_assert(ret == 0);
-
- /*
- * progress auto, don't need to do anything
- */
- while (dgram_match != 1) {
- ret = _gnix_cm_nic_progress(cm_nic);
- cr_assert(ret == 0);
- pthread_yield();
- }
-
- ret = _gnix_dgram_free(dgram_bnd);
- cr_assert(!ret, "_gnix_dgram_free bnd");
-
- ret = _gnix_dgram_free(dgram_wc);
- cr_assert(!ret, "_gnix_dgram_free wc");
-
-}
diff --git a/prov/gni/test/dlist-utils.c b/prov/gni/test/dlist-utils.c
deleted file mode 100644
index ce5a8e9..0000000
--- a/prov/gni/test/dlist-utils.c
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <sys/time.h>
-#include <stdlib.h>
-
-#include <fi_list.h>
-
-#include "gnix_util.h"
-
-#include <criterion/criterion.h>
-
-struct element {
- int val;
- struct dlist_entry entry;
-};
-
-static void setup(void)
-{
- srand(time(NULL));
-}
-
-static void teardown(void)
-{
-}
-
-static void generate_perm(int *perm, int len)
-{
- int i;
- /* good 'nuff */
- for (i = 0; i < len; i++) {
- int t = perm[i];
- int j = rand() % len;
-
- perm[i] = perm[j];
- perm[j] = t;
- }
-}
-
-TestSuite(dlist_utils, .init = setup, .fini = teardown);
-
-struct dlist_test {
- char dummy[3];
- struct dlist_entry le;
- int x;
-};
-
-Test(dlist_utils, for_each)
-{
- int i;
- const int n = 593;
- struct dlist_entry dl;
- struct dlist_test dt[n];
- struct dlist_test *elem;
- int perm[n];
-
- for (i = 0; i < n; i++)
- perm[i] = i;
-
- generate_perm(perm, n);
-
- dlist_init(&dl);
- for (i = 0; i < n; i++) {
- dt[perm[i]].x = i;
- dlist_insert_tail(&dt[perm[i]].le, &dl);
- }
-
- i = 0;
- dlist_for_each(&dl, elem, le) {
- cr_assert(elem->x == i);
- ++i;
- }
-}
-
-Test(dlist_utils, for_each_empty)
-{
- struct dlist_entry dl;
- struct dlist_test *elem;
-
- dlist_init(&dl);
-
- dlist_for_each(&dl, elem, le) {
- cr_assert(false);
- }
-}
-
-Test(dlist_utils, for_each_safe)
-{
- int i;
- const int n = 1129;
- struct dlist_entry dl;
- struct dlist_test dt[n];
- struct dlist_test *elem, *next;
- int perm[n];
-
- for (i = 0; i < n; i++)
- perm[i] = i;
-
- generate_perm(perm, n);
-
- dlist_init(&dl);
- for (i = 0; i < n; i++) {
- dt[perm[i]].x = i;
- dlist_insert_tail(&dt[perm[i]].le, &dl);
- }
-
- i = 0;
- dlist_for_each_safe(&dl, elem, next, le) {
- cr_assert(elem->x == i);
- dlist_remove(&elem->le);
- ++i;
- }
-}
-
-Test(dlist_utils, for_each_safe_empty)
-{
- struct dlist_entry dl;
- struct dlist_test *elem, *next;
-
- dlist_init(&dl);
-
- dlist_for_each_safe(&dl, elem, next, le) {
- cr_assert(false);
- }
-}
-
-Test(dlist_utils, dlist_splice_head_test)
-{
- struct dlist_entry list1, list2;
- struct element values[4], *current;
- int i;
- int expected[4] = {2, 3, 0, 1};
-
- for (i = 0; i < 4; i++) {
- values[i].val = i;
- dlist_init(&values[i].entry);
- }
-
- dlist_init(&list1);
- dlist_init(&list2);
- dlist_insert_tail(&values[0].entry, &list1);
- dlist_insert_tail(&values[1].entry, &list1);
-
- dlist_insert_tail(&values[2].entry, &list2);
- dlist_insert_tail(&values[3].entry, &list2);
-
- dlist_splice_head(&list1, &list2);
-
- cr_assert(dlist_empty(&list2));
-
- i = 0;
- dlist_for_each(&list1, current, entry)
- {
- cr_assert(current->val == expected[i]);
- i++;
- }
-
-}
-
-Test(dlist_utils, dlist_splice_tail_test)
-{
- struct dlist_entry list1, list2;
- struct element values[4], *current;
- int i;
- int expected[4] = {0, 1, 2, 3};
-
- for (i = 0; i < 4; i++) {
- values[i].val = i;
- dlist_init(&values[i].entry);
- }
-
- dlist_init(&list1);
- dlist_init(&list2);
- dlist_insert_tail(&values[0].entry, &list1);
- dlist_insert_tail(&values[1].entry, &list1);
-
- dlist_insert_tail(&values[2].entry, &list2);
- dlist_insert_tail(&values[3].entry, &list2);
-
- dlist_splice_tail(&list1, &list2);
-
- cr_assert(dlist_empty(&list2));
-
- i = 0;
- dlist_for_each(&list1, current, entry)
- {
- cr_assert(current->val == expected[i]);
- i++;
- }
-
-
-}
diff --git a/prov/gni/test/dom.c b/prov/gni/test/dom.c
deleted file mode 100644
index 0b78266..0000000
--- a/prov/gni/test/dom.c
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-
-#include "gnix.h"
-
-#include <criterion/criterion.h>
-
-static struct fid_fabric *fabric;
-static struct fi_info *fi;
-
-static void setup(void)
-{
- int ret;
- struct fi_info *hints;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(ret == FI_SUCCESS, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fabric, NULL);
- cr_assert(ret == FI_SUCCESS, "fi_fabric");
-
- fi_freeinfo(hints);
-}
-
-static void teardown(void)
-{
- int ret;
-
- ret = fi_close(&fabric->fid);
- cr_assert(ret == FI_SUCCESS, "fi_close fabric");
-
- fi_freeinfo(fi);
-}
-
-TestSuite(domain, .init = setup, .fini = teardown);
-
-Test(domain, many_domains)
-{
- int i, ret;
- const int num_doms = 7919;
- struct fid_domain *doms[num_doms];
- struct gnix_fid_domain *gdom;
- struct gnix_fid_fabric *gfab;
-
- memset(doms, 0, num_doms*sizeof(struct fid_domain *));
-
- gfab = container_of(fabric, struct gnix_fid_fabric, fab_fid);
- for (i = 0; i < num_doms; i++) {
- ret = fi_domain(fabric, fi, &doms[i], NULL);
- cr_assert(ret == FI_SUCCESS, "fi_domain");
- gdom = container_of(doms[i], struct gnix_fid_domain,
- domain_fid);
- cr_assert(gdom, "domain not allcoated");
- cr_assert(gdom->fabric == gfab, "Incorrect fabric");
- cr_assert(atomic_get(&gdom->ref_cnt.references) == 1,
- "Incorrect ref_cnt");
-
- }
-
- for (i = num_doms-1; i >= 0; i--) {
- ret = fi_close(&doms[i]->fid);
- cr_assert(ret == FI_SUCCESS, "fi_close domain");
- }
-
-}
-
-Test(domain, open_ops)
-{
- int i, ret;
- const int num_doms = 11;
- struct fid_domain *doms[num_doms];
- struct fi_gni_ops_domain *gni_domain_ops;
- enum dom_ops_val op;
- uint32_t val;
-
- memset(doms, 0, num_doms*sizeof(struct fid_domain *));
-
- for (i = 0; i < num_doms; i++) {
- ret = fi_domain(fabric, fi, &doms[i], NULL);
- cr_assert(ret == FI_SUCCESS, "fi_domain");
- ret = fi_open_ops(&doms[i]->fid, FI_GNI_DOMAIN_OPS_1,
- 0, (void **) &gni_domain_ops, NULL);
- cr_assert(ret == FI_SUCCESS, "fi_open_ops");
- for (op = 0; op < GNI_NUM_DOM_OPS; op++) {
- val = i*op+op;
- ret = gni_domain_ops->set_val(&doms[i]->fid, op, &val);
- cr_assert(ret == FI_SUCCESS, "set_val");
- }
- }
-
- for (i = num_doms-1; i >= 0; i--) {
- for (op = 0; op < GNI_NUM_DOM_OPS; op++) {
- ret = gni_domain_ops->get_val(&doms[i]->fid, op, &val);
- cr_assert(ret == FI_SUCCESS, "get_val");
- cr_assert(val == i*op+op, "Incorrect op value");
- }
- ret = fi_close(&doms[i]->fid);
- cr_assert(ret == FI_SUCCESS, "fi_close domain");
- }
-
-}
-
-Test(domain, invalid_open_ops)
-{
- int ret;
- struct fid_domain *dom;
- struct fi_gni_ops_domain *gni_domain_ops;
- uint32_t val = 0;
-
- ret = fi_domain(fabric, fi, &dom, NULL);
- cr_assert(ret == FI_SUCCESS, "fi_domain");
- ret = fi_open_ops(&dom->fid, FI_GNI_DOMAIN_OPS_1,
- 0, (void **) &gni_domain_ops, NULL);
- cr_assert(ret == FI_SUCCESS, "fi_open_ops");
-
- ret = gni_domain_ops->get_val(&dom->fid, GNI_NUM_DOM_OPS, &val);
- cr_assert(ret == -FI_EINVAL, "get_val");
-
- ret = gni_domain_ops->set_val(&dom->fid, GNI_NUM_DOM_OPS, &val);
- cr_assert(ret == -FI_EINVAL, "set_val");
-
- ret = fi_close(&dom->fid);
- cr_assert(ret == FI_SUCCESS, "fi_close domain");
-}
diff --git a/prov/gni/test/ep.c b/prov/gni/test/ep.c
deleted file mode 100644
index 3b6ccd9..0000000
--- a/prov/gni/test/ep.c
+++ /dev/null
@@ -1,278 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-
-#include "gnix_ep.h"
-
-#include <criterion/criterion.h>
-
-static struct fi_info *hints;
-static struct fi_info *fi;
-static struct fid_fabric *fab;
-static struct fid_domain *dom;
-
-static void setup(void)
-{
- int ret;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- ret = fi_domain(fab, fi, &dom, NULL);
- cr_assert(!ret, "fi_domain");
-
-}
-
-static void teardown(void)
-{
- int ret;
-
- ret = fi_close(&dom->fid);
- cr_assert(!ret, "fi_close domain");
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "fi_close fabric");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-}
-
-TestSuite(endpoint, .init = setup, .fini = teardown);
-
-Test(endpoint_info, info)
-{
- int ret;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
- cr_assert_eq(fi->ep_attr->type, FI_EP_RDM);
- cr_assert_eq(fi->next->ep_attr->type, FI_EP_DGRAM);
-
- fi_freeinfo(fi);
-
- hints->ep_attr->type = FI_EP_RDM;
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
- cr_assert_eq(fi->ep_attr->type, FI_EP_RDM);
- cr_assert_eq(fi->next, NULL);
-
- fi_freeinfo(fi);
-
- hints->ep_attr->type = FI_EP_DGRAM;
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
- cr_assert_eq(fi->ep_attr->type, FI_EP_DGRAM);
- cr_assert_eq(fi->next, NULL);
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-}
-
-Test(endpoint, open_close)
-{
- int i, ret;
- const int num_eps = 61;
- struct fid_ep *eps[num_eps];
-
- memset(eps, 0, num_eps*sizeof(struct fid_ep *));
-
- for (i = 0; i < num_eps; i++) {
- ret = fi_endpoint(dom, fi, &eps[i], NULL);
- cr_assert(!ret, "fi_endpoint");
- struct gnix_fid_ep *ep = container_of(eps[i],
- struct gnix_fid_ep,
- ep_fid);
- cr_assert(ep, "endpoint not allcoated");
-
- /* Check fields (fill in as implemented) */
- cr_assert(ep->nic, "NIC not allocated");
- cr_assert(!_gnix_sfl_empty(&ep->fr_freelist),
- "gnix_fab_req freelist empty");
- }
-
- for (i = num_eps-1; i >= 0; i--) {
- ret = fi_close(&eps[i]->fid);
- cr_assert(!ret, "fi_close endpoint");
- }
-
-}
-
-Test(endpoint, getsetopt)
-{
- int ret;
- struct fid_ep *ep = NULL;
- uint64_t val;
- size_t len;
-
- ret = fi_endpoint(dom, fi, &ep, NULL);
- cr_assert(!ret, "fi_endpoint");
-
- /* Test bad params. */
- ret = fi_getopt(&ep->fid, !FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
- (void *)&val, &len);
- cr_assert(ret == -FI_ENOPROTOOPT, "fi_getopt");
-
- ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, !FI_OPT_MIN_MULTI_RECV,
- (void *)&val, &len);
- cr_assert(ret == -FI_ENOPROTOOPT, "fi_getopt");
-
- ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
- NULL, &len);
- cr_assert(ret == -FI_EINVAL, "fi_getopt");
-
- ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
- (void *)&val, NULL);
- cr_assert(ret == -FI_EINVAL, "fi_getopt");
-
- ret = fi_setopt(&ep->fid, !FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
- (void *)&val, sizeof(size_t));
- cr_assert(ret == -FI_ENOPROTOOPT, "fi_setopt");
-
- ret = fi_setopt(&ep->fid, FI_OPT_ENDPOINT, !FI_OPT_MIN_MULTI_RECV,
- (void *)&val, sizeof(size_t));
- cr_assert(ret == -FI_ENOPROTOOPT, "fi_setopt");
-
- ret = fi_setopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
- NULL, sizeof(size_t));
- cr_assert(ret == -FI_EINVAL, "fi_setopt");
-
- ret = fi_setopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
- (void *)&val, sizeof(size_t) - 1);
- cr_assert(ret == -FI_EINVAL, "fi_setopt");
-
- /* Test update. */
- ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
- (void *)&val, &len);
- cr_assert(!ret, "fi_getopt");
- cr_assert(val == GNIX_OPT_MIN_MULTI_RECV_DEFAULT, "fi_getopt");
- cr_assert(len == sizeof(size_t), "fi_getopt");
-
- val = 128;
- ret = fi_setopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
- (void *)&val, sizeof(size_t));
- cr_assert(!ret, "fi_setopt");
-
- ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
- (void *)&val, &len);
- cr_assert(!ret, "fi_getopt");
- cr_assert(val == 128, "fi_getopt");
- cr_assert(len == sizeof(size_t), "fi_getopt");
-
- ret = fi_close(&ep->fid);
- cr_assert(!ret, "fi_close endpoint");
-}
-
-Test(endpoint, sizeleft)
-{
- int ret;
- size_t sz;
- struct fid_ep *ep = NULL;
-
- ret = fi_endpoint(dom, fi, &ep, NULL);
- cr_assert(!ret, "fi_endpoint");
-
- /* Test default values. */
- sz = fi_rx_size_left(ep);
- cr_assert(sz == 64, "fi_rx_size_left");
-
- sz = fi_tx_size_left(ep);
- cr_assert(sz == 64, "fi_tx_size_left");
-
- ret = fi_close(&ep->fid);
- cr_assert(!ret, "fi_close endpoint");
-}
-
-Test(endpoint, getsetopt_gni_ep)
-{
- int ret;
- int val;
- struct fid_ep *ep = NULL;
- struct fi_gni_ops_ep *ep_ops;
- struct gnix_fid_ep *ep_priv = NULL;
-
- ret = fi_endpoint(dom, fi, &ep, NULL);
- cr_assert(!ret, "fi_endpoint");
-
- ep_priv = (struct gnix_fid_ep *) ep;
-
- ret = fi_open_ops(&ep->fid, "ep ops 1", 0, (void **) &ep_ops, NULL);
- cr_assert(!ret, "fi_open_ops endpoint");
-
- ret = ep_ops->get_val(&ep->fid, GNI_HASH_TAG_IMPL, &val);
- cr_assert(!ret, "ep_ops get_val");
- cr_assert_eq(val, 0);
- cr_assert_eq(ep_priv->use_tag_hlist, 0);
-
- val = 1; // set the hash implementation
- ret = ep_ops->set_val(&ep->fid, GNI_HASH_TAG_IMPL, &val);
- cr_assert(!ret, "ep_ops set_val");
- cr_assert_eq(ep_priv->use_tag_hlist, 1);
- cr_assert_eq(ep_priv->unexp_recv_queue.attr.type, GNIX_TAG_HLIST);
- cr_assert_eq(ep_priv->posted_recv_queue.attr.type, GNIX_TAG_HLIST);
- cr_assert_eq(ep_priv->tagged_unexp_recv_queue.attr.type, GNIX_TAG_HLIST);
- cr_assert_eq(ep_priv->tagged_posted_recv_queue.attr.type, GNIX_TAG_HLIST);
-
-
- val = 0; // reset the value
- ret = ep_ops->get_val(&ep->fid, GNI_HASH_TAG_IMPL, &val);
- cr_assert(!ret, "ep_ops get_val");
- cr_assert_eq(val, 1);
- cr_assert_eq(ep_priv->use_tag_hlist, 1);
-
- ret = fi_close(&ep->fid);
- cr_assert(!ret, "fi_close endpoint");
-}
diff --git a/prov/gni/test/eq.c b/prov/gni/test/eq.c
deleted file mode 100644
index eaa6456..0000000
--- a/prov/gni/test/eq.c
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdlib.h>
-
-#include "gnix_eq.h"
-#include "gnix.h"
-
-#include <criterion/criterion.h>
-
-static struct fid_fabric *fab;
-static struct fi_info *hints;
-static struct fi_info *fi;
-
-static struct gnix_fid_eq *eq_priv;
-static struct fi_eq_attr eq_attr;
-static struct fid_eq *eq;
-
-static struct gnix_fid_wait *wait_priv;
-static struct fi_wait_attr wait_attr;
-static struct fid_wait *wait_set;
-
-void _setup(void)
-{
- int ret = 0;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo failed.");
-
- hints->mode = ~0;
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert_eq(ret, FI_SUCCESS, "fi_getinfo failed.");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert_eq(ret, FI_SUCCESS, "fi_fabric failed.");
-}
-
-void _teardown(void)
-{
- int ret = 0;
-
- ret = fi_close(&fab->fid);
- cr_assert_eq(ret, FI_SUCCESS, "failure in closing fabric.");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-}
-
-void eq_teardown(void)
-{
- cr_assert_eq(FI_SUCCESS, fi_close(&eq->fid), "failure in closing eq.");
- _teardown();
-}
-
-void eq_create(enum fi_wait_obj wait_obj, size_t size)
-{
- int ret = 0;
-
- eq_attr.size = size;
- eq_attr.wait_obj = wait_obj;
-
- ret = fi_eq_open(fab, &eq_attr, &eq, NULL);
- cr_assert_eq(ret, FI_SUCCESS, "fi_eq_open failed.");
-
- eq_priv = container_of(eq, struct gnix_fid_eq, eq_fid);
-
- if (eq_priv->wait) {
- wait_priv = container_of(eq_priv->wait, struct gnix_fid_wait,
- wait);
- }
-}
-
-void eq_wait_none_setup(void)
-{
- _setup();
- eq_create(FI_WAIT_NONE, 8);
-}
-
-void eq_wait_fd_setup(void)
-{
- _setup();
- eq_create(FI_WAIT_FD, 8);
-}
-
-void eq_wait_unspec_setup(void)
-{
- _setup();
- eq_create(FI_WAIT_UNSPEC, 8);
-}
-
-void eq_wait_mutex_cond_setup(void)
-{
- _setup();
- eq_create(FI_WAIT_MUTEX_COND, 8);
-}
-
-TestSuite(eq_wait_obj, .fini = eq_teardown);
-
-Test(eq_wait_obj, none, .init = eq_wait_none_setup)
-{
- cr_expect(!wait_priv, "wait_priv should be null.");
-}
-
-Test(eq_wait_obj, unspec, .init = eq_wait_unspec_setup)
-{
- cr_expect_eq(wait_priv->type, FI_WAIT_FD);
- cr_expect_eq(wait_priv->type, eq_priv->attr.wait_obj);
- cr_expect_eq(wait_priv->type, eq_attr.wait_obj);
- cr_expect_eq(&wait_priv->fabric->fab_fid, fab);
- cr_expect_eq(wait_priv->cond_type, FI_CQ_COND_NONE);
-}
-
-Test(eq_wait_obj, fd, .init = eq_wait_fd_setup)
-{
- cr_expect_eq(wait_priv->type, FI_WAIT_FD);
- cr_expect_eq(wait_priv->type, eq_priv->attr.wait_obj);
- cr_expect_eq(wait_priv->type, eq_attr.wait_obj);
- cr_expect_eq(&wait_priv->fabric->fab_fid, fab);
- cr_expect_eq(wait_priv->cond_type, FI_CQ_COND_NONE);
-}
-
-Test(eq_wait_obj, mutex_cond, .init = eq_wait_mutex_cond_setup)
-{
- cr_expect_eq(wait_priv->type, FI_WAIT_MUTEX_COND);
- cr_expect_eq(wait_priv->type, eq_priv->attr.wait_obj);
- cr_expect_eq(wait_priv->type, eq_attr.wait_obj);
- cr_expect_eq(&wait_priv->fabric->fab_fid, fab);
- cr_expect_eq(wait_priv->cond_type, FI_CQ_COND_NONE);
-}
-
-TestSuite(eq_wait_control, .fini = eq_teardown, .disabled = true);
-
-/*
-Test(eq_wait_control, none, .init = eq_wait_none_setup)
-{
- int ret;
- int fd;
-
- ret = fi_control(&eq_priv->eq_fid.fid, FI_GETWAIT, &fd);
- cr_expect_eq(-FI_ENOSYS, ret, "fi_control exists for none.");
-}
-*/
-
-Test(eq_wait_control, unspec, .init = eq_wait_unspec_setup)
-{
- int ret;
- int fd;
-
- ret = fi_control(&eq_priv->eq_fid.fid, FI_GETWAIT, &fd);
- cr_expect_eq(FI_SUCCESS, ret, "fi_control failed.");
-
- cr_expect_eq(wait_priv->fd[WAIT_READ], fd);
-}
-
-Test(eq_wait_control, fd, .init = eq_wait_fd_setup)
-{
- int ret;
- int fd;
-
- ret = fi_control(&eq_priv->eq_fid.fid, FI_GETWAIT, &fd);
- cr_expect_eq(FI_SUCCESS, ret, "fi_control failed.");
-
- cr_expect_eq(wait_priv->fd[WAIT_READ], fd);
-}
-
-Test(eq_wait_control, mutex_cond, .init = eq_wait_mutex_cond_setup)
-{
- int ret;
- struct fi_mutex_cond mutex_cond;
-
- ret = fi_control(&eq_priv->eq_fid.fid, FI_GETWAIT, &mutex_cond);
- cr_expect_eq(FI_SUCCESS, ret, "fi_control failed.");
-
- ret = memcmp(&wait_priv->mutex, mutex_cond.mutex,
- sizeof(*mutex_cond.mutex));
- cr_expect_eq(0, ret, "mutex compare failed.");
-
- ret = memcmp(&wait_priv->cond, mutex_cond.cond,
- sizeof(*mutex_cond.cond));
- cr_expect_eq(0, ret, "cond compare failed.");
-}
-
-Test(eq_wait_set, fd, .init = _setup, .fini = _teardown, .disabled = true)
-{
- int ret;
- int fd;
-
- wait_attr.wait_obj = FI_WAIT_FD;
-
- ret = fi_wait_open(fab, &wait_attr, &wait_set);
- cr_expect_eq(FI_SUCCESS, ret, "fi_wait_open failed.");
-
- wait_priv = container_of(wait_set, struct gnix_fid_wait, wait);
-
- eq_attr.size = 8;
- eq_attr.wait_obj = FI_WAIT_SET;
- eq_attr.wait_set = wait_set;
-
- ret = fi_eq_open(fab, &eq_attr, &eq, NULL);
- cr_expect_eq(FI_SUCCESS, ret, "fi_eq_open failed.");
-
- eq_priv = container_of(eq, struct gnix_fid_eq, eq_fid);
-
- ret = fi_control(&eq_priv->eq_fid.fid, FI_GETWAIT, &fd);
- cr_expect_eq(FI_SUCCESS, ret, "fi_control failed.");
-
- cr_expect_eq(wait_priv->fd[WAIT_READ], fd);
-
- ret = fi_close(&eq->fid);
- cr_expect_eq(FI_SUCCESS, ret, "failure in closing cq.");
-
- ret = fi_close(&wait_set->fid);
- cr_expect_eq(FI_SUCCESS, ret, "failure in closing waitset.");
-}
diff --git a/prov/gni/test/freelist.c b/prov/gni/test/freelist.c
deleted file mode 100644
index 6744524..0000000
--- a/prov/gni/test/freelist.c
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <stddef.h>
-#include <sys/time.h>
-
-#include "gnix_freelist.h"
-
-#include <criterion/criterion.h>
-
-static void setup(void)
-{
- srand(time(NULL));
-}
-
-static void teardown(void)
-{
-}
-
-static void generate_perm(int *perm, int len)
-{
- int i;
- /* good 'nuff */
- for (i = 0; i < len; i++) {
- int t = perm[i];
- int j = rand() % len;
-
- perm[i] = perm[j];
- perm[j] = t;
- }
-}
-
-TestSuite(gnix_freelist, .init = setup, .fini = teardown);
-
-Test(gnix_freelist, freelist_init_destroy)
-{
- const int n = 13;
- struct gnix_s_freelist fls[n];
- int i, ret;
-
- /* non-optimized code may not zero structures */
- memset(fls, 0x0, n * sizeof(struct gnix_s_freelist));
-
- for (i = 0; i < n; i++) {
- ret = _gnix_sfl_init(sizeof(struct slist_entry), 0,
- 2*n, n, n, 3*n, &fls[i]);
- cr_assert_eq(ret, FI_SUCCESS, "Failed to initialize freelist");
- }
-
- for (i = n-1; i >= 0; i--)
- _gnix_sfl_destroy(&fls[i]);
-}
-
-Test(gnix_freelist, freelist_refill_test)
-{
- struct gnix_s_freelist fl;
- int i, ret;
- const int num_elems = 71;
- struct slist_entry *elems[num_elems];
- const int refill_size = 47;
- struct slist_entry *refill_elems[refill_size];
-
- /* non-optimized code may not zero structures */
- memset(&fl, 0x0, sizeof(struct gnix_s_freelist));
-
- ret = _gnix_sfl_init(sizeof(struct slist_entry), 0,
- num_elems, refill_size, 0, 0, &fl);
- cr_assert_eq(ret, FI_SUCCESS, "Failed to initialize freelist");
-
- for (i = 0; i < num_elems; i++) {
- ret = _gnix_sfe_alloc(&elems[i], &fl);
- cr_assert_eq(ret, FI_SUCCESS, "Failed to obtain slist_entry");
- }
- cr_assert(_gnix_sfl_empty(&fl), "Freelist not empty");
-
- for (i = 0; i < refill_size; i++) {
- ret = _gnix_sfe_alloc(&refill_elems[i], &fl);
- cr_assert_eq(ret, FI_SUCCESS, "Failed to obtain slist_entry");
- if (i != refill_size-1) {
- /* Not the last one, so must not be empty */
- cr_assert(!_gnix_sfl_empty(&fl), "Freelist empty");
- }
- }
- cr_assert(_gnix_sfl_empty(&fl), "Freelist not empty");
-
- for (i = num_elems-1; i >= 0 ; i--)
- _gnix_sfe_free(elems[i], &fl);
-
- for (i = refill_size-1; i >= 0 ; i--)
- _gnix_sfe_free(refill_elems[i], &fl);
-
- _gnix_sfl_destroy(&fl);
-}
-
-struct slist_ts {
- char dummy[7];
- struct slist_entry e;
- int n;
-};
-
-Test(gnix_freelist, freelist_random_alloc_free)
-{
- struct gnix_s_freelist fl;
- int i, ret;
- const int n = 719;
- int perm[n];
- struct slist_entry *se;
- struct slist_ts *ts[n];
-
- for (i = 0; i < n; i++)
- perm[i] = i;
-
- generate_perm(perm, n);
-
- /* non-optimized code may not zero structures */
- memset(&fl, 0x0, sizeof(struct gnix_s_freelist));
-
- ret = _gnix_sfl_init(sizeof(struct slist_ts),
- offsetof(struct slist_ts, e),
- 0, 0, 0, 0, &fl);
- cr_assert_eq(ret, FI_SUCCESS, "Failed to initialize freelist");
-
- for (i = 0; i < n; i++) {
- ret = _gnix_sfe_alloc(&se, &fl);
- cr_assert_eq(ret, FI_SUCCESS,
- "Failed to obtain valid slist_entry");
- ts[i] = container_of(se, struct slist_ts, e);
- ts[i]->n = perm[i];
- }
-
- for (i = 0; i < n; i++) {
- int j = perm[i];
-
- cr_assert(ts[j]->n == perm[j], "Incorrect value");
- _gnix_sfe_free(&ts[j]->e, &fl);
- ts[j] = NULL;
- }
-
- _gnix_sfl_destroy(&fl);
-}
-
diff --git a/prov/gni/test/hashtable.c b/prov/gni/test/hashtable.c
deleted file mode 100644
index 19a7bfa..0000000
--- a/prov/gni/test/hashtable.c
+++ /dev/null
@@ -1,757 +0,0 @@
-/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdlib.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <stdint.h>
-
-#include <rdma/fi_errno.h>
-#include <gnix_hashtable.h>
-#include <gnix_bitmap.h>
-
-#include <criterion/criterion.h>
-
-#define __GNIX_MAGIC_VALUE 0xDEADBEEF
-
-extern const gnix_hashtable_attr_t default_attr;
-
-typedef struct gnix_test_element {
- uint64_t val;
- uint64_t key;
- uint64_t magic;
-} gnix_test_element_t;
-
-#define GNIX_TEST_ELEMENT_INIT(_val, _key) \
- { .val = (_val), .key = (_key), .magic = (__GNIX_MAGIC_VALUE) }
-
-gnix_test_element_t elements[4] = {
- GNIX_TEST_ELEMENT_INIT(1, 100),
- GNIX_TEST_ELEMENT_INIT(2, 200),
- GNIX_TEST_ELEMENT_INIT(10, 300),
- GNIX_TEST_ELEMENT_INIT(777, 500000)
-};
-
-gnix_test_element_t *simple_element = &elements[0];
-gnix_hashtable_t *test_ht = NULL;
-
-void __gnix_hashtable_test_uninitialized(void)
-{
- cr_assert(test_ht->ht_state == GNIX_HT_STATE_UNINITIALIZED);
- cr_assert(test_ht->ht_size == 0);
- cr_assert(test_ht->ht_lf_tbl == NULL);
-}
-
-void __gnix_hashtable_test_setup_bare(void)
-{
- cr_assert(test_ht == NULL);
- test_ht = (gnix_hashtable_t *) calloc(1, sizeof(gnix_hashtable_t));
- cr_assert(test_ht != NULL);
-
- __gnix_hashtable_test_uninitialized();
-}
-
-
-void __gnix_hashtable_test_teardown_bare(void)
-{
- cr_assert(test_ht != NULL);
- free(test_ht);
- test_ht = NULL;
-}
-
-void __gnix_hashtable_test_initialized(void)
-{
- cr_assert(test_ht->ht_state == GNIX_HT_STATE_READY);
- cr_assert(atomic_get(&test_ht->ht_elements) == 0);
- cr_assert(test_ht->ht_size == test_ht->ht_attr.ht_initial_size);
- cr_assert(test_ht->ht_lf_tbl != NULL);
-}
-
-void __gnix_hashtable_test_destroyed_clean(void)
-{
- cr_assert(test_ht->ht_state == GNIX_HT_STATE_DEAD);
- cr_assert(atomic_get(&test_ht->ht_elements) == 0);
- cr_assert(test_ht->ht_size == 0);
- cr_assert(test_ht->ht_lf_tbl == NULL);
-}
-
-void __gnix_hashtable_destroy(void)
-{
- int ret = _gnix_ht_destroy(test_ht);
- cr_assert(ret == 0);
- __gnix_hashtable_test_destroyed_clean();
-}
-
-void __gnix_hashtable_initialize(void)
-{
- int ret;
-
- ret = _gnix_ht_init(test_ht, NULL);
- cr_assert(ret == 0);
-
- __gnix_hashtable_test_initialized();
-}
-
-void __gnix_hashtable_initialize_attr(gnix_hashtable_attr_t *attr)
-{
- int ret;
-
- ret = _gnix_ht_init(test_ht, attr);
- cr_assert(ret == 0);
-
- __gnix_hashtable_test_initialized();
-}
-
-void __gnix_hashtable_test_setup(void)
-{
- __gnix_hashtable_test_setup_bare();
-
- __gnix_hashtable_test_uninitialized();
-
- __gnix_hashtable_initialize();
-}
-
-void __gnix_hashtable_test_setup_locked(void)
-{
- gnix_hashtable_attr_t attr = default_attr;
-
- __gnix_hashtable_test_setup_bare();
-
- __gnix_hashtable_test_uninitialized();
-
- attr.ht_internal_locking = 1;
- __gnix_hashtable_initialize_attr(&attr);
-}
-
-void __gnix_hashtable_test_teardown(void)
-{
- __gnix_hashtable_destroy();
-
- __gnix_hashtable_test_teardown_bare();
-}
-
-/*
- * Basic functionality tests for the gnix_hashtable_t object
- */
-
-TestSuite(gnix_hashtable_basic,
- .init = __gnix_hashtable_test_setup_bare,
- .fini = __gnix_hashtable_test_teardown_bare);
-
-TestSuite(gnix_hashtable_advanced,
- .init = __gnix_hashtable_test_setup,
- .fini = __gnix_hashtable_test_teardown);
-
-TestSuite(gnix_hashtable_locked,
- .init = __gnix_hashtable_test_setup_locked,
- .fini = __gnix_hashtable_test_teardown);
-
-Test(gnix_hashtable_basic, uninitialized)
-{
- __gnix_hashtable_test_uninitialized();
-}
-
-
-Test(gnix_hashtable_basic, initialize_ht)
-{
- __gnix_hashtable_initialize();
- __gnix_hashtable_destroy();
-}
-
-Test(gnix_hashtable_basic, initialize_locked_ht)
-{
- int ret;
- gnix_hashtable_attr_t attr;
-
- memcpy(&attr, &default_attr, sizeof(gnix_hashtable_attr_t));
-
- attr.ht_internal_locking = 1;
-
- ret = _gnix_ht_init(test_ht, &attr);
- cr_assert(ret == 0);
-
- __gnix_hashtable_test_initialized();
-
- __gnix_hashtable_destroy();
-}
-
-Test(gnix_hashtable_basic, err_initialize_twice)
-{
- int ret;
-
- __gnix_hashtable_initialize();
-
- ret = _gnix_ht_init(test_ht, NULL);
- cr_assert(ret == -FI_EINVAL);
- __gnix_hashtable_test_initialized();
-
- __gnix_hashtable_destroy();
-}
-
-Test(gnix_hashtable_basic, err_invalid_initial_size_0)
-{
- int ret;
- gnix_hashtable_attr_t attr;
-
- memcpy(&attr, &default_attr, sizeof(gnix_hashtable_attr_t));
-
- attr.ht_initial_size = 0;
-
- ret = _gnix_ht_init(test_ht, &attr);
- cr_assert(ret == -FI_EINVAL);
- __gnix_hashtable_test_uninitialized();
-}
-
-Test(gnix_hashtable_basic, err_invalid_initial_size_gt_max)
-{
- int ret;
- gnix_hashtable_attr_t attr;
-
- memcpy(&attr, &default_attr, sizeof(gnix_hashtable_attr_t));
-
- attr.ht_initial_size = attr.ht_maximum_size * 2;
-
- ret = _gnix_ht_init(test_ht, &attr);
- cr_assert(ret == -FI_EINVAL);
- __gnix_hashtable_test_uninitialized();
-}
-
-Test(gnix_hashtable_basic, err_invalid_max_size)
-{
- int ret;
- gnix_hashtable_attr_t attr;
-
- memcpy(&attr, &default_attr, sizeof(gnix_hashtable_attr_t));
-
- attr.ht_maximum_size = 0;
-
- ret = _gnix_ht_init(test_ht, &attr);
- cr_assert(ret == -FI_EINVAL);
- __gnix_hashtable_test_uninitialized();
-}
-
-Test(gnix_hashtable_basic, err_invalid_increase_step_all)
-{
- int ret;
- gnix_hashtable_attr_t attr;
-
- memcpy(&attr, &default_attr, sizeof(gnix_hashtable_attr_t));
-
- attr.ht_increase_step = 0;
-
- ret = _gnix_ht_init(test_ht, &attr);
- cr_assert(ret == -FI_EINVAL);
- __gnix_hashtable_test_uninitialized();
-}
-
-Test(gnix_hashtable_basic, err_invalid_increase_step_mult)
-{
- int ret;
- gnix_hashtable_attr_t attr;
-
- memcpy(&attr, &default_attr, sizeof(gnix_hashtable_attr_t));
-
- attr.ht_increase_step = 1;
- attr.ht_increase_type = GNIX_HT_INCREASE_MULT;
-
- ret = _gnix_ht_init(test_ht, &attr);
- cr_assert(ret == -FI_EINVAL);
- __gnix_hashtable_test_uninitialized();
-}
-
-Test(gnix_hashtable_basic, err_invalid_increase_type)
-{
- int ret;
- gnix_hashtable_attr_t attr;
-
- memcpy(&attr, &default_attr, sizeof(gnix_hashtable_attr_t));
-
- attr.ht_increase_type = -1;
-
- ret = _gnix_ht_init(test_ht, &attr);
- cr_assert(ret == -FI_EINVAL);
- __gnix_hashtable_test_uninitialized();
-}
-
-Test(gnix_hashtable_basic, err_invalid_collision)
-{
- int ret;
- gnix_hashtable_attr_t attr;
-
- memcpy(&attr, &default_attr, sizeof(gnix_hashtable_attr_t));
-
- attr.ht_collision_thresh = 0;
-
- ret = _gnix_ht_init(test_ht, &attr);
- cr_assert(ret == -FI_EINVAL);
- __gnix_hashtable_test_uninitialized();
-}
-
-Test(gnix_hashtable_basic, err_destroy_uninitialized)
-{
- int ret;
-
- ret = _gnix_ht_destroy(test_ht);
- cr_assert(ret == -FI_EINVAL);
-
- __gnix_hashtable_test_uninitialized();
-}
-
-Test(gnix_hashtable_basic, destroy)
-{
- __gnix_hashtable_initialize();
-
- __gnix_hashtable_destroy();
-}
-
-Test(gnix_hashtable_basic, destroy_twice)
-{
- int ret;
-
- __gnix_hashtable_initialize();
-
- __gnix_hashtable_destroy();
-
- ret = _gnix_ht_destroy(test_ht);
- cr_assert(ret == -FI_EINVAL);
- __gnix_hashtable_test_destroyed_clean();
-}
-
-Test(gnix_hashtable_advanced, insert_1)
-{
- int ret;
-
- ret = _gnix_ht_insert(test_ht, simple_element->key, simple_element);
- cr_assert(ret == 0);
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 1);
-}
-
-Test(gnix_hashtable_advanced, insert_duplicate)
-{
- int ret;
-
- ret = _gnix_ht_insert(test_ht, simple_element->key, simple_element);
- cr_assert(ret == 0);
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 1);
-
- ret = _gnix_ht_insert(test_ht, simple_element->key, simple_element);
- cr_assert(ret == -FI_ENOSPC);
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 1);
-}
-
-Test(gnix_hashtable_advanced, insert_1_remove_1)
-{
- int ret;
-
- srand(time(NULL));
-
- ret = _gnix_ht_insert(test_ht, simple_element->key, simple_element);
- cr_assert(ret == 0);
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 1);
-
- ret = _gnix_ht_remove(test_ht, simple_element->key);
- cr_assert(ret == 0);
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 0);
-}
-
-
-Test(gnix_hashtable_advanced, insert_1024)
-{
- int ret, i;
-
- gnix_test_element_t test_elements[1024];
-
- srand(time(NULL));
-
- for (i = 0; i < 1024; ++i) {
- test_elements[i].key = rand();
- test_elements[i].val = rand() % (1024 * 1024);
- test_elements[i].magic = __GNIX_MAGIC_VALUE;
- }
-
- for (i = 0; i < 1024; ++i) {
- ret = _gnix_ht_insert(test_ht,
- test_elements[i].key, &test_elements[i]);
- cr_assert(ret == 0);
- cr_assert(atomic_get(&test_ht->ht_elements) == (i + 1));
- }
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 1024);
-}
-
-
-Test(gnix_hashtable_advanced, insert_1024_remove_1024)
-{
- int ret, i;
-
- gnix_test_element_t test_elements[1024];
- gnix_test_element_t *item;
-
- srand(time(NULL));
-
- for (i = 0; i < 1024; ++i) {
- item = &test_elements[i];
- item->key = i;
- item->val = rand() % (1024 * 1024);
- item->magic = __GNIX_MAGIC_VALUE;
- }
-
- for (i = 0; i < 1024; ++i) {
- item = &test_elements[i];
- ret = _gnix_ht_insert(test_ht,
- item->key, item);
- cr_assert(ret == 0);
- cr_assert(atomic_get(&test_ht->ht_elements) == (i + 1));
- }
-
- for (i = 1023; i >= 0; --i) {
- item = &test_elements[i];
- cr_assert(i == item->key);
-
- ret = _gnix_ht_remove(test_ht,
- item->key);
- cr_assert(ret == 0);
- cr_assert(atomic_get(&test_ht->ht_elements) == i);
- }
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 0);
-}
-
-Test(gnix_hashtable_advanced, insert_2048_remove_all_resize_down)
-{
- int ret, i;
- int nelem = 2048;
- gnix_test_element_t test_elements[2048];
- gnix_test_element_t *item;
-
- srand(time(NULL));
-
- for (i = 0; i < nelem; ++i) {
- item = &test_elements[i];
- item->key = i;
- item->val = rand() % (1024 * 1024);
- item->magic = __GNIX_MAGIC_VALUE;
- }
-
- for (i = 0; i < nelem; ++i) {
- item = &test_elements[i];
- ret = _gnix_ht_insert(test_ht,
- item->key, item);
- cr_assert(ret == 0);
- cr_assert(atomic_get(&test_ht->ht_elements) == (i + 1));
- }
-
- cr_assert(test_ht->ht_size > test_ht->ht_attr.ht_initial_size);
-
- for (i = nelem - 1; i >= 0; --i) {
- item = &test_elements[i];
- cr_assert(i == item->key);
-
- ret = _gnix_ht_remove(test_ht,
- item->key);
- cr_assert(ret == 0);
- cr_assert(atomic_get(&test_ht->ht_elements) == i);
- }
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 0);
- /* on default settings, the hash table should resize to initial on
- * removal of all elements
- */
- cr_assert(test_ht->ht_size == test_ht->ht_attr.ht_initial_size);
-}
-
-
-Test(gnix_hashtable_advanced, insert_1_lookup_pass)
-{
- int ret;
- gnix_test_element_t *found = NULL;
-
- ret = _gnix_ht_insert(test_ht,
- simple_element->key, simple_element);
- cr_assert(ret == 0);
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 1);
-
- found = _gnix_ht_lookup(test_ht, simple_element->key);
- cr_assert(found == simple_element);
- cr_assert(found->magic == __GNIX_MAGIC_VALUE);
-}
-
-Test(gnix_hashtable_advanced, insert_1_lookup_fail)
-{
- int ret;
- gnix_test_element_t *found = NULL;
-
- ret = _gnix_ht_insert(test_ht,
- simple_element->key, simple_element);
- cr_assert(ret == 0);
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 1);
-
- found = _gnix_ht_lookup(test_ht, simple_element->key - 1);
- cr_assert(found != simple_element);
- cr_assert(found == NULL);
-}
-
-Test(gnix_hashtable_advanced, insert_1024_lookup_all)
-{
- int ret, i;
- gnix_test_element_t test_elements[1024];
- gnix_test_element_t *item;
- gnix_test_element_t *found = NULL;
-
- srand(time(NULL));
-
- for (i = 0; i < 1024; ++i) {
- item = &test_elements[i];
-
- item->key = i;
- item->val = rand() % (1024 * 1024);
- item->magic = __GNIX_MAGIC_VALUE;
- }
-
- for (i = 0; i < 1024; ++i) {
- item = &test_elements[i];
-
- ret = _gnix_ht_insert(test_ht,
- item->key, item);
- cr_assert(ret == 0);
- cr_assert(atomic_get(&test_ht->ht_elements) == (i + 1));
- }
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 1024);
-
- for (i = 0; i < 1024; ++i) {
- found = _gnix_ht_lookup(test_ht, test_elements[i].key);
- cr_assert(found != NULL);
- cr_assert(found == &test_elements[i]);
- cr_assert(found->magic == __GNIX_MAGIC_VALUE);
- }
-}
-
-Test(gnix_hashtable_advanced, insert_1024_lookup_random)
-{
- int ret, i;
- gnix_test_element_t test_elements[1024];
- gnix_test_element_t *found = NULL, *to_find = NULL;
- gnix_test_element_t *item;
-
- srand(time(NULL));
-
- for (i = 0; i < 1024; ++i) {
- item = &test_elements[i];
-
- item->key = i;
- item->val = rand() % (1024 * 1024);
- item->magic = __GNIX_MAGIC_VALUE;
- }
-
- for (i = 0; i < 1024; ++i) {
- item = &test_elements[i];
-
- ret = _gnix_ht_insert(test_ht,
- item->key, item);
- cr_assert(ret == 0);
- cr_assert(atomic_get(&test_ht->ht_elements) == (i + 1));
- }
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 1024);
-
- for (i = 0; i < 1024; ++i) {
- to_find = &test_elements[rand() % 1024];
- found = _gnix_ht_lookup(test_ht, to_find->key);
- cr_assert(found != NULL);
- cr_assert(found == to_find);
- cr_assert(found->magic == __GNIX_MAGIC_VALUE);
- }
-}
-
-Test(gnix_hashtable_advanced, insert_8K_lookup_128K_random)
-{
- int ret, i, index;
- gnix_test_element_t *test_elements;
- gnix_test_element_t *found = NULL, *to_find = NULL;
- gnix_test_element_t *item;
- gnix_bitmap_t allocated = {0};
- int test_size = 8 * 1024;
- int bitmap_size = 64 * test_size;
- int lookups = 128 * 1024;
-
- test_elements = calloc(test_size, sizeof(gnix_test_element_t));
- cr_assert(test_elements != NULL);
-
- ret = _gnix_alloc_bitmap(&allocated, bitmap_size);
- cr_assert(ret == 0);
-
- srand(time(NULL));
-
- for (i = 0; i < test_size; ++i) {
- do {
- index = rand() % bitmap_size;
- } while (_gnix_test_and_set_bit(&allocated, index));
-
- item = &test_elements[i];
-
- item->key = index;
- item->val = rand() % lookups;
- item->magic = __GNIX_MAGIC_VALUE;
- }
-
- for (i = 0; i < test_size; ++i) {
- item = &test_elements[i];
-
- ret = _gnix_ht_insert(test_ht,
- item->key, item);
- cr_assert(ret == 0);
- cr_assert(atomic_get(&test_ht->ht_elements) == (i + 1));
- }
-
- cr_assert(atomic_get(&test_ht->ht_elements) == test_size);
-
- for (i = 0; i < lookups; ++i) {
- to_find = &test_elements[rand() % test_size];
- found = _gnix_ht_lookup(test_ht, to_find->key);
- cr_assert(found != NULL);
- cr_assert(found == to_find);
- cr_assert(found->magic == __GNIX_MAGIC_VALUE);
- }
-
- ret = _gnix_free_bitmap(&allocated);
- cr_expect(ret == 0);
-
- free(test_elements);
-}
-
-Test(gnix_hashtable_advanced, iterate)
-{
- int ret, i;
-
- gnix_test_element_t test_elements[1024];
- gnix_test_element_t *item;
- char test_elements_found[1024] = {0};
-
- srand(time(NULL));
-
- for (i = 0; i < 1024; ++i) {
- item = &test_elements[i];
- item->key = i;
- item->val = rand() % (1024 * 1024);
- item->magic = __GNIX_MAGIC_VALUE;
- }
-
- for (i = 0; i < 1024; ++i) {
- item = &test_elements[i];
- ret = _gnix_ht_insert(test_ht,
- item->key, item);
- cr_assert(ret == 0);
- cr_assert(atomic_get(&test_ht->ht_elements) == (i + 1));
- }
-
- {
- GNIX_HASHTABLE_ITERATOR(test_ht, iter);
-
- for (i = 0; i < 1024; ++i) {
- item = (gnix_test_element_t *)
- _gnix_ht_iterator_next(&iter);
- cr_assert(item);
- cr_assert(!test_elements_found[item->key]);
- test_elements_found[item->key] = 1;
- }
- }
-
- for (i = 1023; i >= 0; --i) {
- item = &test_elements[i];
- cr_assert(i == item->key);
-
- ret = _gnix_ht_remove(test_ht,
- item->key);
- cr_assert(ret == 0);
- cr_assert(atomic_get(&test_ht->ht_elements) == i);
- }
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 0);
-}
-
-Test(gnix_hashtable_locked, iterate)
-{
- int ret, i;
-
- gnix_test_element_t test_elements[1024];
- gnix_test_element_t *item;
- char test_elements_found[1024] = {0};
-
- srand(time(NULL));
-
- for (i = 0; i < 1024; ++i) {
- item = &test_elements[i];
- item->key = i;
- item->val = rand() % (1024 * 1024);
- item->magic = __GNIX_MAGIC_VALUE;
- }
-
- for (i = 0; i < 1024; ++i) {
- item = &test_elements[i];
- ret = _gnix_ht_insert(test_ht,
- item->key, item);
- cr_assert(ret == 0);
- cr_assert(atomic_get(&test_ht->ht_elements) == (i + 1));
- }
-
- {
- GNIX_HASHTABLE_ITERATOR(test_ht, iter);
-
- for (i = 0; i < 1024; ++i) {
- item = (gnix_test_element_t *)
- _gnix_ht_iterator_next(&iter);
- cr_assert(item);
- cr_assert(!test_elements_found[item->key]);
- test_elements_found[item->key] = 1;
- }
- }
-
- for (i = 1023; i >= 0; --i) {
- item = &test_elements[i];
- cr_assert(i == item->key);
-
- ret = _gnix_ht_remove(test_ht,
- item->key);
- cr_assert(ret == 0);
- cr_assert(atomic_get(&test_ht->ht_elements) == i);
- }
-
- cr_assert(atomic_get(&test_ht->ht_elements) == 0);
-}
diff --git a/prov/gni/test/mr.c b/prov/gni/test/mr.c
deleted file mode 100644
index a080b8a..0000000
--- a/prov/gni/test/mr.c
+++ /dev/null
@@ -1,933 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <stdint.h>
-#include <sys/time.h>
-
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include "gnix_cq.h"
-#include "gnix.h"
-#include "common.h"
-
-#include <criterion/criterion.h>
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom;
-static struct fid_ep *ep;
-static struct fid_mr *mr;
-static struct fi_info *hints;
-static struct fi_info *fi;
-static struct fi_cq_attr cq_attr;
-
-#define __BUF_LEN 4096
-static unsigned char *buf;
-static int buf_len = __BUF_LEN * sizeof(unsigned char);
-static struct gnix_fid_domain *domain;
-static gnix_mr_cache_t *cache;
-static int regions;
-
-uint64_t default_access = (FI_REMOTE_READ | FI_REMOTE_WRITE
- | FI_READ | FI_WRITE);
-
-uint64_t default_flags = 0;
-uint64_t default_req_key = 0;
-uint64_t default_offset = 0;
-
-struct timeval s1, s2;
-
-static void mr_setup(void)
-{
- int ret = 0;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = 4;
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- ret = fi_domain(fab, fi, &dom, NULL);
- cr_assert(!ret, "fi_domain");
-
- ret = fi_endpoint(dom, fi, &ep, NULL);
- cr_assert(!ret, "fi_endpoint");
-
- cq_attr.wait_obj = FI_WAIT_NONE;
-
- buf = calloc(__BUF_LEN, sizeof(unsigned char));
- cr_assert(buf, "buffer allocation");
-
- domain = container_of(dom, struct gnix_fid_domain, domain_fid.fid);
- regions = 1024;
-}
-
-static void mr_teardown(void)
-{
- int ret = 0;
-
- ret = fi_close(&ep->fid);
- cr_assert(!ret, "failure in closing ep.");
- ret = fi_close(&dom->fid);
- cr_assert(!ret, "failure in closing domain.");
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-
- domain = NULL;
- cache = NULL;
-
- free(buf);
-}
-
-TestSuite(memory_registration_bare, .init = mr_setup, .fini = mr_teardown);
-
-TestSuite(memory_registration_cache, .init = mr_setup, .fini = mr_teardown);
-
-TestSuite(perf_memory_registration, .init = mr_setup, .fini = mr_teardown,
- .disabled = true);
-
-/* Test simple init, register and deregister */
-Test(memory_registration_bare, basic_init)
-{
- int ret;
-
- ret = fi_mr_reg(dom, (void *) buf, buf_len, default_access,
- default_offset, default_req_key,
- default_flags, &mr, NULL);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_close(&mr->fid);
- cr_assert(ret == FI_SUCCESS);
-}
-
-/* Test invalid flags to fi_mr_reg */
-Test(memory_registration_bare, invalid_flags)
-{
- int ret;
-
- ret = fi_mr_reg(dom, (void *) buf, buf_len, default_access,
- default_offset, default_req_key,
- ~0, &mr, NULL);
- cr_assert(ret == -FI_EBADFLAGS);
-}
-
-/* Test invalid access param to fi_mr_reg */
-Test(memory_registration_bare, invalid_access)
-{
- int ret;
-
- ret = fi_mr_reg(dom, (void *) buf, buf_len, 0,
- default_offset, default_req_key,
- default_flags, &mr, NULL);
- cr_assert(ret == -FI_EINVAL);
-}
-
-/* Test invalid offset param to fi_mr_reg */
-Test(memory_registration_bare, invalid_offset)
-{
- int ret;
-
- ret = fi_mr_reg(dom, (void *) buf, buf_len, default_access,
- ~0, default_req_key, default_flags,
- &mr, NULL);
- cr_assert(ret == -FI_EINVAL);
-}
-
-/* Test invalid buf param to fi_mr_reg */
-Test(memory_registration_bare, invalid_buf)
-{
- int ret;
-
- ret = fi_mr_reg(dom, NULL, buf_len, default_access,
- default_offset, default_req_key, default_flags,
- &mr, NULL);
- cr_assert(ret == -FI_EINVAL);
-}
-
-/* Test invalid mr_o param to fi_mr_reg */
-Test(memory_registration_bare, invalid_mr_ptr)
-{
- int ret;
-
- ret = fi_mr_reg(dom, (void *) buf, buf_len, default_access,
- default_offset, default_req_key, default_flags,
- NULL, NULL);
- cr_assert(ret == -FI_EINVAL);
-}
-
-/* Test invalid fid param to fi_mr_reg */
-Test(memory_registration_bare, invalid_fid_class)
-{
- int ret;
- size_t old_class = dom->fid.fclass;
-
- dom->fid.fclass = FI_CLASS_UNSPEC;
-
- ret = fi_mr_reg(dom, (void *) buf, buf_len, default_access,
- default_offset, default_req_key, default_flags,
- &mr, NULL);
- cr_assert(ret == -FI_EINVAL);
-
- /* restore old fclass for teardown */
- dom->fid.fclass = old_class;
-}
-
-/* Test simple cache initialization */
-Test(memory_registration_cache, basic_init)
-{
- int ret;
-
- ret = fi_mr_reg(dom, (void *) buf, buf_len, default_access,
- default_offset, default_req_key,
- default_flags, &mr, NULL);
- cr_assert(ret == FI_SUCCESS);
-
- cache = domain->mr_cache;
- cr_assert(cache->state == GNIX_MRC_STATE_READY);
-
- cr_assert(atomic_get(&cache->inuse.elements) == 1);
- cr_assert(atomic_get(&cache->stale.elements) == 0);
-
- ret = fi_close(&mr->fid);
- cr_assert(ret == FI_SUCCESS);
-
- cr_assert(atomic_get(&cache->inuse.elements) == 0);
- cr_assert(atomic_get(&cache->stale.elements) == 1);
-}
-
-/* Test duplicate registration. Since this is a valid operation, we
- * provide a unique fid_mr but internally, a second reference to the same
- * entry is provided to prevent expensive calls to GNI_MemRegister
- */
-Test(memory_registration_cache, duplicate_registration)
-{
- int ret;
- struct fid_mr *f_mr;
-
- ret = fi_mr_reg(dom, (void *) buf, buf_len, default_access,
- default_offset, default_req_key,
- default_flags, &mr, NULL);
- cr_assert(ret == FI_SUCCESS);
-
- cache = domain->mr_cache;
- cr_assert(cache->state == GNIX_MRC_STATE_READY);
-
- cr_assert(atomic_get(&cache->inuse.elements) == 1);
- cr_assert(atomic_get(&cache->stale.elements) == 0);
-
- ret = fi_mr_reg(dom, (void *) buf, buf_len, default_access,
- default_offset, default_req_key,
- default_flags, &f_mr, NULL);
- cr_assert(ret == FI_SUCCESS);
-
- cr_assert(atomic_get(&cache->inuse.elements) == 1);
- cr_assert(atomic_get(&cache->stale.elements) == 0);
-
- ret = fi_close(&mr->fid);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_close(&f_mr->fid);
- cr_assert(ret == FI_SUCCESS);
-
- cr_assert(atomic_get(&cache->inuse.elements) == 0);
- cr_assert(atomic_get(&cache->stale.elements) == 1);
-}
-
-/* Test registration of 1024 elements, all distinct. Cache element counts
- * should meet expected values
- */
-Test(memory_registration_cache, register_1024_distinct_regions)
-{
- int ret;
- uint64_t **buffers;
- void *buffer;
- struct fid_mr **mr_arr;
- int i;
-
- mr_arr = calloc(regions, sizeof(struct fid_mr *));
- cr_assert(mr_arr);
-
- buffers = calloc(regions, sizeof(uint64_t *));
- cr_assert(buffers, "failed to allocate array of buffers");
-
- buffer = calloc(regions * 4 * __BUF_LEN, sizeof(char));
- cr_assert(buffer);
-
- for (i = 0; i < regions; ++i) {
- buffers[i] = (uint64_t *) (buffer + ((i * 4) * __BUF_LEN));
- }
-
- for (i = 0; i < regions; ++i) {
- ret = fi_mr_reg(dom, (void *) buffers[i], __BUF_LEN,
- default_access, default_offset, default_req_key,
- default_flags, &mr_arr[i], NULL);
- cr_assert(ret == FI_SUCCESS);
- }
-
- cache = domain->mr_cache;
- cr_assert(atomic_get(&cache->inuse.elements) == regions);
- cr_assert(atomic_get(&cache->stale.elements) == 0);
-
- for (i = 0; i < regions; ++i) {
- ret = fi_close(&mr_arr[i]->fid);
- cr_assert(ret == FI_SUCCESS);
- }
-
- free(buffers);
- buffers = NULL;
-
- free(mr_arr);
- mr_arr = NULL;
-
- free(buffer);
- buffer = NULL;
-
- cr_assert(atomic_get(&cache->inuse.elements) == 0);
- cr_assert(atomic_get(&cache->stale.elements) >= 0);
-}
-
-/* Test registration of 1024 registrations backed by the same initial
- * registration. There should only be a single registration in the cache
- */
-Test(memory_registration_cache, register_1024_non_unique_regions)
-{
- int ret;
- char *hugepage;
- struct fid_mr *hugepage_mr;
- char **buffers;
- struct fid_mr **mr_arr;
- int i;
-
- mr_arr = calloc(regions, sizeof(struct fid_mr *));
- cr_assert(mr_arr);
-
- buffers = calloc(regions, sizeof(uint64_t *));
- cr_assert(buffers, "failed to allocate array of buffers");
-
- hugepage = calloc(regions * regions, sizeof(char));
- cr_assert(hugepage);
-
- for (i = 0; i < regions; ++i) {
- buffers[i] = &hugepage[i * regions];
- cr_assert(buffers[i]);
- }
-
- ret = fi_mr_reg(dom, (void *) hugepage,
- regions * regions * sizeof(char),
- default_access, default_offset, default_req_key,
- default_flags, &hugepage_mr, NULL);
- cr_assert(ret == FI_SUCCESS);
-
- for (i = 0; i < regions; ++i) {
- ret = fi_mr_reg(dom, (void *) buffers[i], regions,
- default_access, default_offset, default_req_key,
- default_flags, &mr_arr[i], NULL);
- cr_assert(ret == FI_SUCCESS);
- }
-
- cache = domain->mr_cache;
- cr_assert(atomic_get(&cache->inuse.elements) == 1);
- cr_assert(atomic_get(&cache->stale.elements) == 0);
-
- for (i = 0; i < regions; ++i) {
- ret = fi_close(&mr_arr[i]->fid);
- cr_assert(ret == FI_SUCCESS);
- }
-
- ret = fi_close(&hugepage_mr->fid);
- cr_assert(ret == FI_SUCCESS);
-
- free(hugepage);
- hugepage = NULL;
-
- free(buffers);
- buffers = NULL;
-
- free(mr_arr);
- mr_arr = NULL;
-
- cr_assert(atomic_get(&cache->inuse.elements) == 0);
- cr_assert(atomic_get(&cache->stale.elements) > 0);
-}
-
-/* Test registration of 128 regions that will be cycled in and out of the
- * inuse and stale trees. inuse + stale should never exceed 128
- */
-Test(memory_registration_cache, cyclic_register_128_distinct_regions)
-{
- int ret;
- char **buffers;
- char *hugepage;
- struct fid_mr **mr_arr;
- int i;
- int buf_size = __BUF_LEN * sizeof(char);
- int lazy_limit;
-
- regions = 128;
- mr_arr = calloc(regions, sizeof(struct fid_mr *));
- cr_assert(mr_arr);
-
- buffers = calloc(regions, sizeof(char *));
- cr_assert(buffers, "failed to allocate array of buffers");
-
- hugepage = calloc(regions * 4 * __BUF_LEN, sizeof(char));
- cr_assert(hugepage);
-
- for (i = 0; i < regions; ++i) {
- buffers[i] = (char *) (hugepage + ((i * 4) * __BUF_LEN));
- }
-
- /* create the initial memory registrations */
- for (i = 0; i < regions; ++i) {
- ret = fi_mr_reg(dom, (void *) buffers[i], buf_size,
- default_access, default_offset, default_req_key,
- default_flags, &mr_arr[i], NULL);
- cr_assert(ret == FI_SUCCESS);
- }
-
- /* all registrations should now be 'in-use' */
- cache = domain->mr_cache;
- lazy_limit = MIN(cache->attr.hard_stale_limit, regions);
-
- cr_assert(atomic_get(&cache->inuse.elements) == regions);
- cr_assert(atomic_get(&cache->stale.elements) == 0);
-
- for (i = 0; i < regions; ++i) {
- ret = fi_close(&mr_arr[i]->fid);
- cr_assert(ret == FI_SUCCESS);
- }
-
- /* all registrations should now be 'stale' */
- cr_assert(atomic_get(&cache->inuse.elements) == 0);
- cr_assert(atomic_get(&cache->stale.elements) == lazy_limit);
-
- for (i = 0; i < regions; ++i) {
- ret = fi_mr_reg(dom, (void *) buffers[i], buf_size,
- default_access, default_offset, default_req_key,
- default_flags, &mr_arr[i], NULL);
- cr_assert(ret == FI_SUCCESS);
-
- cr_assert(atomic_get(&cache->inuse.elements) == i + 1);
- cr_assert(atomic_get(&cache->stale.elements) == regions - (i + 1));
- }
-
- /* all registrations should have been moved from 'stale' to 'in-use' */
- cr_assert(atomic_get(&cache->inuse.elements) == regions);
- cr_assert(atomic_get(&cache->stale.elements) == 0);
-
- for (i = 0; i < regions; ++i) {
- ret = fi_close(&mr_arr[i]->fid);
- cr_assert(ret == FI_SUCCESS);
- }
-
- /* all registrations should now be 'stale' */
- cr_assert(atomic_get(&cache->inuse.elements) == 0);
- cr_assert(atomic_get(&cache->stale.elements) == regions);
-
- free(buffers);
- buffers = NULL;
-
- free(mr_arr);
- mr_arr = NULL;
-
- free(hugepage);
- hugepage = NULL;
-}
-
-Test(memory_registration_cache, lru_evict_first_entry)
-{
- int ret;
- char **buffers;
- char *hugepage;
- struct fid_mr **mr_arr;
- int i;
- int buf_size = __BUF_LEN * sizeof(char);
-
- regions = domain->mr_cache_attr.hard_stale_limit << 1;
- cr_assert(regions > 0);
- mr_arr = calloc(regions, sizeof(struct fid_mr *));
- cr_assert(mr_arr);
-
- buffers = calloc(regions, sizeof(char *));
- cr_assert(buffers, "failed to allocate array of buffers");
-
- hugepage = calloc(regions * 4 * __BUF_LEN, sizeof(char));
- cr_assert(hugepage);
-
- for (i = 0; i < regions; ++i) {
- buffers[i] = (char *) (hugepage + ((i * 4) * __BUF_LEN));
- }
-
- /* create the initial memory registrations */
- for (i = 0; i < regions; ++i) {
- ret = fi_mr_reg(dom, (void *) buffers[i], buf_size,
- default_access, default_offset, default_req_key,
- default_flags, &mr_arr[i], NULL);
- cr_assert(ret == FI_SUCCESS);
- }
-
- /* all registrations should now be 'in-use' */
- cache = domain->mr_cache;
- cr_assert(atomic_get(&cache->inuse.elements) == regions);
- cr_assert(atomic_get(&cache->stale.elements) == 0);
-
- /* deregister cache->stale_reg_limit + 1 to test if the first region was
- * deregistered
- */
- for (i = 0; i < cache->attr.hard_stale_limit + 1; ++i) {
- ret = fi_close(&mr_arr[i]->fid);
- cr_assert(ret == FI_SUCCESS);
- }
-
- for (i = 1; i < MIN(cache->attr.hard_stale_limit + 1, regions); ++i) {
- ret = fi_mr_reg(dom, (void *) buffers[i], buf_size,
- default_access, default_offset, default_req_key,
- default_flags, &mr_arr[i], NULL);
- cr_assert(ret == FI_SUCCESS);
- }
-
- /* all registrations should now be 'stale' */
- cr_assert(atomic_get(&cache->inuse.elements) == regions - 1);
- cr_assert(atomic_get(&cache->stale.elements) == 0);
-
- for (i = 1; i < regions; ++i) {
- ret = fi_close(&mr_arr[i]->fid);
- cr_assert(ret == FI_SUCCESS);
- }
-
- /* all registrations should now be 'stale' */
- cr_assert(atomic_get(&cache->inuse.elements) == 0);
- cr_assert(atomic_get(&cache->stale.elements) == MIN(regions - 1,
- cache->attr.hard_stale_limit));
-
- free(buffers);
- buffers = NULL;
-
- free(mr_arr);
- mr_arr = NULL;
-
- free(hugepage);
- hugepage = NULL;
-}
-
-Test(memory_registration_cache, lru_evict_middle_entry)
-{
- int ret;
- char **buffers;
- char *hugepage;
- struct fid_mr **mr_arr;
- int i, limit;
- int buf_size = __BUF_LEN * sizeof(char);
-
- regions = domain->mr_cache_attr.hard_stale_limit << 1;
- cr_assert(regions > 0);
- mr_arr = calloc(regions, sizeof(struct fid_mr *));
- cr_assert(mr_arr);
-
- buffers = calloc(regions, sizeof(char *));
- cr_assert(buffers, "failed to allocate array of buffers");
-
- hugepage = calloc(regions * 4 * __BUF_LEN, sizeof(char));
- cr_assert(hugepage);
-
- for (i = 0; i < regions; ++i) {
- buffers[i] = (char *) (hugepage + ((i * 4) * __BUF_LEN));
- }
-
- /* create the initial memory registrations */
- for (i = 0; i < regions; ++i) {
- ret = fi_mr_reg(dom, (void *) buffers[i], buf_size,
- default_access, default_offset, default_req_key,
- default_flags, &mr_arr[i], NULL);
- cr_assert(ret == FI_SUCCESS);
- }
-
- /* all registrations should now be 'in-use' */
- cache = domain->mr_cache;
- limit = cache->attr.hard_stale_limit;
- cr_assert(limit < regions);
-
- cr_assert(atomic_get(&cache->inuse.elements) == regions);
- cr_assert(atomic_get(&cache->stale.elements) == 0);
-
- /* deregister cache->stale_reg_limit + 1 to test if the first region was
- * deregistered
- */
- for (i = 0; i < limit + 1; ++i) {
- ret = fi_close(&mr_arr[i]->fid);
- cr_assert(ret == FI_SUCCESS);
- }
-
- cr_assert(atomic_get(&cache->inuse.elements) == (regions - (limit + 1)));
- cr_assert(atomic_get(&cache->stale.elements) == limit);
-
- /* re-register this region in the middle to test removal */
- i = (regions >> 2);
- ret = fi_mr_reg(dom, (void *) buffers[i], buf_size,
- default_access, default_offset, default_req_key,
- default_flags, &mr_arr[i], NULL);
- cr_assert(ret == FI_SUCCESS);
-
- cr_assert(atomic_get(&cache->inuse.elements) == (regions - limit));
- cr_assert(atomic_get(&cache->stale.elements) == (limit - 1));
-
- for (i = limit + 1; i < regions; ++i) {
- ret = fi_close(&mr_arr[i]->fid);
- cr_assert(ret == FI_SUCCESS);
- }
-
- cr_assert(atomic_get(&cache->inuse.elements) == 1);
- cr_assert(atomic_get(&cache->stale.elements) == limit);
-
- i = (regions >> 2);
- ret = fi_close(&mr_arr[i]->fid);
- cr_assert(ret == FI_SUCCESS);
-
- /* all registrations should now be 'stale' */
- cr_assert(atomic_get(&cache->inuse.elements) == 0);
- cr_assert(atomic_get(&cache->stale.elements) == limit);
-
- free(buffers);
- buffers = NULL;
-
- free(mr_arr);
- mr_arr = NULL;
-
- free(hugepage);
- hugepage = NULL;
-}
-
-/* Test repeated registration of a memory region with the same base
- * address, increasing the size each time.. This is an explicit
- * version of what the test rdm_sr::send_autoreg_uncached does under
- * the covers (currently).
- */
-Test(memory_registration_cache, same_addr_incr_size)
-{
- int ret;
- int i;
-
- for (i = 2; i <= buf_len; i *= 2) {
- ret = fi_mr_reg(dom, (void *) buf, i, default_access,
- default_offset, default_req_key,
- default_flags, &mr, NULL);
- cr_assert(ret == FI_SUCCESS);
-
- cache = domain->mr_cache;
- cr_assert(cache->state == GNIX_MRC_STATE_READY);
-
- cr_assert(atomic_get(&cache->inuse.elements) == 1);
- cr_assert(atomic_get(&cache->stale.elements) <= 1);
-
- ret = fi_close(&mr->fid);
- cr_assert(ret == FI_SUCCESS);
-
- cr_assert(atomic_get(&cache->inuse.elements) == 0);
- cr_assert(atomic_get(&cache->stale.elements) == 1);
- }
-}
-
-/* Same as above, except with decreasing sizes */
-Test(memory_registration_cache, same_addr_decr_size)
-{
- int ret;
- int i;
-
- for (i = buf_len; i >= 2; i /= 2) {
- ret = fi_mr_reg(dom, (void *) buf, i, default_access,
- default_offset, default_req_key,
- default_flags, &mr, NULL);
- cr_assert(ret == FI_SUCCESS);
-
- cache = domain->mr_cache;
- cr_assert(cache->state == GNIX_MRC_STATE_READY);
-
- cr_assert(atomic_get(&cache->inuse.elements) == 1);
- cr_assert(atomic_get(&cache->stale.elements) == 0);
-
- ret = fi_close(&mr->fid);
- cr_assert(ret == FI_SUCCESS);
-
- cr_assert(atomic_get(&cache->inuse.elements) == 0);
- cr_assert(atomic_get(&cache->stale.elements) == 1);
- }
-}
-
-Test(perf_memory_registration, repeated_registration)
-{
- int ret, i;
- int region_len = 1 << 24;
- int registrations = 4096 * 16;
- unsigned char *region = calloc(region_len, sizeof(unsigned char));
- struct fid_mr **f_mr;
- int reg_time, dereg_time, seconds;
-
- cr_assert(region != NULL);
-
- f_mr = calloc(registrations, sizeof(*f_mr));
- cr_assert(f_mr != NULL);
-
- ret = fi_mr_reg(dom, (void *) region,
- region_len, default_access,
- default_offset, default_req_key,
- default_flags, &mr, NULL);
-
- cache = domain->mr_cache;
-
- gettimeofday(&s1, 0);
- for (i = 0; i < registrations; i++) {
- ret = fi_mr_reg(dom, (void *) region,
- region_len, default_access,
- default_offset, default_req_key,
- default_flags, &f_mr[i], NULL);
- cr_assert(ret == FI_SUCCESS);
- }
- gettimeofday(&s2, 0);
-
- cr_assert(atomic_get(&cache->inuse.elements) == 1);
-
-
- calculate_time_difference(&s1, &s2, &seconds, ®_time);
- reg_time += seconds * 1000000;
-
- gettimeofday(&s1, 0);
- for (i = 0; i < registrations; i++) {
- ret = fi_close(&f_mr[i]->fid);
- cr_assert(ret == FI_SUCCESS);
- }
- gettimeofday(&s2, 0);
-
- calculate_time_difference(&s1, &s2, &seconds, &dereg_time);
- dereg_time += seconds * 1000000;
-
- ret = fi_close(&mr->fid);
- cr_assert(ret == FI_SUCCESS);
-
- fprintf(stderr, "best(repeated) case: reg_time=%.3f usec dereg_time=%.3f usec\n",
- reg_time / (registrations * 1.0),
- dereg_time / (registrations * 1.0));
-
- free(region);
-}
-
-Test(perf_memory_registration, single_large_registration)
-{
- int ret, i;
- int region_len = 1 << 24;
- int registration_width = 1 << 12;
- int registrations = region_len / registration_width;
- unsigned char *region = calloc(region_len, sizeof(unsigned char));
- struct fid_mr **f_mr;
- int reg_time, dereg_time, seconds;
-
- cr_assert(region != NULL);
-
- f_mr = calloc(registrations, sizeof(*f_mr));
- cr_assert(f_mr != NULL);
-
- ret = fi_mr_reg(dom, (void *) region,
- region_len, default_access,
- default_offset, default_req_key,
- default_flags, &mr, NULL);
-
- cache = domain->mr_cache;
-
- gettimeofday(&s1, 0);
- for (i = 0; i < registrations; i++) {
- ret = fi_mr_reg(dom, (void *) region + (registration_width * i),
- registration_width, default_access,
- default_offset, default_req_key,
- default_flags, &f_mr[i], NULL);
- cr_assert(ret == FI_SUCCESS);
- }
- gettimeofday(&s2, 0);
-
- calculate_time_difference(&s1, &s2, &seconds, ®_time);
- reg_time += seconds * 1000000;
-
- gettimeofday(&s1, 0);
- for (i = 0; i < registrations; i++) {
- ret = fi_close(&f_mr[i]->fid);
- cr_assert(ret == FI_SUCCESS);
- }
- gettimeofday(&s2, 0);
-
- calculate_time_difference(&s1, &s2, &seconds, &dereg_time);
- dereg_time += seconds * 1000000;
-
- ret = fi_close(&mr->fid);
- cr_assert(ret == FI_SUCCESS);
-
- fprintf(stderr, "best(overlap) case: reg_time=%.3f usec dereg_time=%.3f usec\n",
- reg_time / (registrations * 1.0),
- dereg_time / (registrations * 1.0));
-
- free(region);
-}
-
-Test(perf_memory_registration, random_analysis)
-{
- int ret, i;
- int region_len = 1 << 24;
- int registration_width = 1 << 12;
- int registrations = region_len / registration_width;
- unsigned char *region = calloc(region_len, sizeof(unsigned char));
- struct fid_mr **f_mr;
- int reg_time, dereg_time, seconds;
- void *ptr;
- uint64_t ptr_len;
-
- srand(0xDEADBEEF);
- cr_assert(region != NULL);
-
- f_mr = calloc(registrations, sizeof(*f_mr));
- cr_assert(f_mr != NULL);
-
- /* prep the cache by adding and removing an entry before timing */
- ret = fi_mr_reg(dom, (void *) buf, buf_len, default_access,
- default_offset, default_req_key,
- default_flags, &mr, NULL);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_close(&mr->fid);
- cr_assert(ret == FI_SUCCESS);
-
-
- gettimeofday(&s1, 0);
- for (i = 0; i < registrations; i++) {
- ptr = region + rand() % region_len;
- ptr_len = registration_width;
- if ((uint64_t) (ptr + ptr_len) > (uint64_t) (region + region_len))
- ptr_len = ((uint64_t) region + region_len) - (uint64_t) ptr;
-
- ret = fi_mr_reg(dom, (void *) ptr,
- ptr_len, default_access,
- default_offset, default_req_key,
- default_flags, &f_mr[i], NULL);
- cr_assert(ret == FI_SUCCESS);
- }
- gettimeofday(&s2, 0);
-
- calculate_time_difference(&s1, &s2, &seconds, ®_time);
- reg_time += seconds * 1000000;
-
- gettimeofday(&s1, 0);
- for (i = 0; i < registrations; i++) {
- ret = fi_close(&f_mr[i]->fid);
- cr_assert(ret == FI_SUCCESS);
- }
- gettimeofday(&s2, 0);
-
- calculate_time_difference(&s1, &s2, &seconds, &dereg_time);
- dereg_time += seconds * 1000000;
-
- fprintf(stderr, "random case: reg_time=%.3f usec "
- "dereg_time=%.3f usec\n",
- reg_time / (registrations * 1.0),
- dereg_time / (registrations * 1.0));
-
- free(region);
-}
-
-/*
- * This test exercises the ability of the cache to drop registrations that
- * have been subsumed by other registrations
- */
-Test(memory_registration_cache, regression_615)
-{
- int ret;
- struct fid_mr *f_mr;
- void *buffer = calloc(1 << 19, sizeof(char));
-
- cr_assert(buffer != NULL);
-
- /* set up stale cache */
- ret = fi_mr_reg(dom, (void *) buffer + 0x18000, 0x8000,
- default_access, default_offset, default_req_key,
- default_flags, &f_mr, NULL);
- cr_assert(ret == FI_SUCCESS);
- ret = fi_close(&f_mr->fid);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_mr_reg(dom, (void *) buffer + 0x0, 0x80000,
- default_access, default_offset, default_req_key,
- default_flags, &f_mr, NULL);
- cr_assert(ret == FI_SUCCESS);
- ret = fi_close(&f_mr->fid);
- cr_assert(ret == FI_SUCCESS);
-
- /* set up inuse */
- ret = fi_mr_reg(dom, (void *) buffer + 0x28000, 0x4000,
- default_access, default_offset, default_req_key,
- default_flags, &f_mr, NULL);
- cr_assert(ret == FI_SUCCESS);
-
- cache = domain->mr_cache;
-
- cr_assert(atomic_get(&cache->inuse.elements) == 1);
- cr_assert(atomic_get(&cache->stale.elements) == 0);
-
- ret = fi_close(&f_mr->fid);
- cr_assert(ret == FI_SUCCESS);
-
- free(buffer);
-}
-
-
-
diff --git a/prov/gni/test/nic.c b/prov/gni/test/nic.c
deleted file mode 100644
index 0fe045e..0000000
--- a/prov/gni/test/nic.c
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-
-#include "gnix.h"
-#include "gnix_nic.h"
-
-#include <criterion/criterion.h>
-
-static struct fi_info *hints;
-static struct fi_info *fi;
-static struct fid_fabric *fab;
-static struct fid_domain *dom;
-
-static void setup(void)
-{
- int ret;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- ret = fi_domain(fab, fi, &dom, NULL);
- cr_assert(!ret, "fi_domain");
-
-}
-
-static void teardown(void)
-{
- int ret;
-
- ret = fi_close(&dom->fid);
- cr_assert(!ret, "fi_close domain");
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "fi_close fabric");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-}
-
-TestSuite(nic, .init = setup, .fini = teardown);
-
-Test(nic, alloc_free)
-{
- int i, ret;
- const int num_nics = 79;
- struct gnix_nic *nics[num_nics];
-
- for (i = 0; i < num_nics; i++) {
- ret = gnix_nic_alloc(container_of(dom, struct gnix_fid_domain,
- domain_fid),
- NULL,
- &nics[i]);
- cr_assert_eq(ret, FI_SUCCESS, "Could not allocate nic");
- }
-
- for (i = 0; i < num_nics; i++) {
- ret = _gnix_nic_free(nics[i]);
- cr_assert_eq(ret, FI_SUCCESS, "Could not free nic");
- }
-}
-
diff --git a/prov/gni/test/queue.c b/prov/gni/test/queue.c
deleted file mode 100644
index 8742515..0000000
--- a/prov/gni/test/queue.c
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdlib.h>
-#include <rdma/fabric.h> /* for container_of definition */
-
-#include "gnix_queue.h"
-
-#include <criterion/criterion.h>
-
-struct gnix_queue *queue;
-
-struct int_entry {
- int x;
- struct slist_entry item;
-};
-
-static struct slist_entry *alloc_int_entry(size_t size)
-{
- (void) size;
- struct int_entry *entry = malloc(sizeof(*entry));
-
- if (!entry)
- return NULL;
-
- return &entry->item;
-}
-
-static void free_int_entry(struct slist_entry *list_entry)
-{
- struct int_entry *entry;
-
- entry = container_of(list_entry, struct int_entry, item);
-
- free(entry);
-}
-
-void setup_count_eight(void)
-{
- int ret;
-
- ret = _gnix_queue_create(&queue, alloc_int_entry, free_int_entry,
- 0, 8);
- cr_assert_eq(ret, FI_SUCCESS, "failed to create queue.");
-}
-
-void setup_count_zero(void)
-{
- int ret;
-
- ret = _gnix_queue_create(&queue, alloc_int_entry, free_int_entry,
- 0, 0);
- cr_assert_eq(ret, FI_SUCCESS, "failed to create queue.");
-}
-
-void teardown_queue(void)
-{
- _gnix_queue_destroy(queue);
-}
-
-TestSuite(empty, .init = setup_count_zero, .fini = teardown_queue);
-
-Test(empty, null_read)
-{
- struct slist_entry *list_entry;
-
- list_entry = _gnix_queue_dequeue(queue);
- cr_expect(!list_entry, "non null read on empty queue.");
-
- list_entry = _gnix_queue_dequeue_free(queue);
- cr_expect(!list_entry, "non null read on free list of empty queue.");
-
- list_entry = _gnix_queue_peek(queue);
- cr_expect(!list_entry, "non null peek on empty queue.");
-}
-
-Test(empty, single_write)
-{
- struct slist_entry *list_entry;
- struct int_entry *entry;
-
- /*
- * Write single entry with value 4 to queue.
- */
- list_entry = _gnix_queue_get_free(queue);
- cr_expect(list_entry, "null entry from queue get free function.");
-
- entry = container_of(list_entry, struct int_entry, item);
-
- entry->x = 4;
-
- _gnix_queue_enqueue(queue, &entry->item);
-
- /*
- * Read back entry with value 4.
- */
- list_entry = _gnix_queue_dequeue(queue);
- cr_expect(list_entry, "null entry from queue after enqueue.");
-
- entry = container_of(list_entry, struct int_entry, item);
-
- cr_expect_eq(4, entry->x, "entry does not contain assigned value.");
-
- /*
- * Add to free list.
- */
- _gnix_queue_enqueue_free(queue, &entry->item);
-
- /*
- * Read from now empty queue.
- */
- list_entry = _gnix_queue_dequeue(queue);
- cr_expect(!list_entry, "entry read from empty queue is non-null.");
-
- /*
- * Read from free and make sure it's the same.
- */
- list_entry = _gnix_queue_get_free(queue);
- cr_expect(list_entry,
- "null entry from free queue after adding to free.");
-
- entry = container_of(list_entry, struct int_entry, item);
-
- cr_expect_eq(4, entry->x, "entry does not contain assigned value.");
-
- /*
- * Completely empty list. Shouldn't seg fault on teardown.
- */
- queue->free_item(&entry->item);
-}
-
-TestSuite(eight, .init = setup_count_eight, .fini = teardown_queue);
-
-Test(eight, read_nine)
-{
- struct slist_entry *list_entry;
- struct int_entry *entry;
-
- /*
- * Fill the queue. The value of each will be the counter position.
- */
- for (size_t i = 0; i < 8; i++) {
- list_entry = _gnix_queue_get_free(queue);
- cr_expect(list_entry,
- "null entry from queue get free function.");
-
- entry = container_of(list_entry, struct int_entry, item);
-
- entry->x = i;
-
- _gnix_queue_enqueue(queue, &entry->item);
- }
-
- /*
- * Peek and make sure the top of queue is 0.
- */
- list_entry = _gnix_queue_peek(queue);
- cr_expect(list_entry, "null entry from peek.");
-
- entry = container_of(list_entry, struct int_entry, item);
-
- cr_expect_eq(0, entry->x, "value of peek isn't first added to queue.");
-
- /*
- * Peek again and make sure it's still 0.
- */
- list_entry = _gnix_queue_peek(queue);
- cr_expect(list_entry, "null entry from peek.");
-
- entry = container_of(list_entry, struct int_entry, item);
-
- cr_expect_eq(0, entry->x, "value of peek isn't first added to queue.");
-
- /*
- * Read it back.
- */
- for (size_t i = 0; i < 8; i++) {
- list_entry = _gnix_queue_dequeue(queue);
- cr_expect(list_entry, "null entry from queue dequeue.");
-
- entry = container_of(list_entry, struct int_entry, item);
-
- cr_expect_eq(i, entry->x, "value not same as assigned.");
-
- _gnix_queue_enqueue_free(queue, &entry->item);
- }
-
- /*
- * Read an extra item. Should return null.
- */
- list_entry = _gnix_queue_dequeue(queue);
- cr_expect(!list_entry, "entry from empty queue not null.");
-}
diff --git a/prov/gni/test/rdm_atomic.c b/prov/gni/test/rdm_atomic.c
deleted file mode 100644
index ed34a02..0000000
--- a/prov/gni/test/rdm_atomic.c
+++ /dev/null
@@ -1,4420 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <pthread.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include "gnix_vc.h"
-#include "gnix_cm_nic.h"
-#include "gnix_hashtable.h"
-#include "gnix_atomic.h"
-
-#include <criterion/criterion.h>
-
-#if 1
-#define dbg_printf(...)
-#else
-#define dbg_printf(...) \
- do { \
- printf(__VA_ARGS__); \
- fflush(stdout); \
- } while (0)
-#endif
-
-#define NUMEPS 2
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom[NUMEPS];
-struct fi_gni_ops_domain *gni_domain_ops[NUMEPS];
-static struct fid_ep *ep[NUMEPS];
-static struct fid_av *av[NUMEPS];
-static struct fi_info *hints;
-static struct fi_info *fi;
-void *ep_name[NUMEPS];
-size_t gni_addr[NUMEPS];
-static struct fid_cq *send_cq[NUMEPS];
-static struct fid_cq *recv_cq[NUMEPS];
-static struct fi_cq_attr cq_attr;
-
-#define BUF_SZ (64*1024)
-char *target;
-char *source;
-char *uc_source;
-struct fid_mr *rem_mr[NUMEPS], *loc_mr[NUMEPS];
-uint64_t mr_key[NUMEPS];
-
-static struct fid_cntr *write_cntr[NUMEPS], *read_cntr[NUMEPS];
-static struct fi_cntr_attr cntr_attr = {
- .events = FI_CNTR_EVENTS_COMP,
- .flags = 0
-};
-static uint64_t writes[NUMEPS] = {0}, reads[NUMEPS] = {0},
- write_errs[NUMEPS] = {0}, read_errs[NUMEPS] = {0};
-
-void rdm_atomic_setup(void)
-{
- int ret = 0, i = 0, j = 0;
- struct fi_av_attr attr;
- size_t addrlen = 0;
-
- attr.type = FI_AV_MAP;
- attr.count = NUMEPS;
-
- cq_attr.format = FI_CQ_FORMAT_TAGGED;
- cq_attr.size = 1024;
- cq_attr.wait_obj = 0;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
- hints->domain_attr->cq_data_size = 4;
- hints->mode = ~0;
- hints->fabric_attr->name = strdup("gni");
-
- target = malloc(BUF_SZ);
- assert(target);
-
- source = malloc(BUF_SZ);
- assert(source);
-
- uc_source = malloc(BUF_SZ);
- assert(uc_source);
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- for (; i < NUMEPS; i++) {
- ret = fi_domain(fab, fi, dom + i, NULL);
- cr_assert(!ret, "fi_domain");
-
- ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
- 0, (void **) (gni_domain_ops + i), NULL);
- cr_assert(!ret, "fi_open_ops");
-
- ret = fi_av_open(dom[i], &attr, av + i, NULL);
- cr_assert(!ret, "fi_av_open");
-
- ret = fi_endpoint(dom[i], fi, &ep[i], NULL);
- cr_assert(!ret, "fi_endpoint");
-
- ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
- cr_assert(addrlen > 0);
-
- ep_name[i] = malloc(addrlen);
- cr_assert(ep_name[i] != NULL);
-
- ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
- cr_assert(addrlen > 0);
-
- ret = fi_cq_open(dom[i], &cq_attr, send_cq + i, 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_ep_bind(ep[i], &send_cq[i]->fid, FI_TRANSMIT);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_cq_open(dom[i], &cq_attr, recv_cq + i, 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_ep_bind(ep[i], &recv_cq[i]->fid, FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
- }
-
- for (i = 0; i < NUMEPS; i++) {
- for (j = 0; j < NUMEPS; j++) {
- ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
- 0, NULL);
- cr_assert(ret == 1);
- }
-
- ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_enable(ep[i]);
- cr_assert(!ret, "fi_ep_enable");
-
- ret = fi_mr_reg(dom[i], target, BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, rem_mr + i, &target);
- cr_assert_eq(ret, 0);
-
- ret = fi_mr_reg(dom[i], source, BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, loc_mr + i, &source);
- cr_assert_eq(ret, 0);
-
- mr_key[i] = fi_mr_key(rem_mr[i]);
-
- ret = fi_cntr_open(dom[i], &cntr_attr, write_cntr + i, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[i], &write_cntr[i]->fid, FI_WRITE);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_cntr_open(dom[i], &cntr_attr, read_cntr + i, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[i], &read_cntr[i]->fid, FI_READ);
- cr_assert(!ret, "fi_ep_bind");
- }
-}
-
-void rdm_atomic_teardown(void)
-{
- int ret = 0, i = 0;
-
- for (; i < NUMEPS; i++) {
- ret = fi_close(&read_cntr[i]->fid);
- cr_assert(!ret, "failure in closing read counter.");
-
- ret = fi_close(&write_cntr[i]->fid);
- cr_assert(!ret, "failure in closing write counter.");
-
- ret = fi_close(&loc_mr[i]->fid);
- cr_assert(!ret, "failure in closing av.");
-
- ret = fi_close(&rem_mr[i]->fid);
- cr_assert(!ret, "failure in closing av.");
-
- ret = fi_close(&ep[i]->fid);
- cr_assert(!ret, "failure in closing ep.");
-
- ret = fi_close(&recv_cq[i]->fid);
- cr_assert(!ret, "failure in recv cq.");
-
- ret = fi_close(&send_cq[i]->fid);
- cr_assert(!ret, "failure in send cq.");
-
- ret = fi_close(&av[i]->fid);
- cr_assert(!ret, "failure in closing av.");
-
- ret = fi_close(&dom[i]->fid);
- cr_assert(!ret, "failure in closing domain.");
-
- free(ep_name[i]);
- }
-
- free(target);
- free(source);
- free(uc_source);
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-}
-
-void rdm_atomic_check_tcqe(struct fi_cq_tagged_entry *tcqe, void *ctx,
- uint64_t flags, uint64_t data)
-{
- cr_assert(tcqe->op_context == ctx, "CQE Context mismatch");
- cr_assert(tcqe->flags == flags, "CQE flags mismatch");
-
- if (flags & FI_REMOTE_CQ_DATA) {
- cr_assert(tcqe->data == data, "CQE data invalid");
- } else {
- cr_assert(tcqe->data == 0, "CQE data invalid");
- }
-
- cr_assert(tcqe->len == 0, "CQE length mismatch");
- cr_assert(tcqe->buf == 0, "CQE address mismatch");
- cr_assert(tcqe->tag == 0, "CQE tag invalid");
-}
-
-void rdm_atomic_check_cntrs(uint64_t w[], uint64_t r[], uint64_t w_e[],
- uint64_t r_e[])
-{
- int i = 0;
-
- for (; i < NUMEPS; i++) {
- writes[i] += w[i];
- reads[i] += r[i];
- write_errs[i] += w_e[i];
- read_errs[i] += r_e[i];
-
- cr_assert(fi_cntr_read(write_cntr[i]) == writes[i],
- "Bad write count");
- cr_assert(fi_cntr_read(read_cntr[i]) == reads[i],
- "Bad read count");
- cr_assert(fi_cntr_readerr(write_cntr[i]) == write_errs[i],
- "Bad write err count");
- cr_assert(fi_cntr_readerr(read_cntr[i]) == read_errs[i],
- "Bad read err count");
- }
-}
-
-void rdm_atomic_xfer_for_each_size(void (*xfer)(int len), int slen, int elen)
-{
- int i;
-
- for (i = slen; i <= elen; i *= 2) {
- xfer(i);
- }
-}
-
-void rdm_atomic_err_inject_enable(void)
-{
- int ret, err_count_val = 1, i = 0;
-
- for (; i < NUMEPS; i++) {
- ret = gni_domain_ops[i]->set_val(&dom[i]->fid,
- GNI_ERR_INJECT_COUNT,
- &err_count_val);
- cr_assert(!ret, "setval(GNI_ERR_INJECT_COUNT)");
- }
-}
-
-/*******************************************************************************
- * Test RMA functions
- ******************************************************************************/
-
-TestSuite(rdm_atomic, .init = rdm_atomic_setup, .fini = rdm_atomic_teardown,
- .disabled = false);
-
-#if 1
-#define SOURCE_DATA 0xBBBB0000CCCCULL
-#define TARGET_DATA 0xAAAA0000DDDDULL
-#define SOURCE_DATA_FP 0.83203125
-#define TARGET_DATA_FP 0.83984375
-#else
-/* swapped */
-#define TARGET_DATA 0xB0000000CULL
-#define SOURCE_DATA 0xA0000000DULL
-#define TARGET_DATA_FP 0.83203125
-#define SOURCE_DATA_FP 0.83984375
-#endif
-#define FETCH_SOURCE_DATA 0xACEDACEDULL
-#define DATA_MASK 0xa5a5a5a5a5a5a5a5
-#define U32_MASK 0xFFFFFFFFULL
-
-#define ALL_GNI_DATATYPES_SUPPORTED { 0,0,0,0,1,1,1,1,1,1 }
-#define GNI_DATATYPES_NO_FP_SUPPORTED { 0,0,0,0,1,1,1,1,0,0 }
-#define NO_DATATYPES_SUPPORTED { }
-
-/******************************************************************************
- *
- * Basic atomics
- *
- *****************************************************************************/
-
-int supported_atomic_ops[FI_ATOMIC_OP_LAST][FI_DATATYPE_LAST] = {
- [FI_MIN] = { 0,0,0,0,1,0,1,0,1,1 },
- [FI_MAX] = { 0,0,0,0,1,0,1,0,1,1 },
- [FI_SUM] = { 0,0,0,0,1,1,1,1,1,0 }, /* GNI DP sum is broken */
- [FI_PROD] = NO_DATATYPES_SUPPORTED,
- [FI_LOR] = NO_DATATYPES_SUPPORTED,
- [FI_LAND] = NO_DATATYPES_SUPPORTED,
- [FI_BOR] = GNI_DATATYPES_NO_FP_SUPPORTED,
- [FI_BAND] = GNI_DATATYPES_NO_FP_SUPPORTED,
- [FI_LXOR] = NO_DATATYPES_SUPPORTED,
- [FI_BXOR] = GNI_DATATYPES_NO_FP_SUPPORTED,
- [FI_ATOMIC_READ] = NO_DATATYPES_SUPPORTED,
- [FI_ATOMIC_WRITE] = ALL_GNI_DATATYPES_SUPPORTED,
- [FI_CSWAP] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_NE] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_LE] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_LT] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_GE] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_GT] = NO_DATATYPES_SUPPORTED,
- [FI_MSWAP] = NO_DATATYPES_SUPPORTED,
-};
-
-void do_invalid_atomic(enum fi_datatype dt, enum fi_op op)
-{
- ssize_t sz;
- size_t count;
-
- if (!supported_atomic_ops[op][dt]) {
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], dt, op, target);
-
- cr_assert(sz == -FI_ENOENT);
-
- sz = fi_atomicvalid(ep[0], dt, op, &count);
- cr_assert(sz == -FI_ENOENT, "fi_atomicvalid() succeeded\n");
- } else {
- sz = fi_atomicvalid(ep[0], dt, op, &count);
- cr_assert(!sz, "fi_atomicvalid() failed\n");
- cr_assert(count == 1, "fi_atomicvalid(): bad count \n");
- }
-}
-
-Test(rdm_atomic, invalid_atomic)
-{
- int i, j;
-
- for(i = 0; i < FI_ATOMIC_OP_LAST; i++) {
- for(j = 0; j < FI_DATATYPE_LAST; j++) {
- do_invalid_atomic(j, i);
- }
- }
-}
-
-void do_min(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t min;
- float min_fp;
- double min_dp;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* i64 */
- *((int64_t *)source) = SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT64, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
-
- /* i32 */
- *((int64_t *)source) = SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT32, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
-
- /* float */
- *((float *)source) = SOURCE_DATA_FP;
- *((float *)target) = TARGET_DATA_FP;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_FLOAT, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((float *)target) == min_fp;
- cr_assert(ret, "Data mismatch");
-
- /* double */
- *((double *)source) = SOURCE_DATA_FP;
- *((double *)target) = TARGET_DATA_FP;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_DOUBLE, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((double *)target) == min_dp;
- cr_assert(ret, "Data mismatch");
-}
-
-Test(rdm_atomic, min)
-{
- rdm_atomic_xfer_for_each_size(do_min, 1, 1);
-}
-
-void do_max(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t min;
- float min_fp;
- double min_dp;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* i64 */
- *((int64_t *)source) = SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT64, FI_MAX, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int64_t)SOURCE_DATA > (int64_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
-
- /* i32 */
- *((int64_t *)source) = SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT32, FI_MAX, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int32_t)SOURCE_DATA > (int32_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
-
- /* float */
- *((float *)source) = SOURCE_DATA_FP;
- *((float *)target) = TARGET_DATA_FP;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_FLOAT, FI_MAX, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_fp = (float)SOURCE_DATA_FP > (float)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((float *)target) == min_fp;
- cr_assert(ret, "Data mismatch");
-
- /* double */
- *((double *)source) = SOURCE_DATA_FP;
- *((double *)target) = TARGET_DATA_FP;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_DOUBLE, FI_MAX, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_dp = (double)SOURCE_DATA_FP > (double)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((double *)target) == min_dp;
- cr_assert(ret, "Data mismatch");
-}
-
-Test(rdm_atomic, max)
-{
- rdm_atomic_xfer_for_each_size(do_max, 1, 1);
-}
-
-void do_sum(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT64, FI_SUM, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == (SOURCE_DATA + TARGET_DATA);
- cr_assert(ret, "Data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT32, FI_SUM, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) + TARGET_DATA);
- cr_assert(ret, "Data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT64, FI_SUM, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == (SOURCE_DATA + TARGET_DATA);
- cr_assert(ret, "Data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT32, FI_SUM, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) + TARGET_DATA);
- cr_assert(ret, "Data mismatch");
-
- /* float */
- *((float *)source) = SOURCE_DATA_FP;
- *((float *)target) = TARGET_DATA_FP;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_FLOAT, FI_SUM, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((float *)target) ==
- (float)(SOURCE_DATA_FP + TARGET_DATA_FP);
- cr_assert(ret, "Data mismatch");
-}
-
-Test(rdm_atomic, sum)
-{
- rdm_atomic_xfer_for_each_size(do_sum, 1, 1);
-}
-
-void do_bor(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t res;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT64, FI_BOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA | TARGET_DATA;
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT32, FI_BOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA | TARGET_DATA;
- res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT64, FI_BOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA | TARGET_DATA;
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT32, FI_BOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA | TARGET_DATA;
- res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
-}
-
-Test(rdm_atomic, bor)
-{
- rdm_atomic_xfer_for_each_size(do_bor, 1, 1);
-}
-
-void do_band(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t res;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT64, FI_BAND, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA & TARGET_DATA;
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT32, FI_BAND, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA & TARGET_DATA;
- res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT64, FI_BAND, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA & TARGET_DATA;
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT32, FI_BAND, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA & TARGET_DATA;
- res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
-}
-
-Test(rdm_atomic, band)
-{
- rdm_atomic_xfer_for_each_size(do_band, 1, 1);
-}
-
-void do_bxor(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t res;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT64, FI_BXOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA ^ TARGET_DATA;
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT32, FI_BXOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA ^ TARGET_DATA;
- res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT64, FI_BXOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA ^ TARGET_DATA;
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT32, FI_BXOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA ^ TARGET_DATA;
- res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
-}
-
-Test(rdm_atomic, bxor)
-{
- rdm_atomic_xfer_for_each_size(do_bxor, 1, 1);
-}
-
-void do_atomic_write(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT64, FI_ATOMIC_WRITE, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == SOURCE_DATA;
- cr_assert(ret, "Data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT32, FI_ATOMIC_WRITE, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT64, FI_ATOMIC_WRITE, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == SOURCE_DATA;
- cr_assert(ret, "Data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT32, FI_ATOMIC_WRITE, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
-
- /* float */
- *((float *)source) = SOURCE_DATA_FP;
- *((float *)target) = TARGET_DATA_FP;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_FLOAT, FI_ATOMIC_WRITE, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((float *)target) == (float)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
-
- /* double */
- *((double *)source) = SOURCE_DATA_FP;
- *((double *)target) = TARGET_DATA_FP;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_DOUBLE, FI_ATOMIC_WRITE, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((double *)target) == (double)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
-}
-
-Test(rdm_atomic, write)
-{
- rdm_atomic_xfer_for_each_size(do_atomic_write, 1, 1);
-}
-
-void do_min_buf(void *s, void *t)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t min;
- float min_fp;
- double min_dp;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* i64 */
- *((int64_t *)s) = SOURCE_DATA;
- *((int64_t *)t) = TARGET_DATA;
- sz = fi_atomic(ep[0], s, 1,
- loc_mr[0], gni_addr[1], (uint64_t)t, mr_key[1],
- FI_INT64, FI_MIN, t);
- if ((uint64_t)t & 0x7) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- ret = *((int64_t *)t) == min;
- cr_assert(ret, "Data mismatch");
- }
-
- /* i32 */
- *((int64_t *)s) = SOURCE_DATA;
- *((int64_t *)t) = TARGET_DATA;
- sz = fi_atomic(ep[0], s, 1,
- loc_mr[0], gni_addr[1], (uint64_t)t, mr_key[1],
- FI_INT32, FI_MIN, t);
- if ((uint64_t)t & 0x3) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((int64_t *)t) == min;
- cr_assert(ret, "Data mismatch");
- }
-
- /* float */
- *((float *)s) = SOURCE_DATA_FP;
- *((float *)t) = TARGET_DATA_FP;
- sz = fi_atomic(ep[0], s, 1,
- loc_mr[0], gni_addr[1], (uint64_t)t, mr_key[1],
- FI_FLOAT, FI_MIN, t);
- if ((uint64_t)t & 0x3) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((float *)t) == min_fp;
- cr_assert(ret, "Data mismatch");
- }
-
- /* double */
- *((double *)s) = SOURCE_DATA_FP;
- *((double *)t) = TARGET_DATA_FP;
- sz = fi_atomic(ep[0], s, 1,
- loc_mr[0], gni_addr[1], (uint64_t)t, mr_key[1],
- FI_DOUBLE, FI_MIN, t);
- if ((uint64_t)t & 0x7) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((double *)t) == min_dp;
- cr_assert(ret, "Data mismatch");
- }
-}
-
-Test(rdm_atomic, atomic_alignment)
-{
- int s_off, t_off;
-
- for (s_off = 0; s_off < 7; s_off++) {
- for (t_off = 0; t_off < 7; t_off++) {
- do_min_buf(source + s_off, target + t_off);
- }
- }
-}
-
-
-Test(rdm_atomic, atomicv)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t min;
- float min_fp;
- double min_dp;
- struct fi_ioc iov;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- iov.addr = source;
- iov.count = 1;
-
- /* i64 */
- *((int64_t *)source) = SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_atomicv(ep[0], &iov, (void **)loc_mr, 1, gni_addr[1],
- (uint64_t)target, mr_key[1], FI_INT64, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
-
- /* i32 */
- *((int64_t *)source) = SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_atomicv(ep[0], &iov, (void **)loc_mr, 1, gni_addr[1],
- (uint64_t)target, mr_key[1], FI_INT32, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
-
- /* float */
- *((float *)source) = SOURCE_DATA_FP;
- *((float *)target) = TARGET_DATA_FP;
- sz = fi_atomicv(ep[0], &iov, (void **)loc_mr, 1, gni_addr[1],
- (uint64_t)target, mr_key[1], FI_FLOAT, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((float *)target) == min_fp;
- cr_assert(ret, "Data mismatch");
-
- /* double */
- *((double *)source) = SOURCE_DATA_FP;
- *((double *)target) = TARGET_DATA_FP;
- sz = fi_atomicv(ep[0], &iov, (void **)loc_mr, 1, gni_addr[1],
- (uint64_t)target, mr_key[1], FI_DOUBLE, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((double *)target) == min_dp;
- cr_assert(ret, "Data mismatch");
-}
-
-Test(rdm_atomic, atomicmsg)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t min;
- float min_fp;
- double min_dp;
- struct fi_msg_atomic msg;
- struct fi_ioc msg_iov;
- struct fi_rma_ioc rma_iov;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- msg_iov.addr = source;
- msg_iov.count = 1;
- msg.msg_iov = &msg_iov;
- msg.desc = (void **)loc_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[1];
- rma_iov.addr = (uint64_t)target;
- rma_iov.count = 1;
- rma_iov.key = mr_key[1];
- msg.rma_iov = &rma_iov;
- msg.context = target;
- msg.op = FI_MIN;
-
- /* i64 */
- *((int64_t *)source) = SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- msg.datatype = FI_INT64;
- sz = fi_atomicmsg(ep[0], &msg, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
-
- /* i32 */
- *((int64_t *)source) = SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- msg.datatype = FI_INT32;
- sz = fi_atomicmsg(ep[0], &msg, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
-
- /* float */
- *((float *)source) = SOURCE_DATA_FP;
- *((float *)target) = TARGET_DATA_FP;
- msg.datatype = FI_FLOAT;
- sz = fi_atomicmsg(ep[0], &msg, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((float *)target) == min_fp;
- cr_assert(ret, "Data mismatch");
-
- /* double */
- *((double *)source) = SOURCE_DATA_FP;
- *((double *)target) = TARGET_DATA_FP;
- msg.datatype = FI_DOUBLE;
- sz = fi_atomicmsg(ep[0], &msg, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((double *)target) == min_dp;
- cr_assert(ret, "Data mismatch");
-}
-
-Test(rdm_atomic, atomicinject)
-{
- int ret, loops;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t min;
- float min_fp;
- double min_dp;
-
- /* i64 */
- *((int64_t *)source) = SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_inject_atomic(ep[0], source, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT64, FI_MIN);
- cr_assert_eq(sz, 0);
-
- min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- loops = 0;
- while (*((int64_t *)target) != min) {
- ret = fi_cq_read(send_cq[0], &cqe, 1); /* for progress */
- cr_assert(ret == -FI_EAGAIN,
- "Received unexpected event\n");
-
- pthread_yield();
- cr_assert(++loops < 10000, "Data mismatch");
- }
-
- /* i32 */
- *((int64_t *)source) = SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_inject_atomic(ep[0], source, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT32, FI_MIN);
- cr_assert_eq(sz, 0);
-
- min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- loops = 0;
- while (*((int64_t *)target) != min) {
- ret = fi_cq_read(send_cq[0], &cqe, 1); /* for progress */
- cr_assert(ret == -FI_EAGAIN,
- "Received unexpected event\n");
-
- pthread_yield();
- cr_assert(++loops < 10000, "Data mismatch");
- }
-
- /* float */
- *((float *)source) = SOURCE_DATA_FP;
- *((float *)target) = TARGET_DATA_FP;
- sz = fi_inject_atomic(ep[0], source, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_FLOAT, FI_MIN);
- cr_assert_eq(sz, 0);
-
- min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- loops = 0;
- while (*((float *)target) != min_fp) {
- ret = fi_cq_read(send_cq[0], &cqe, 1); /* for progress */
- cr_assert(ret == -FI_EAGAIN,
- "Received unexpected event\n");
-
- pthread_yield();
- cr_assert(++loops < 10000, "Data mismatch");
- }
-
- /* double */
- *((double *)source) = SOURCE_DATA_FP;
- *((double *)target) = TARGET_DATA_FP;
- sz = fi_inject_atomic(ep[0], source, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_DOUBLE, FI_MIN);
- cr_assert_eq(sz, 0);
-
- min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- loops = 0;
- while (*((double *)target) != min_dp) {
- ret = fi_cq_read(send_cq[0], &cqe, 1); /* for progress */
- cr_assert(ret == -FI_EAGAIN,
- "Received unexpected event\n");
-
- pthread_yield();
- cr_assert(++loops < 10000, "Data mismatch");
- }
-}
-
-/******************************************************************************
- *
- * Fetching atomics
- *
- *****************************************************************************/
-
-int supported_fetch_atomic_ops[FI_ATOMIC_OP_LAST][FI_DATATYPE_LAST] = {
- [FI_MIN] = { 0,0,0,0,1,0,1,0,1,1 },
- [FI_MAX] = { 0,0,0,0,1,0,1,0,1,1 },
- [FI_SUM] = { 0,0,0,0,1,1,1,1,1,0 }, /* GNI DP sum is broken */
- [FI_PROD] = NO_DATATYPES_SUPPORTED,
- [FI_LOR] = NO_DATATYPES_SUPPORTED,
- [FI_LAND] = NO_DATATYPES_SUPPORTED,
- [FI_BOR] = GNI_DATATYPES_NO_FP_SUPPORTED,
- [FI_BAND] = GNI_DATATYPES_NO_FP_SUPPORTED,
- [FI_LXOR] = NO_DATATYPES_SUPPORTED,
- [FI_BXOR] = GNI_DATATYPES_NO_FP_SUPPORTED,
- [FI_ATOMIC_READ] = ALL_GNI_DATATYPES_SUPPORTED,
- [FI_ATOMIC_WRITE] = ALL_GNI_DATATYPES_SUPPORTED,
- [FI_CSWAP] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_NE] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_LE] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_LT] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_GE] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_GT] = NO_DATATYPES_SUPPORTED,
- [FI_MSWAP] = NO_DATATYPES_SUPPORTED,
-};
-
-void do_invalid_fetch_atomic(enum fi_datatype dt, enum fi_op op)
-{
- ssize_t sz;
- size_t count;
- uint64_t operand;
-
- if (!supported_fetch_atomic_ops[op][dt]) {
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0],
- gni_addr[1], (uint64_t)target, mr_key[1],
- dt, op, target);
- cr_assert(sz == -FI_ENOENT);
-
- sz = fi_fetch_atomicvalid(ep[0], dt, op, &count);
- cr_assert(sz == -FI_ENOENT, "fi_atomicvalid() succeeded\n");
- } else {
- sz = fi_fetch_atomicvalid(ep[0], dt, op, &count);
- cr_assert(!sz, "fi_atomicvalid() failed\n");
- cr_assert(count == 1, "fi_atomicvalid(): bad count \n");
- }
-}
-
-Test(rdm_atomic, invalid_fetch_atomic)
-{
- int i, j;
-
- for(i = 0; i < FI_ATOMIC_OP_LAST; i++) {
- for(j = 0; j < FI_DATATYPE_LAST; j++) {
- do_invalid_fetch_atomic(j, i);
- }
- }
-}
-
-void do_fetch_min(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t min;
- float min_fp;
- double min_dp;
- uint64_t operand = SOURCE_DATA;
- float operand_fp;
- double operand_dp;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* i64 */
- *((int64_t *)source) = FETCH_SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT64, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
- ret = *((int64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i32 */
- *((int64_t *)source) = FETCH_SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT32, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
- min = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
- ret = *((int64_t *)source) == min;
- cr_assert(ret, "Fetch data mismatch");
-
- /* float */
- *((float *)&operand_fp) = SOURCE_DATA_FP;
- *((float *)source) = FETCH_SOURCE_DATA;
- *((float *)target) = TARGET_DATA_FP;
- sz = fi_fetch_atomic(ep[0], &operand_fp, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_FLOAT, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((float *)target) == min_fp;
- cr_assert(ret, "Data mismatch");
- ret = *((float *)source) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-
- /* double */
- *((double *)&operand_dp) = SOURCE_DATA_FP;
- *((double *)source) = SOURCE_DATA_FP;
- *((double *)target) = TARGET_DATA_FP;
- sz = fi_fetch_atomic(ep[0], &operand_dp, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_DOUBLE, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((double *)target) == min_dp;
- cr_assert(ret, "Data mismatch");
- ret = *((double *)source) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-}
-
-Test(rdm_atomic, fetch_min)
-{
- rdm_atomic_xfer_for_each_size(do_fetch_min, 1, 1);
-}
-
-void do_fetch_max(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t min;
- float min_fp;
- double min_dp;
- uint64_t operand = SOURCE_DATA;
- float operand_fp;
- double operand_dp;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* i64 */
- *((int64_t *)source) = FETCH_SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT64, FI_MAX, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int64_t)SOURCE_DATA > (int64_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
- ret = *((int64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i32 */
- *((int64_t *)source) = FETCH_SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT32, FI_MAX, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int32_t)SOURCE_DATA > (int32_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
- min = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
- ret = *((int64_t *)source) == min;
- cr_assert(ret, "Fetch data mismatch");
-
- /* float */
- *((float *)&operand_fp) = SOURCE_DATA_FP;
- *((float *)source) = FETCH_SOURCE_DATA;
- *((float *)target) = TARGET_DATA_FP;
- sz = fi_fetch_atomic(ep[0], &operand_fp, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_FLOAT, FI_MAX, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_fp = (float)SOURCE_DATA_FP > (float)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((float *)target) == min_fp;
- cr_assert(ret, "Data mismatch");
- ret = *((float *)source) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-
- /* double */
- *((double *)&operand_dp) = SOURCE_DATA_FP;
- *((double *)source) = SOURCE_DATA_FP;
- *((double *)target) = TARGET_DATA_FP;
- sz = fi_fetch_atomic(ep[0], &operand_dp, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_DOUBLE, FI_MAX, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_dp = (double)SOURCE_DATA_FP > (double)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((double *)target) == min_dp;
- cr_assert(ret, "Data mismatch");
- ret = *((double *)source) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-}
-
-Test(rdm_atomic, fetch_max)
-{
- rdm_atomic_xfer_for_each_size(do_fetch_max, 1, 1);
-}
-
-void do_fetch_sum(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t operand = SOURCE_DATA;
- float operand_fp;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT64, FI_SUM, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == (SOURCE_DATA + TARGET_DATA);
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT32, FI_SUM, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) + TARGET_DATA);
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT64, FI_SUM, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == (SOURCE_DATA + TARGET_DATA);
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT32, FI_SUM, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) + TARGET_DATA);
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
-
- /* float */
- *((float *)&operand_fp) = SOURCE_DATA_FP;
- *((float *)source) = FETCH_SOURCE_DATA;
- *((float *)target) = TARGET_DATA_FP;
- sz = fi_fetch_atomic(ep[0], &operand_fp, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_FLOAT, FI_SUM, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((float *)target) ==
- (float)(SOURCE_DATA_FP + TARGET_DATA_FP);
- cr_assert(ret, "Data mismatch");
- ret = *((float *)source) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-}
-
-Test(rdm_atomic, fetch_sum)
-{
- rdm_atomic_xfer_for_each_size(do_fetch_sum, 1, 1);
-}
-
-void do_fetch_bor(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t res;
- uint64_t operand = SOURCE_DATA;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT64, FI_BOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA | TARGET_DATA;
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT32, FI_BOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA | TARGET_DATA;
- res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- res = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)source) == res;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT64, FI_BOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA | TARGET_DATA;
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT32, FI_BOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA | TARGET_DATA;
- res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- res = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)source) == res;
- cr_assert(ret, "Fetch data mismatch");
-}
-
-Test(rdm_atomic, fetch_bor)
-{
- rdm_atomic_xfer_for_each_size(do_fetch_bor, 1, 1);
-}
-
-void do_fetch_band(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t res;
- uint64_t operand = SOURCE_DATA;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT64, FI_BAND, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA & TARGET_DATA;
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT32, FI_BAND, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA & TARGET_DATA;
- res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- res = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)source) == res;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT64, FI_BAND, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA & TARGET_DATA;
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT32, FI_BAND, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA & TARGET_DATA;
- res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- res = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)source) == res;
- cr_assert(ret, "Fetch data mismatch");
-}
-
-Test(rdm_atomic, fetch_band)
-{
- rdm_atomic_xfer_for_each_size(do_fetch_band, 1, 1);
-}
-
-void do_fetch_bxor(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t res;
- uint64_t operand = SOURCE_DATA;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT64, FI_BXOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA ^ TARGET_DATA;
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT32, FI_BXOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA ^ TARGET_DATA;
- res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- res = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)source) == res;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT64, FI_BXOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA ^ TARGET_DATA;
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT32, FI_BXOR, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = SOURCE_DATA ^ TARGET_DATA;
- res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- res = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
- ret = *((uint64_t *)source) == res;
- cr_assert(ret, "Fetch data mismatch");
-}
-
-Test(rdm_atomic, fetch_bxor)
-{
- rdm_atomic_xfer_for_each_size(do_fetch_bxor, 1, 1);
-}
-
-void do_fetch_atomic_write(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t operand = SOURCE_DATA;
- float operand_fp;
- double operand_dp;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT64, FI_ATOMIC_WRITE, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == SOURCE_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT32, FI_ATOMIC_WRITE, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = 0;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT64, FI_ATOMIC_WRITE, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == SOURCE_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT32, FI_ATOMIC_WRITE, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
-
- /* float */
- *((float *)&operand_fp) = SOURCE_DATA_FP;
- *((float *)source) = FETCH_SOURCE_DATA;
- *((float *)target) = TARGET_DATA_FP;
- sz = fi_fetch_atomic(ep[0], &operand_fp, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_FLOAT, FI_ATOMIC_WRITE, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((float *)target) == (float)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((float *)source) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-
- /* double */
- *(double *)&operand_dp = SOURCE_DATA_FP;
- *((double *)source) = FETCH_SOURCE_DATA;
- *((double *)target) = TARGET_DATA_FP;
- sz = fi_fetch_atomic(ep[0], &operand_dp, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_DOUBLE, FI_ATOMIC_WRITE, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((double *)target) == (double)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((double *)source) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-}
-
-Test(rdm_atomic, fetch_atomic_write)
-{
- rdm_atomic_xfer_for_each_size(do_fetch_atomic_write, 1, 1);
-}
-
-void do_fetch_atomic_read(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t operand;
- float operand_fp;
- double operand_dp;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- operand = SOURCE_DATA;
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT64, FI_ATOMIC_READ, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == TARGET_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == (uint64_t)TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
- ret = *((uint64_t *)&operand) == TARGET_DATA;
- cr_assert(ret, "Read data mismatch");
-
- /* U32 */
- operand = SOURCE_DATA;
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT32, FI_ATOMIC_READ, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == TARGET_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
- ret = *((uint64_t *)&operand) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Read data mismatch");
-
- /* i64 */
- operand = SOURCE_DATA;
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT64, FI_ATOMIC_READ, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == TARGET_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
- ret = *((uint64_t *)&operand) == TARGET_DATA;
- cr_assert(ret, "Read data mismatch");
-
- /* i32 */
- operand = SOURCE_DATA;
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT32, FI_ATOMIC_READ, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == TARGET_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
- ret = *((uint64_t *)&operand) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Read data mismatch");
-
- /* float */
- *((float *)&operand_fp) = SOURCE_DATA_FP;
- *((float *)source) = FETCH_SOURCE_DATA;
- *((float *)target) = TARGET_DATA_FP;
- sz = fi_fetch_atomic(ep[0], &operand_fp, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_FLOAT, FI_ATOMIC_READ, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((float *)target) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((float *)source) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
- ret = *((float *)&operand_fp) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Read data mismatch");
-
- /* double */
- *(double *)&operand_dp = SOURCE_DATA_FP;
- *((double *)source) = FETCH_SOURCE_DATA;
- *((double *)target) = TARGET_DATA_FP;
- sz = fi_fetch_atomic(ep[0], &operand_dp, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_DOUBLE, FI_ATOMIC_READ, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((double *)target) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((double *)source) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
- ret = *((double *)&operand_dp) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Read data mismatch");
-}
-
-Test(rdm_atomic, fetch_atomic_read)
-{
- rdm_atomic_xfer_for_each_size(do_fetch_atomic_read, 1, 1);
-}
-
-void do_fetch_min_buf(void *s, void *t)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t min;
- float min_fp;
- double min_dp;
- uint64_t operand = SOURCE_DATA;
- float operand_fp;
- double operand_dp;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* i64 */
- *((int64_t *)s) = FETCH_SOURCE_DATA;
- *((int64_t *)t) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- s, loc_mr[0], gni_addr[1], (uint64_t)t, mr_key[1],
- FI_INT64, FI_MIN, t);
- if ((uint64_t)s & 0x7 || (uint64_t)t & 0x7) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- ret = *((int64_t *)t) == min;
- cr_assert(ret, "Data mismatch");
- ret = *((int64_t *)s) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
- }
-
- /* i32 */
- *((int64_t *)s) = FETCH_SOURCE_DATA;
- *((int64_t *)t) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- s, loc_mr[0], gni_addr[1], (uint64_t)t, mr_key[1],
- FI_INT32, FI_MIN, t);
- if ((uint64_t)s & 0x3 || (uint64_t)t & 0x3) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((int64_t *)t) == min;
- cr_assert(ret, "Data mismatch");
- min = (TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32));
- ret = *((int64_t *)s) == min;
- cr_assert(ret, "Fetch data mismatch");
- }
-
- /* float */
- *((float *)&operand_fp) = SOURCE_DATA_FP;
- *((float *)s) = FETCH_SOURCE_DATA;
- *((float *)t) = TARGET_DATA_FP;
- sz = fi_fetch_atomic(ep[0], &operand_fp, 1, NULL,
- s, loc_mr[0], gni_addr[1], (uint64_t)t, mr_key[1],
- FI_FLOAT, FI_MIN, t);
- if ((uint64_t)s & 0x3 || (uint64_t)t & 0x3) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((float *)t) == min_fp;
- cr_assert(ret, "Data mismatch");
- ret = *((float *)s) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
- }
-
- /* double */
- *((double *)&operand_dp) = SOURCE_DATA_FP;
- *((double *)s) = SOURCE_DATA_FP;
- *((double *)t) = TARGET_DATA_FP;
- sz = fi_fetch_atomic(ep[0], &operand_dp, 1, NULL,
- s, loc_mr[0], gni_addr[1], (uint64_t)t, mr_key[1],
- FI_DOUBLE, FI_MIN, t);
- if ((uint64_t)s & 0x7 || (uint64_t)t & 0x7) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((double *)t) == min_dp;
- cr_assert(ret, "Data mismatch");
- ret = *((double *)s) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
- }
-}
-
-Test(rdm_atomic, atomic_fetch_alignment)
-{
- int s_off, t_off;
-
- for (s_off = 0; s_off < 7; s_off++) {
- for (t_off = 0; t_off < 7; t_off++) {
- do_fetch_min_buf(source + s_off, target + t_off);
- }
- }
-}
-
-Test(rdm_atomic, fatomicv)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t min;
- float min_fp;
- double min_dp;
- uint64_t operand = SOURCE_DATA;
- float operand_fp;
- double operand_dp;
- struct fi_ioc iov, r_iov;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- iov.count = 1;
- r_iov.count = 1;
-
- /* i64 */
- *((int64_t *)source) = FETCH_SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- iov.addr = &operand;
- r_iov.addr = source;
- sz = fi_fetch_atomicv(ep[0], &iov, NULL, 1,
- &r_iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT64, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
- ret = *((int64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i32 */
- *((int64_t *)source) = FETCH_SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- iov.addr = &operand;
- r_iov.addr = source;
- sz = fi_fetch_atomicv(ep[0], &iov, NULL, 1,
- &r_iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT32, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
- min = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
- ret = *((int64_t *)source) == min;
- cr_assert(ret, "Fetch data mismatch");
-
- /* float */
- *((float *)&operand_fp) = SOURCE_DATA_FP;
- *((float *)source) = FETCH_SOURCE_DATA;
- *((float *)target) = TARGET_DATA_FP;
- iov.addr = &operand_fp;
- r_iov.addr = source;
- sz = fi_fetch_atomicv(ep[0], &iov, NULL, 1,
- &r_iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_FLOAT, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((float *)target) == min_fp;
- cr_assert(ret, "Data mismatch");
- ret = *((float *)source) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-
- /* double */
- *((double *)&operand_dp) = SOURCE_DATA_FP;
- *((double *)source) = SOURCE_DATA_FP;
- *((double *)target) = TARGET_DATA_FP;
- iov.addr = &operand_dp;
- r_iov.addr = source;
- sz = fi_fetch_atomicv(ep[0], &iov, NULL, 1,
- &r_iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_DOUBLE, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((double *)target) == min_dp;
- cr_assert(ret, "Data mismatch");
- ret = *((double *)source) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-}
-
-Test(rdm_atomic, fatomicmsg)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t min;
- float min_fp;
- double min_dp;
- uint64_t operand = SOURCE_DATA;
- float operand_fp;
- double operand_dp;
- struct fi_msg_atomic msg;
- struct fi_ioc msg_iov, res_iov;
- struct fi_rma_ioc rma_iov;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- msg_iov.count = 1;
- msg.msg_iov = &msg_iov;
- msg.desc = (void **)loc_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[1];
- rma_iov.addr = (uint64_t)target;
- rma_iov.count = 1;
- rma_iov.key = mr_key[1];
- msg.rma_iov = &rma_iov;
- msg.context = target;
- msg.op = FI_MIN;
-
- res_iov.addr = source;
- res_iov.count = 1;
-
- /* i64 */
- *((int64_t *)source) = FETCH_SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- msg_iov.addr = &operand;
- msg.datatype = FI_INT64;
- sz = fi_fetch_atomicmsg(ep[0], &msg, &res_iov, (void **)loc_mr, 1, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
- ret = *((int64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i32 */
- *((int64_t *)source) = FETCH_SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- msg_iov.addr = &operand;
- msg.datatype = FI_INT32;
- sz = fi_fetch_atomicmsg(ep[0], &msg, &res_iov, (void **)loc_mr, 1, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
- SOURCE_DATA : TARGET_DATA;
- min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
- ret = *((int64_t *)target) == min;
- cr_assert(ret, "Data mismatch");
- min = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
- ret = *((int64_t *)source) == min;
- cr_assert(ret, "Fetch data mismatch");
-
- /* float */
- *((float *)&operand_fp) = SOURCE_DATA_FP;
- *((float *)source) = FETCH_SOURCE_DATA;
- *((float *)target) = TARGET_DATA_FP;
- msg_iov.addr = &operand_fp;
- msg.datatype = FI_FLOAT;
- sz = fi_fetch_atomicmsg(ep[0], &msg, &res_iov, (void **)loc_mr, 1, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((float *)target) == min_fp;
- cr_assert(ret, "Data mismatch");
- ret = *((float *)source) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-
- /* double */
- *((double *)&operand_dp) = SOURCE_DATA_FP;
- *((double *)source) = SOURCE_DATA_FP;
- *((double *)target) = TARGET_DATA_FP;
- msg_iov.addr = &operand_dp;
- msg.datatype = FI_DOUBLE;
- sz = fi_fetch_atomicmsg(ep[0], &msg, &res_iov, (void **)loc_mr, 1, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
- SOURCE_DATA_FP : TARGET_DATA_FP;
- ret = *((double *)target) == min_dp;
- cr_assert(ret, "Data mismatch");
- ret = *((double *)source) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-}
-
-/******************************************************************************
- *
- * Compare atomics
- *
- *****************************************************************************/
-
-int supported_compare_atomic_ops[FI_ATOMIC_OP_LAST][FI_DATATYPE_LAST] = {
- [FI_MIN] = NO_DATATYPES_SUPPORTED,
- [FI_MAX] = NO_DATATYPES_SUPPORTED,
- [FI_SUM] = NO_DATATYPES_SUPPORTED,
- [FI_PROD] = NO_DATATYPES_SUPPORTED,
- [FI_LOR] = NO_DATATYPES_SUPPORTED,
- [FI_LAND] = NO_DATATYPES_SUPPORTED,
- [FI_BOR] = NO_DATATYPES_SUPPORTED,
- [FI_BAND] = NO_DATATYPES_SUPPORTED,
- [FI_LXOR] = NO_DATATYPES_SUPPORTED,
- [FI_BXOR] = NO_DATATYPES_SUPPORTED,
- [FI_ATOMIC_READ] = NO_DATATYPES_SUPPORTED,
- [FI_ATOMIC_WRITE] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP] = ALL_GNI_DATATYPES_SUPPORTED,
- [FI_CSWAP_NE] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_LE] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_LT] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_GE] = NO_DATATYPES_SUPPORTED,
- [FI_CSWAP_GT] = NO_DATATYPES_SUPPORTED,
- [FI_MSWAP] = ALL_GNI_DATATYPES_SUPPORTED,
-};
-
-void do_invalid_compare_atomic(enum fi_datatype dt, enum fi_op op)
-{
- ssize_t sz;
- size_t count;
- uint64_t operand, op2;
-
- if (!supported_compare_atomic_ops[op][dt]) {
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- source, loc_mr,
- gni_addr[1], (uint64_t)target, mr_key[1],
- dt, op, target);
- cr_assert(sz == -FI_ENOENT);
-
- sz = fi_compare_atomicvalid(ep[0], dt, op, &count);
- cr_assert(sz == -FI_ENOENT, "fi_atomicvalid() succeeded\n");
- } else {
- sz = fi_compare_atomicvalid(ep[0], dt, op, &count);
- cr_assert(!sz, "fi_atomicvalid() failed\n");
- cr_assert(count == 1, "fi_atomicvalid(): bad count \n");
- }
-}
-
-Test(rdm_atomic, invalid_compare_atomic)
-{
- int i, j;
-
- for(i = 0; i < FI_ATOMIC_OP_LAST; i++) {
- for(j = 0; j < FI_DATATYPE_LAST; j++) {
- do_invalid_compare_atomic(j, i);
- }
- }
-}
-
-void do_cswap(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t operand = SOURCE_DATA, op2 = TARGET_DATA;
- float operand_fp, op2_fp;
- double operand_dp, op2_dp;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT64, FI_CSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == SOURCE_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT32, FI_CSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT64, FI_CSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == SOURCE_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT32, FI_CSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
-
- /* float */
- *((float *)&operand_fp) = SOURCE_DATA_FP;
- *((float *)&op2_fp) = TARGET_DATA_FP;
- *((float *)source) = FETCH_SOURCE_DATA;
- *((float *)target) = TARGET_DATA_FP;
- sz = fi_compare_atomic(ep[0], &operand_fp, 1, NULL, &op2_fp, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_FLOAT, FI_CSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((float *)target) == (float)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((float *)source) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-
- /* double */
- *((double *)&operand_dp) = SOURCE_DATA_FP;
- *((double *)&op2_dp) = TARGET_DATA_FP;
- *((double *)source) = FETCH_SOURCE_DATA;
- *((double *)target) = TARGET_DATA_FP;
- sz = fi_compare_atomic(ep[0], &operand_dp, 1, NULL, &op2_dp, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_DOUBLE, FI_CSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((double *)target) == (double)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((double *)source) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-}
-
-Test(rdm_atomic, cswap)
-{
- rdm_atomic_xfer_for_each_size(do_cswap, 1, 1);
-}
-
-void do_mswap(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t res;
- uint64_t operand = SOURCE_DATA, op2 = DATA_MASK;
- float operand_fp, op2_fp;
- double operand_dp, op2_dp;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT64, FI_MSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = (SOURCE_DATA & DATA_MASK) | (TARGET_DATA & ~DATA_MASK);
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT32, FI_MSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = (SOURCE_DATA & DATA_MASK) | (TARGET_DATA & ~DATA_MASK);
- ret = *((uint64_t *)target) ==
- (uint64_t)((res & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT64, FI_MSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = (SOURCE_DATA & DATA_MASK) | (TARGET_DATA & ~DATA_MASK);
- ret = *((uint64_t *)target) == res;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT32, FI_MSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- res = (SOURCE_DATA & DATA_MASK) | (TARGET_DATA & ~DATA_MASK);
- ret = *((uint64_t *)target) ==
- (uint64_t)((res & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
-
- /* float */
- *((float *)&operand_fp) = SOURCE_DATA_FP;
- *((float *)&op2_fp) = TARGET_DATA_FP;
- *((float *)source) = FETCH_SOURCE_DATA;
- *((float *)target) = TARGET_DATA_FP;
- sz = fi_compare_atomic(ep[0], &operand_fp, 1, NULL, &op2_fp, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_FLOAT, FI_MSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((float *)target) == (float)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((float *)source) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-
- /* double */
- *((double *)&operand_dp) = SOURCE_DATA_FP;
- *((double *)&op2_dp) = TARGET_DATA_FP;
- *((double *)source) = FETCH_SOURCE_DATA;
- *((double *)target) = TARGET_DATA_FP;
- sz = fi_compare_atomic(ep[0], &operand_dp, 1, NULL, &op2_dp, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_DOUBLE, FI_MSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((double *)target) == (double)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((double *)source) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-}
-
-Test(rdm_atomic, mswap)
-{
- rdm_atomic_xfer_for_each_size(do_mswap, 1, 1);
-}
-
-void do_cswap_buf(void *s, void *t)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t operand = SOURCE_DATA, op2 = TARGET_DATA;
- float operand_fp, op2_fp;
- double operand_dp, op2_dp;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- /* u64 */
- *((uint64_t *)s) = FETCH_SOURCE_DATA;
- *((uint64_t *)t) = TARGET_DATA;
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- s, loc_mr[0], gni_addr[1], (uint64_t)t,
- mr_key[1], FI_UINT64, FI_CSWAP, t);
- if ((uint64_t)s & 0x7 || (uint64_t)t & 0x7) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)t) == SOURCE_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)s) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
- }
-
- /* U32 */
- *((uint64_t *)s) = FETCH_SOURCE_DATA;
- *((uint64_t *)t) = TARGET_DATA;
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- s, loc_mr[0], gni_addr[1], (uint64_t)t,
- mr_key[1], FI_UINT32, FI_CSWAP, t);
- if ((uint64_t)s & 0x3 || (uint64_t)t & 0x3) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)t) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)s) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA &
- (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
- }
-
- /* i64 */
- *((uint64_t *)s) = FETCH_SOURCE_DATA;
- *((uint64_t *)t) = TARGET_DATA;
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- s, loc_mr[0], gni_addr[1], (uint64_t)t,
- mr_key[1], FI_INT64, FI_CSWAP, t);
- if ((uint64_t)s & 0x7 || (uint64_t)t & 0x7) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)t) == SOURCE_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)s) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
- }
-
- /* i32 */
- *((uint64_t *)s) = FETCH_SOURCE_DATA;
- *((uint64_t *)t) = TARGET_DATA;
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- s, loc_mr[0], gni_addr[1], (uint64_t)t,
- mr_key[1], FI_INT32, FI_CSWAP, t);
- if ((uint64_t)s & 0x3 || (uint64_t)t & 0x3) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)t) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)s) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA &
- (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
- }
-
- /* float */
- *((float *)&operand_fp) = SOURCE_DATA_FP;
- *((float *)&op2_fp) = TARGET_DATA_FP;
- *((float *)s) = FETCH_SOURCE_DATA;
- *((float *)t) = TARGET_DATA_FP;
- sz = fi_compare_atomic(ep[0], &operand_fp, 1, NULL, &op2_fp, NULL,
- s, loc_mr[0], gni_addr[1], (uint64_t)t,
- mr_key[1], FI_FLOAT, FI_CSWAP, t);
- if ((uint64_t)s & 0x3 || (uint64_t)t & 0x3) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((float *)t) == (float)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((float *)s) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
- }
-
- /* double */
- *((double *)&operand_dp) = SOURCE_DATA_FP;
- *((double *)&op2_dp) = TARGET_DATA_FP;
- *((double *)s) = FETCH_SOURCE_DATA;
- *((double *)t) = TARGET_DATA_FP;
- sz = fi_compare_atomic(ep[0], &operand_dp, 1, NULL, &op2_dp, NULL,
- s, loc_mr[0], gni_addr[1], (uint64_t)t,
- mr_key[1], FI_DOUBLE, FI_CSWAP, t);
- if ((uint64_t)s & 0x7 || (uint64_t)t & 0x7) {
- cr_assert_eq(sz, -FI_EINVAL);
- } else {
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((double *)t) == (double)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((double *)s) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
- }
-}
-
-Test(rdm_atomic, atomic_compare_alignment)
-{
- int s_off, t_off;
-
- for (s_off = 0; s_off < 7; s_off++) {
- for (t_off = 0; t_off < 7; t_off++) {
- do_cswap_buf(source + s_off, target + t_off);
- }
- }
-}
-
-Test(rdm_atomic, catomicv)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t operand = SOURCE_DATA, op2 = TARGET_DATA;
- float operand_fp, op2_fp;
- double operand_dp, op2_dp;
- struct fi_ioc iov, r_iov, c_iov;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- iov.count = 1;
- r_iov.count = 1;
- c_iov.count = 1;
-
- /* u64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- iov.addr = &operand;
- r_iov.addr = source;
- c_iov.addr = &op2;
- sz = fi_compare_atomicv(ep[0],
- &iov, NULL, 1,
- &c_iov, NULL, 1,
- &r_iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT64, FI_CSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == SOURCE_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- iov.addr = &operand;
- r_iov.addr = source;
- c_iov.addr = &op2;
- sz = fi_compare_atomicv(ep[0],
- &iov, NULL, 1,
- &c_iov, NULL, 1,
- &r_iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_UINT32, FI_CSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- iov.addr = &operand;
- r_iov.addr = source;
- c_iov.addr = &op2;
- sz = fi_compare_atomicv(ep[0],
- &iov, NULL, 1,
- &c_iov, NULL, 1,
- &r_iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT64, FI_CSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == SOURCE_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- iov.addr = &operand;
- r_iov.addr = source;
- c_iov.addr = &op2;
- sz = fi_compare_atomicv(ep[0],
- &iov, NULL, 1,
- &c_iov, NULL, 1,
- &r_iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT32, FI_CSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
-
- /* float */
- *((float *)&operand_fp) = SOURCE_DATA_FP;
- *((float *)&op2_fp) = TARGET_DATA_FP;
- *((float *)source) = FETCH_SOURCE_DATA;
- *((float *)target) = TARGET_DATA_FP;
- iov.addr = &operand_fp;
- r_iov.addr = source;
- c_iov.addr = &op2_fp;
- sz = fi_compare_atomicv(ep[0],
- &iov, NULL, 1,
- &c_iov, NULL, 1,
- &r_iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_FLOAT, FI_CSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((float *)target) == (float)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((float *)source) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-
- /* double */
- *((double *)&operand_dp) = SOURCE_DATA_FP;
- *((double *)&op2_dp) = TARGET_DATA_FP;
- *((double *)source) = FETCH_SOURCE_DATA;
- *((double *)target) = TARGET_DATA_FP;
- iov.addr = &operand_dp;
- r_iov.addr = source;
- c_iov.addr = &op2_dp;
- sz = fi_compare_atomicv(ep[0],
- &iov, NULL, 1,
- &c_iov, NULL, 1,
- &r_iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- FI_DOUBLE, FI_CSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((double *)target) == (double)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((double *)source) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-}
-
-Test(rdm_atomic, catomicmsg)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t operand = SOURCE_DATA, op2 = TARGET_DATA;
- float operand_fp, op2_fp;
- double operand_dp, op2_dp;
- struct fi_msg_atomic msg;
- struct fi_ioc msg_iov, res_iov, cmp_iov;
- struct fi_rma_ioc rma_iov;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- msg_iov.count = 1;
- msg.msg_iov = &msg_iov;
- msg.desc = (void **)loc_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[1];
- rma_iov.addr = (uint64_t)target;
- rma_iov.count = 1;
- rma_iov.key = mr_key[1];
- msg.rma_iov = &rma_iov;
- msg.context = target;
- msg.op = FI_CSWAP;
-
- res_iov.count = 1;
- cmp_iov.count = 1;
-
- /* u64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- msg_iov.addr = &operand;
- msg.datatype = FI_UINT64;
- res_iov.addr = source;
- cmp_iov.addr = &op2;
- sz = fi_compare_atomicmsg(ep[0], &msg, &cmp_iov, NULL, 1,
- &res_iov, (void **)loc_mr, 1, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
-
- r[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == SOURCE_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* U32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- msg_iov.addr = &operand;
- msg.datatype = FI_UINT32;
- res_iov.addr = source;
- cmp_iov.addr = &op2;
- sz = fi_compare_atomicmsg(ep[0], &msg, &cmp_iov, NULL, 1,
- &res_iov, (void **)loc_mr, 1, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
-
- /* i64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- msg_iov.addr = &operand;
- msg.datatype = FI_INT64;
- res_iov.addr = source;
- cmp_iov.addr = &op2;
- sz = fi_compare_atomicmsg(ep[0], &msg, &cmp_iov, NULL, 1,
- &res_iov, (void **)loc_mr, 1, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) == SOURCE_DATA;
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) == TARGET_DATA;
- cr_assert(ret, "Fetch data mismatch");
-
- /* i32 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- msg_iov.addr = &operand;
- msg.datatype = FI_INT32;
- res_iov.addr = source;
- cmp_iov.addr = &op2;
- sz = fi_compare_atomicmsg(ep[0], &msg, &cmp_iov, NULL, 1,
- &res_iov, (void **)loc_mr, 1, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((uint64_t *)target) ==
- (uint64_t)((SOURCE_DATA & U32_MASK) |
- (TARGET_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Data mismatch");
- ret = *((uint64_t *)source) ==
- (uint64_t)((TARGET_DATA & U32_MASK) |
- (FETCH_SOURCE_DATA & (U32_MASK << 32)));
- cr_assert(ret, "Fetch data mismatch");
-
- /* float */
- *((float *)&operand_fp) = SOURCE_DATA_FP;
- *((float *)&op2_fp) = TARGET_DATA_FP;
- *((float *)source) = FETCH_SOURCE_DATA;
- *((float *)target) = TARGET_DATA_FP;
- msg_iov.addr = &operand_fp;
- msg.datatype = FI_FLOAT;
- res_iov.addr = source;
- cmp_iov.addr = &op2_fp;
- sz = fi_compare_atomicmsg(ep[0], &msg, &cmp_iov, NULL, 1,
- &res_iov, (void **)loc_mr, 1, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((float *)target) == (float)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((float *)source) == (float)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-
- /* double */
- *((double *)&operand_dp) = SOURCE_DATA_FP;
- *((double *)&op2_dp) = TARGET_DATA_FP;
- *((double *)source) = FETCH_SOURCE_DATA;
- *((double *)target) = TARGET_DATA_FP;
- msg_iov.addr = &operand_dp;
- msg.datatype = FI_DOUBLE;
- res_iov.addr = source;
- cmp_iov.addr = &op2_dp;
- sz = fi_compare_atomicmsg(ep[0], &msg, &cmp_iov, NULL, 1,
- &res_iov, (void **)loc_mr, 1, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- ret = *((double *)target) == (double)SOURCE_DATA_FP;
- cr_assert(ret, "Data mismatch");
- ret = *((double *)source) == (double)TARGET_DATA_FP;
- cr_assert(ret, "Fetch data mismatch");
-}
-
-/******************************************************************************
- *
- * Other
- *
- *****************************************************************************/
-
-Test(rdm_atomic, atomic_err)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- struct fi_cq_err_entry err_cqe;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_atomic_err_inject_enable();
-
- /* i64 */
- *((int64_t *)source) = SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_atomic(ep[0], source, 1,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- FI_INT64, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
- cr_assert_eq(ret, -FI_EAVAIL);
-
- ret = fi_cq_readerr(send_cq[0], &err_cqe, 0);
- cr_assert_eq(ret, 1);
-
- cr_assert((uint64_t)err_cqe.op_context == (uint64_t)target,
- "Bad error context");
- cr_assert(err_cqe.flags == (FI_ATOMIC | FI_WRITE));
- cr_assert(err_cqe.len == 0, "Bad error len");
- cr_assert(err_cqe.buf == 0, "Bad error buf");
- cr_assert(err_cqe.data == 0, "Bad error data");
- cr_assert(err_cqe.tag == 0, "Bad error tag");
- cr_assert(err_cqe.olen == 0, "Bad error olen");
- cr_assert(err_cqe.err == FI_ECANCELED, "Bad error errno");
- cr_assert(err_cqe.prov_errno == GNI_RC_TRANSACTION_ERROR,
- "Bad prov errno");
- cr_assert(err_cqe.err_data == NULL, "Bad error provider data");
-
- w_e[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-}
-
-Test(rdm_atomic, fetch_atomic_err)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t operand = SOURCE_DATA;
- struct fi_cq_err_entry err_cqe;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_atomic_err_inject_enable();
-
- /* i64 */
- *((int64_t *)source) = FETCH_SOURCE_DATA;
- *((int64_t *)target) = TARGET_DATA;
- sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_INT64, FI_MIN, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
- cr_assert_eq(ret, -FI_EAVAIL);
-
- ret = fi_cq_readerr(send_cq[0], &err_cqe, 0);
- cr_assert_eq(ret, 1);
-
- cr_assert((uint64_t)err_cqe.op_context == (uint64_t)target,
- "Bad error context");
- cr_assert(err_cqe.flags == (FI_ATOMIC | FI_READ));
- cr_assert(err_cqe.len == 0, "Bad error len");
- cr_assert(err_cqe.buf == 0, "Bad error buf");
- cr_assert(err_cqe.data == 0, "Bad error data");
- cr_assert(err_cqe.tag == 0, "Bad error tag");
- cr_assert(err_cqe.olen == 0, "Bad error olen");
- cr_assert(err_cqe.err == FI_ECANCELED, "Bad error errno");
- cr_assert(err_cqe.prov_errno == GNI_RC_TRANSACTION_ERROR,
- "Bad prov errno");
- cr_assert(err_cqe.err_data == NULL, "Bad error provider data");
-
- r_e[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-}
-
-Test(rdm_atomic, compare_atomic_err)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t operand = SOURCE_DATA, op2 = TARGET_DATA;
- struct fi_cq_err_entry err_cqe;
- uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_atomic_err_inject_enable();
-
- /* u64 */
- *((uint64_t *)source) = FETCH_SOURCE_DATA;
- *((uint64_t *)target) = TARGET_DATA;
- sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
- source, loc_mr[0], gni_addr[1], (uint64_t)target,
- mr_key[1], FI_UINT64, FI_CSWAP, target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
- cr_assert_eq(ret, -FI_EAVAIL);
-
- ret = fi_cq_readerr(send_cq[0], &err_cqe, 0);
- cr_assert_eq(ret, 1);
-
- cr_assert((uint64_t)err_cqe.op_context == (uint64_t)target,
- "Bad error context");
- cr_assert(err_cqe.flags == (FI_ATOMIC | FI_READ));
- cr_assert(err_cqe.len == 0, "Bad error len");
- cr_assert(err_cqe.buf == 0, "Bad error buf");
- cr_assert(err_cqe.data == 0, "Bad error data");
- cr_assert(err_cqe.tag == 0, "Bad error tag");
- cr_assert(err_cqe.olen == 0, "Bad error olen");
- cr_assert(err_cqe.err == FI_ECANCELED, "Bad error errno");
- cr_assert(err_cqe.prov_errno == GNI_RC_TRANSACTION_ERROR,
- "Bad prov errno");
- cr_assert(err_cqe.err_data == NULL, "Bad error provider data");
-
- r_e[0] = 1;
- rdm_atomic_check_cntrs(w, r, w_e, r_e);
-}
diff --git a/prov/gni/test/rdm_dgram_rma.c b/prov/gni/test/rdm_dgram_rma.c
deleted file mode 100644
index d0cf89c..0000000
--- a/prov/gni/test/rdm_dgram_rma.c
+++ /dev/null
@@ -1,1624 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC.
- * All rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <pthread.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include "gnix_vc.h"
-#include "gnix_cm_nic.h"
-#include "gnix_hashtable.h"
-#include "gnix_rma.h"
-
-#include <criterion/criterion.h>
-
-#if 1
-#define dbg_printf(...)
-#else
-#define dbg_printf(...) \
- do { \
- printf(__VA_ARGS__); \
- fflush(stdout); \
- } while (0)
-#endif
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom[2];
-struct fi_gni_ops_domain *gni_domain_ops[2];
-static struct fid_ep *ep[2];
-static struct fid_av *av[2];
-static struct fi_info *hints;
-static struct fi_info *fi;
-void *ep_name[2];
-size_t gni_addr[2];
-static struct fid_cq *send_cq[2];
-static struct fid_cq *recv_cq[2];
-static struct fi_cq_attr cq_attr[2];
-static struct fid_stx *stx_ctx[2];
-
-#define BUF_SZ (64*1024)
-char *target;
-char *source;
-char *uc_source;
-struct fid_mr *rem_mr[2], *loc_mr[2];
-uint64_t mr_key[2];
-
-static struct fid_cntr *write_cntr[2], *read_cntr[2];
-static struct fi_cntr_attr cntr_attr = {
- .events = FI_CNTR_EVENTS_COMP,
- .flags = 0
-};
-static uint64_t writes[2] = {0}, reads[2] = {0}, write_errs[2] = {0},
- read_errs[2] = {0};
-#define MLOOPS 1000
-static int dgm_fail;
-
-void common_setup(void)
-{
- int ret = 0;
- struct fi_av_attr attr;
- size_t addrlen = 0;
-
- dgm_fail = 0;
-
- hints->domain_attr->cq_data_size = 4;
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- ret = fi_domain(fab, fi, dom, NULL);
- cr_assert(!ret, "fi_domain");
-
- ret = fi_open_ops(&dom[0]->fid, FI_GNI_DOMAIN_OPS_1,
- 0, (void **) gni_domain_ops, NULL);
-
- attr.type = FI_AV_MAP;
- attr.count = 2;
-
- ret = fi_av_open(dom[0], &attr, av, NULL);
- cr_assert(!ret, "fi_av_open");
-
- ret = fi_endpoint(dom[0], fi, &ep[0], NULL);
- cr_assert(!ret, "fi_endpoint");
-
- cq_attr[0].format = FI_CQ_FORMAT_TAGGED;
- cq_attr[0].size = 1024;
- cq_attr[0].wait_obj = 0;
-
- ret = fi_cq_open(dom[0], cq_attr, send_cq, 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_cq_open(dom[0], cq_attr, recv_cq, 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_stx_context(dom[0], NULL, &stx_ctx[0], 0);
- cr_assert(!ret, "fi_stx_context");
-
- ret = fi_domain(fab, fi, dom + 1, NULL);
- cr_assert(!ret, "fi_domain");
-
- ret = fi_open_ops(&dom[1]->fid, FI_GNI_DOMAIN_OPS_1,
- 0, (void **) gni_domain_ops + 1, NULL);
-
- ret = fi_av_open(dom[1], &attr, av + 1, NULL);
- cr_assert(!ret, "fi_av_open");
-
- ret = fi_endpoint(dom[1], fi, &ep[1], NULL);
- cr_assert(!ret, "fi_endpoint");
-
- ret = fi_stx_context(dom[1], NULL, &stx_ctx[1], 0);
- cr_assert(!ret, "fi_stx_context");
-
- cq_attr[1].format = FI_CQ_FORMAT_TAGGED;
- cq_attr[1].size = 1024;
- cq_attr[1].wait_obj = 0;
-
- ret = fi_cq_open(dom[1], cq_attr + 1, send_cq + 1, 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_cq_open(dom[1], cq_attr + 1, recv_cq + 1, 0);
- cr_assert(!ret, "fi_cq_open");
-
- /*
- * imitate shmem, etc. use FI_WRITE for bind
- * flag
- */
- ret = fi_ep_bind(ep[0], &send_cq[0]->fid, FI_TRANSMIT);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_ep_bind(ep[0], &recv_cq[0]->fid, FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_ep_bind(ep[0], &stx_ctx[0]->fid, 0);
- cr_assert(!ret, "fi_ep_bind stx");
-
- /*
- * this shouldn't work, wrond domain
- */
-
- ret = fi_ep_bind(ep[0], &stx_ctx[1]->fid, 0);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
- cr_assert(addrlen > 0);
-
- ep_name[0] = malloc(addrlen);
- cr_assert(ep_name[0] != NULL);
-
- ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_ep_bind(ep[1], &send_cq[1]->fid, FI_TRANSMIT);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_ep_bind(ep[1], &recv_cq[1]->fid, FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_ep_bind(ep[1], &stx_ctx[1]->fid, 0);
- cr_assert(!ret, "fi_ep_bind stx");
-
- ret = fi_getname(&ep[1]->fid, NULL, &addrlen);
- cr_assert(addrlen > 0);
-
- ep_name[1] = malloc(addrlen);
- cr_assert(ep_name[1] != NULL);
-
- ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_av_insert(av[0], ep_name[0], 1, &gni_addr[0], 0,
- NULL);
- cr_assert(ret == 1);
- ret = fi_av_insert(av[0], ep_name[1], 1, &gni_addr[1], 0,
- NULL);
- cr_assert(ret == 1);
-
- ret = fi_av_insert(av[1], ep_name[0], 1, &gni_addr[0], 0,
- NULL);
- cr_assert(ret == 1);
- ret = fi_av_insert(av[1], ep_name[1], 1, &gni_addr[1], 0,
- NULL);
- cr_assert(ret == 1);
-
- ret = fi_ep_bind(ep[0], &av[0]->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_ep_bind(ep[1], &av[1]->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_enable(ep[0]);
- cr_assert(!ret, "fi_ep_enable");
-
- ret = fi_enable(ep[1]);
- cr_assert(!ret, "fi_ep_enable");
-
- target = malloc(BUF_SZ);
- assert(target);
- source = malloc(BUF_SZ);
- assert(source);
-
- ret = fi_mr_reg(dom[0], target, BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, &rem_mr[0], &target);
- cr_assert_eq(ret, 0);
- ret = fi_mr_reg(dom[1], target, BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, &rem_mr[1], &target);
- cr_assert_eq(ret, 0);
-
- ret = fi_mr_reg(dom[0], source, BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, &loc_mr[0], &source);
- cr_assert_eq(ret, 0);
- ret = fi_mr_reg(dom[1], source, BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, &loc_mr[1], &source);
- cr_assert_eq(ret, 0);
-
- uc_source = malloc(BUF_SZ);
- assert(uc_source);
-
- mr_key[0] = fi_mr_key(rem_mr[0]);
- mr_key[1] = fi_mr_key(rem_mr[1]);
-
- ret = fi_cntr_open(dom[0], &cntr_attr, write_cntr, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[0], &write_cntr[0]->fid, FI_WRITE);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_cntr_open(dom[0], &cntr_attr, read_cntr, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[0], &read_cntr[0]->fid, FI_READ);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_cntr_open(dom[1], &cntr_attr, write_cntr + 1, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[1], &write_cntr[1]->fid, FI_WRITE);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_cntr_open(dom[1], &cntr_attr, read_cntr + 1, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[1], &read_cntr[1]->fid, FI_READ);
- cr_assert(!ret, "fi_ep_bind");
-}
-
-void rdm_rma_setup(void)
-{
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
- hints->ep_attr->type = FI_EP_RDM;
- common_setup();
-}
-
-void dgram_setup(void)
-{
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
- hints->ep_attr->type = FI_EP_DGRAM;
- common_setup();
-}
-
-void rdm_rma_teardown(void)
-{
- int ret = 0;
-
- ret = fi_close(&read_cntr[0]->fid);
- cr_assert(!ret, "failure in closing dom[0] read counter.");
-
- ret = fi_close(&read_cntr[1]->fid);
- cr_assert(!ret, "failure in closing dom[1] read counter.");
-
- ret = fi_close(&write_cntr[0]->fid);
- cr_assert(!ret, "failure in closing dom[0] write counter.");
-
- ret = fi_close(&write_cntr[1]->fid);
- cr_assert(!ret, "failure in closing dom[1] write counter.");
-
- free(uc_source);
-
- ret = fi_close(&loc_mr[0]->fid);
- cr_assert(!ret, "failure in closing dom[0] local mr.");
-
- ret = fi_close(&loc_mr[1]->fid);
- cr_assert(!ret, "failure in closing dom[1] local mr.");
-
- ret = fi_close(&rem_mr[0]->fid);
- cr_assert(!ret, "failure in closing dom[0] remote mr.");
-
- ret = fi_close(&rem_mr[1]->fid);
- cr_assert(!ret, "failure in closing dom[1] remote mr.");
-
- ret = fi_close(&stx_ctx[0]->fid);
- cr_assert(!ret, "failure in closing dom[0] stx_ctx.");
-
- ret = fi_close(&stx_ctx[1]->fid);
- cr_assert(!ret, "failure in closing dom[1] stx_ctx.");
-
- free(target);
- free(source);
-
- ret = fi_close(&ep[0]->fid);
- cr_assert(!ret, "failure in closing ep[0].");
-
- ret = fi_close(&ep[1]->fid);
- cr_assert(!ret, "failure in closing ep[1].");
-
- ret = fi_close(&recv_cq[0]->fid);
- cr_assert(!ret, "failure in dom[0] recv cq.");
-
- ret = fi_close(&recv_cq[1]->fid);
- cr_assert(!ret, "failure in dom[1] recv cq.");
-
- ret = fi_close(&send_cq[0]->fid);
- cr_assert(!ret, "failure in dom[0] send cq.");
-
- ret = fi_close(&send_cq[1]->fid);
- cr_assert(!ret, "failure in dom[1] send cq.");
-
- ret = fi_close(&av[0]->fid);
- cr_assert(!ret, "failure in closing dom[0] av.");
-
- ret = fi_close(&av[1]->fid);
- cr_assert(!ret, "failure in closing dom[1] av.");
-
- ret = fi_close(&dom[0]->fid);
- cr_assert(!ret, "failure in closing domain dom[0].");
-
- ret = fi_close(&dom[1]->fid);
- cr_assert(!ret, "failure in closing domain dom[1].");
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
- hints = NULL;
- dgm_fail = 0;
- free(ep_name[0]);
- free(ep_name[1]);
-}
-
-void init_data(char *buf, int len, char seed)
-{
- int i;
-
- for (i = 0; i < len; i++) {
- buf[i] = seed++;
- }
-}
-
-int check_data(char *buf1, char *buf2, int len)
-{
- int i;
-
- for (i = 0; i < len; i++) {
- if (buf1[i] != buf2[i]) {
- printf("data mismatch, elem: %d, b1: 0x%hhx, b2: 0x%hhx, len: %d\n",
- i, buf1[i], buf2[i], len);
- return 0;
- }
- }
-
- return 1;
-}
-
-void rdm_rma_check_tcqe(struct fi_cq_tagged_entry *tcqe, void *ctx,
- uint64_t flags, uint64_t data)
-{
- cr_assert(tcqe->op_context == ctx, "CQE Context mismatch");
- cr_assert(tcqe->flags == flags, "CQE flags mismatch");
-
- if (flags & FI_REMOTE_CQ_DATA) {
- cr_assert(tcqe->data == data, "CQE data invalid");
- } else {
- cr_assert(tcqe->data == 0, "CQE data invalid");
- }
-
- cr_assert(tcqe->len == 0, "CQE length mismatch");
- cr_assert(tcqe->buf == 0, "CQE address mismatch");
- cr_assert(tcqe->tag == 0, "CQE tag invalid");
-}
-
-void rdm_rma_check_cntrs(uint64_t w[2], uint64_t r[2], uint64_t w_e[2],
- uint64_t r_e[2])
-{
- /* Domain 0 */
- writes[0] += w[0];
- reads[0] += r[0];
- write_errs[0] += w_e[0];
- read_errs[0] += r_e[0];
- /*dbg_printf("%ld, %ld\n", fi_cntr_read(write_cntr[0]), writes[0]);*/
- cr_assert(fi_cntr_read(write_cntr[0]) == writes[0], "Bad write count");
- cr_assert(fi_cntr_read(read_cntr[0]) == reads[0], "Bad read count");
- cr_assert(fi_cntr_readerr(write_cntr[0]) == write_errs[0],
- "Bad write err count");
- cr_assert(fi_cntr_readerr(read_cntr[0]) == read_errs[0],
- "Bad read err count");
-
- /* Domain 1 */
- writes[1] += w[1];
- reads[1] += r[1];
- write_errs[1] += w_e[1];
- read_errs[1] += r_e[1];
- cr_assert(fi_cntr_read(write_cntr[1]) == writes[1], "Bad write count");
- cr_assert(fi_cntr_read(read_cntr[1]) == reads[1], "Bad read count");
- cr_assert(fi_cntr_readerr(write_cntr[1]) == write_errs[1],
- "Bad write err count");
- cr_assert(fi_cntr_readerr(read_cntr[1]) == read_errs[1],
- "Bad read err count");
-}
-
-void xfer_for_each_size(void (*xfer)(int len), int slen, int elen)
-{
- int i;
-
- for (i = slen; i <= elen; i *= 2) {
- xfer(i);
- }
-}
-
-void err_inject_enable(void)
-{
- int ret, err_count_val = 1;
-
- ret = gni_domain_ops[0]->set_val(&dom[0]->fid, GNI_ERR_INJECT_COUNT,
- &err_count_val);
- cr_assert(!ret, "setval(GNI_ERR_INJECT_COUNT)");
-
- ret = gni_domain_ops[1]->set_val(&dom[1]->fid, GNI_ERR_INJECT_COUNT,
- &err_count_val);
- cr_assert(!ret, "setval(GNI_ERR_INJECT_COUNT)");
-}
-
-/*******************************************************************************
- * Test RMA functions
- ******************************************************************************/
-
-TestSuite(dgram_rma, .init = dgram_setup, .fini = rdm_rma_teardown,
- .disabled = false);
-
-TestSuite(rdm_rma, .init = rdm_rma_setup, .fini = rdm_rma_teardown,
- .disabled = false);
-
-void do_write(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
- init_data(source, len, 0xab);
- init_data(target, len, 0);
-
- sz = fi_write(ep[0], source, len,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- if (dgm_fail) {
- cr_assert_eq(ret, -FI_EAVAIL);
- return;
- }
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_rma, write)
-{
- xfer_for_each_size(do_write, 8, BUF_SZ);
-}
-
-Test(rdm_rma, write_retrans)
-{
- err_inject_enable();
- xfer_for_each_size(do_write, 8, BUF_SZ);
-}
-
-Test(dgram_rma, write)
-{
- xfer_for_each_size(do_write, 8, BUF_SZ);
-}
-
-Test(dgram_rma, write_retrans)
-{
- dgm_fail = 1;
- err_inject_enable();
- xfer_for_each_size(do_write, 8, BUF_SZ);
-}
-
-void do_writev(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- struct iovec iov;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
- iov.iov_base = source;
- iov.iov_len = len;
-
- init_data(source, len, 0x25);
- init_data(target, len, 0);
-
- sz = fi_writev(ep[0], &iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- if (dgm_fail) {
- cr_assert_eq(ret, -FI_EAVAIL);
- return;
- }
-
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_rma, writev)
-{
- xfer_for_each_size(do_writev, 8, BUF_SZ);
-}
-
-Test(rdm_rma, writev_retrans)
-{
- err_inject_enable();
- xfer_for_each_size(do_writev, 8, BUF_SZ);
-}
-
-Test(dgram_rma, writev)
-{
- xfer_for_each_size(do_writev, 8, BUF_SZ);
-}
-
-Test(dgram_rma, writev_retrans)
-{
- dgm_fail = 1;
- err_inject_enable();
- xfer_for_each_size(do_writev, 8, BUF_SZ);
-}
-
-void do_writemsg(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- struct iovec iov;
- struct fi_msg_rma msg;
- struct fi_rma_iov rma_iov;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
- iov.iov_base = source;
- iov.iov_len = len;
-
- rma_iov.addr = (uint64_t)target;
- rma_iov.len = len;
- rma_iov.key = mr_key[1];
-
- msg.msg_iov = &iov;
- msg.desc = (void **)loc_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[1];
- msg.rma_iov = &rma_iov;
- msg.rma_iov_count = 1;
- msg.context = target;
- msg.data = (uint64_t)target;
-
- init_data(source, len, 0xef);
- init_data(target, len, 0);
- sz = fi_writemsg(ep[0], &msg, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- if (dgm_fail) {
- cr_assert_eq(ret, -FI_EAVAIL);
- return;
- }
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_rma, writemsg)
-{
- xfer_for_each_size(do_writemsg, 8, BUF_SZ);
-}
-
-Test(rdm_rma, writemsg_retrans)
-{
- err_inject_enable();
- xfer_for_each_size(do_writemsg, 8, BUF_SZ);
-}
-
-Test(dgram_rma, writemsg)
-{
- xfer_for_each_size(do_writemsg, 8, BUF_SZ);
-}
-
-Test(dgram_rma, writemsg_retrans)
-{
- dgm_fail = 1;
- err_inject_enable();
- xfer_for_each_size(do_writemsg, 8, BUF_SZ);
-}
-
-/*
- * write_fence should be validated by inspecting debug.
- *
- * The following sequence of events should be seen:
- *
- * TX request processed: A
- * TX request queue stalled on FI_FENCE request: B
- * Added event: A
- * TX request processed: B
- *
- */
-
-void do_write_fence(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- struct iovec iov;
- struct fi_msg_rma msg;
- struct fi_rma_iov rma_iov;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
- iov.iov_base = source;
- iov.iov_len = len;
-
- rma_iov.addr = (uint64_t)target;
- rma_iov.len = sizeof(target);
- rma_iov.key = mr_key[1];
-
- msg.msg_iov = &iov;
- msg.desc = (void **)loc_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[1];
- msg.rma_iov = &rma_iov;
- msg.rma_iov_count = 1;
- msg.context = target;
- msg.data = (uint64_t)target;
-
- init_data(source, len, 0xef);
- init_data(target, len, 0);
-
- /* write A */
- sz = fi_writemsg(ep[0], &msg, 0);
- cr_assert_eq(sz, 0);
-
- /* write B */
- sz = fi_writemsg(ep[0], &msg, FI_FENCE);
- cr_assert_eq(sz, 0);
-
- /* event A */
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- if (dgm_fail) {
- cr_assert_eq(ret, -FI_EAVAIL);
- return;
- }
-
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
-
- /* event B */
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
-
- w[0] = 2;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_rma, write_fence)
-{
- xfer_for_each_size(do_write_fence, 8, BUF_SZ);
-}
-
-Test(rdm_rma, write_fence_retrans)
-{
- err_inject_enable();
- xfer_for_each_size(do_write_fence, 8, BUF_SZ);
-}
-
-Test(dgram_rma, write_fence)
-{
- xfer_for_each_size(do_write_fence, 8, BUF_SZ);
-}
-
-Test(dgram_rma, write_fence_retrans)
-{
- dgm_fail = 1;
- err_inject_enable();
- xfer_for_each_size(do_write_fence, 8, BUF_SZ);
-}
-
-#define INJECT_SIZE 64
-void do_inject_write(int len)
-{
- ssize_t sz;
- int ret, i, loops = 0;
- struct fi_cq_tagged_entry cqe;
-
- init_data(source, len, 0x23);
- init_data(target, len, 0);
- sz = fi_inject_write(ep[0], source, len,
- gni_addr[1], (uint64_t)target, mr_key[1]);
- cr_assert_eq(sz, 0);
-
- for (i = 0; i < len; i++) {
- loops = 0;
- while (source[i] != target[i]) {
- /* for progress */
- ret = fi_cq_read(send_cq[0], &cqe, 1);
- cr_assert(ret == -FI_EAGAIN || ret == -FI_EAVAIL,
- "Received unexpected event\n");
-
- pthread_yield();
- cr_assert(++loops < MLOOPS || dgm_fail,
- "Data mismatch");
- if (dgm_fail && loops > MLOOPS)
- break;
- }
- }
- cr_assert(!dgm_fail || (dgm_fail && loops >= MLOOPS), "Should fail");
-}
-
-Test(rdm_rma, inject_write)
-{
- xfer_for_each_size(do_inject_write, 8, INJECT_SIZE);
-}
-
-Test(rdm_rma, inject_write_retrans)
-{
- err_inject_enable();
- xfer_for_each_size(do_inject_write, 8, INJECT_SIZE);
-}
-
-Test(dgram_rma, inject_write)
-{
- xfer_for_each_size(do_inject_write, 8, INJECT_SIZE);
-}
-
-Test(dgram_rma, inject_write_retrans)
-{
- dgm_fail = 1;
- err_inject_enable();
- xfer_for_each_size(do_inject_write, 8, INJECT_SIZE);
-}
-
-void do_writedata(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe, dcqe;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
-
-#define WRITE_DATA 0x5123da1a145
- init_data(source, len, 0x23);
- init_data(target, len, 0);
- sz = fi_writedata(ep[0], source, len, loc_mr[0], WRITE_DATA,
- gni_addr[1], (uint64_t)target, mr_key[1],
- target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- if (dgm_fail) {
- cr_assert_eq(ret, -FI_EAVAIL);
- return;
- }
-
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-
- while ((ret = fi_cq_read(recv_cq[1], &dcqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
- cr_assert(ret != FI_SUCCESS, "Missing remote data");
-
- rdm_rma_check_tcqe(&dcqe, NULL,
- (FI_RMA | FI_REMOTE_WRITE | FI_REMOTE_CQ_DATA),
- WRITE_DATA);
-}
-
-Test(rdm_rma, writedata)
-{
- xfer_for_each_size(do_writedata, 8, BUF_SZ);
-}
-
-Test(rdm_rma, writedata_retrans)
-{
- err_inject_enable();
- xfer_for_each_size(do_writedata, 8, BUF_SZ);
-}
-
-Test(dgram_rma, writedata)
-{
- xfer_for_each_size(do_writedata, 8, BUF_SZ);
-}
-
-Test(dgram_rma, writedata_retrans)
-{
- dgm_fail = 1;
- err_inject_enable();
- xfer_for_each_size(do_writedata, 8, BUF_SZ);
-}
-
-#define INJECTWRITE_DATA 0xdededadadeadbeaf
-void do_inject_writedata(int len)
-{
- ssize_t sz;
- int ret, i, loops = 0;
- struct fi_cq_tagged_entry cqe, dcqe;
-
- init_data(source, len, 0x23);
- init_data(target, len, 0);
- sz = fi_inject_writedata(ep[0], source, len, INJECTWRITE_DATA,
- gni_addr[1], (uint64_t)target, mr_key[1]);
- cr_assert_eq(sz, 0);
-
- for (i = 0; i < len; i++) {
- loops = 0;
- while (source[i] != target[i]) {
- /* for progress */
- ret = fi_cq_read(send_cq[0], &cqe, 1);
- cr_assert(ret == -FI_EAGAIN || ret == -FI_EAVAIL,
- "Received unexpected event\n");
-
- pthread_yield();
- cr_assert(++loops < MLOOPS || dgm_fail,
- "Data mismatch");
- if (dgm_fail && loops > MLOOPS)
- break;
- }
- }
- cr_assert(!dgm_fail || (dgm_fail && loops >= MLOOPS), "Should fail");
- if (dgm_fail && loops >= MLOOPS)
- return;
-
- while ((ret = fi_cq_read(recv_cq[1], &dcqe, 1)) == -FI_EAGAIN) {
- ret = fi_cq_read(send_cq[0], &cqe, 1); /* for progress */
- pthread_yield();
- }
- cr_assert(ret != FI_SUCCESS, "Missing remote data");
-
- rdm_rma_check_tcqe(&dcqe, NULL,
- (FI_RMA | FI_REMOTE_WRITE | FI_REMOTE_CQ_DATA),
- INJECTWRITE_DATA);
-}
-
-Test(rdm_rma, inject_writedata)
-{
- xfer_for_each_size(do_inject_writedata, 8, INJECT_SIZE);
-}
-
-Test(rdm_rma, inject_writedata_retrans)
-{
- err_inject_enable();
- xfer_for_each_size(do_inject_writedata, 8, INJECT_SIZE);
-}
-
-Test(dgram_rma, inject_writedata)
-{
- xfer_for_each_size(do_inject_writedata, 8, INJECT_SIZE);
-}
-
-Test(dgram_rma, inject_writedata_retrans)
-{
- dgm_fail = 1;
- err_inject_enable();
- xfer_for_each_size(do_inject_writedata, 8, INJECT_SIZE);
-}
-
-void do_read(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
-#define READ_CTX 0x4e3dda1aULL
- init_data(source, len, 0);
- init_data(target, len, 0xad);
-
- /* domain 0 from domain 1 */
- sz = fi_read(ep[0], source, len,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- (void *)READ_CTX);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, (void *)READ_CTX, FI_RMA | FI_READ, 0);
-
- r[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got read context event!\n");
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_rma, read)
-{
- xfer_for_each_size(do_read, 8, BUF_SZ);
-}
-
-Test(rdm_rma, read_retrans)
-{
- err_inject_enable();
- xfer_for_each_size(do_read, 8, BUF_SZ);
-}
-
-Test(dgram_rma, read)
-{
- xfer_for_each_size(do_read, 8, BUF_SZ);
-}
-
-void do_readv(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- struct iovec iov;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
- iov.iov_base = source;
- iov.iov_len = len;
-
- init_data(target, len, 0x25);
- init_data(source, len, 0);
- sz = fi_readv(ep[0], &iov, (void **)loc_mr, 1,
- gni_addr[1], (uint64_t)target, mr_key[1],
- target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_READ, 0);
-
- r[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_rma, readv)
-{
- xfer_for_each_size(do_readv, 8, BUF_SZ);
-}
-
-Test(rdm_rma, readv_retrans)
-{
- err_inject_enable();
- xfer_for_each_size(do_readv, 8, BUF_SZ);
-}
-
-Test(dgram_rma, readv)
-{
- xfer_for_each_size(do_readv, 8, BUF_SZ);
-}
-
-void do_readmsg(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- struct iovec iov;
- struct fi_msg_rma msg;
- struct fi_rma_iov rma_iov;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
- iov.iov_base = source;
- iov.iov_len = len;
-
- rma_iov.addr = (uint64_t)target;
- rma_iov.len = len;
- rma_iov.key = mr_key[1];
-
- msg.msg_iov = &iov;
- msg.desc = (void **)loc_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[1];
- msg.rma_iov = &rma_iov;
- msg.rma_iov_count = 1;
- msg.context = target;
- msg.data = (uint64_t)target;
-
- init_data(target, len, 0xef);
- init_data(source, len, 0);
- sz = fi_readmsg(ep[0], &msg, 0);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_READ, 0);
-
- r[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-
- iov.iov_base = source;
- iov.iov_len = len;
-
- rma_iov.addr = (uint64_t)target;
- rma_iov.len = len;
- rma_iov.key = mr_key[0];
-
- msg.msg_iov = &iov;
- msg.desc = (void **)(loc_mr + 1);
- msg.iov_count = 1;
- msg.addr = gni_addr[0];
- msg.rma_iov = &rma_iov;
- msg.rma_iov_count = 1;
- msg.context = target;
- msg.data = (uint64_t)target;
-}
-
-Test(rdm_rma, readmsg)
-{
- xfer_for_each_size(do_readmsg, 8, BUF_SZ);
-}
-
-Test(rdm_rma, readmsg_retrans)
-{
- err_inject_enable();
- xfer_for_each_size(do_readmsg, 8, BUF_SZ);
-}
-
-Test(dgram_rma, readmsg)
-{
- xfer_for_each_size(do_readmsg, 8, BUF_SZ);
-}
-
-#define READ_DATA 0xdededadadeaddeef
-void do_readmsgdata(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe, dcqe;
- struct iovec iov;
- struct fi_msg_rma msg;
- struct fi_rma_iov rma_iov;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
- iov.iov_base = source;
- iov.iov_len = len;
-
- rma_iov.addr = (uint64_t)target;
- rma_iov.len = len;
- rma_iov.key = mr_key[1];
-
- msg.msg_iov = &iov;
- msg.desc = (void **)loc_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[1];
- msg.rma_iov = &rma_iov;
- msg.rma_iov_count = 1;
- msg.context = target;
- msg.data = (uint64_t)READ_DATA;
-
- init_data(target, len, 0xef);
- init_data(source, len, 0);
- sz = fi_readmsg(ep[0], &msg, FI_REMOTE_CQ_DATA);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_READ, 0);
-
- r[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-
- while ((ret = fi_cq_read(recv_cq[1], &dcqe, 1)) == -FI_EAGAIN) {
- ret = fi_cq_read(send_cq[0], &cqe, 1); /* for progress */
- pthread_yield();
- }
-
- cr_assert(ret != FI_SUCCESS, "Missing remote data");
-
- rdm_rma_check_tcqe(&dcqe, NULL,
- (FI_RMA | FI_REMOTE_READ | FI_REMOTE_CQ_DATA),
- READ_DATA);
-}
-
-Test(rdm_rma, readmsgdata)
-{
- xfer_for_each_size(do_readmsgdata, 8, BUF_SZ);
-}
-
-Test(rdm_rma, readmsgdata_retrans)
-{
- err_inject_enable();
- xfer_for_each_size(do_readmsgdata, 8, BUF_SZ);
-}
-
-Test(dgram_rma, readmsgdata)
-{
- xfer_for_each_size(do_readmsgdata, 8, BUF_SZ);
-}
-
-void inject_common(void)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- struct iovec iov;
- struct fi_msg_rma msg;
- struct fi_rma_iov rma_iov;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
- iov.iov_base = source;
- iov.iov_len = GNIX_INJECT_SIZE;
-
- rma_iov.addr = (uint64_t)target;
- rma_iov.len = GNIX_INJECT_SIZE;
- rma_iov.key = mr_key[1];
-
- msg.msg_iov = &iov;
- msg.desc = (void **)loc_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[1];
- msg.rma_iov = &rma_iov;
- msg.rma_iov_count = 1;
- msg.context = target;
- msg.data = (uint64_t)target;
-
- init_data(source, GNIX_INJECT_SIZE, 0xef);
- init_data(target, GNIX_INJECT_SIZE, 0);
-
- sz = fi_writemsg(ep[0], &msg, FI_INJECT);
- cr_assert_eq(sz, 0);
-
- iov.iov_len = GNIX_INJECT_SIZE+1;
- sz = fi_writemsg(ep[0], &msg, FI_INJECT);
- cr_assert_eq(sz, -FI_EINVAL);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- cr_assert(check_data(source, target, GNIX_INJECT_SIZE),
- "Data mismatch");
-}
-
-Test(rdm_rma, inject)
-{
- inject_common();
-}
-
-Test(dgram_rma, inject)
-{
- inject_common();
-}
-
-void do_write_autoreg(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
- init_data(source, len, 0xab);
- init_data(target, len, 0);
- sz = fi_write(ep[0], source, len,
- NULL, gni_addr[1], (uint64_t)target, mr_key[1],
- target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- cr_assert(check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_rma, write_autoreg)
-{
- xfer_for_each_size(do_write_autoreg, 8, BUF_SZ);
-}
-
-Test(dgram_rma, write_autoreg)
-{
- xfer_for_each_size(do_write_autoreg, 8, BUF_SZ);
-}
-
-void do_write_autoreg_uncached(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
- init_data(uc_source, len, 0xab);
- init_data(target, len, 0);
- sz = fi_write(ep[0], uc_source, len,
- NULL, gni_addr[1], (uint64_t)target, mr_key[1],
- target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- cr_assert(check_data(uc_source, target, len), "Data mismatch");
-}
-
-Test(rdm_rma, write_autoreg_uncached)
-{
- xfer_for_each_size(do_write_autoreg_uncached, 8, BUF_SZ);
-}
-
-Test(dgram_rma, write_autoreg_uncached)
-{
- xfer_for_each_size(do_write_autoreg_uncached, 8, BUF_SZ);
-}
-
-void do_write_error(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- struct fi_cq_err_entry err_cqe;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
- init_data(source, len, 0xab);
- init_data(target, len, 0);
- sz = fi_write(ep[0], source, len,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, -FI_EAVAIL);
-
- ret = fi_cq_readerr(send_cq[0], &err_cqe, 0);
- cr_assert_eq(ret, 1);
-
- cr_assert((uint64_t)err_cqe.op_context == (uint64_t)target,
- "Bad error context");
- cr_assert(err_cqe.flags == (FI_RMA | FI_WRITE));
- cr_assert(err_cqe.len == 0, "Bad error len");
- cr_assert(err_cqe.buf == 0, "Bad error buf");
- cr_assert(err_cqe.data == 0, "Bad error data");
- cr_assert(err_cqe.tag == 0, "Bad error tag");
- cr_assert(err_cqe.olen == 0, "Bad error olen");
- cr_assert(err_cqe.err == FI_ECANCELED, "Bad error errno");
- cr_assert(err_cqe.prov_errno == GNI_RC_TRANSACTION_ERROR,
- "Bad prov errno");
- cr_assert(err_cqe.err_data == NULL, "Bad error provider data");
-
- w_e[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-}
-
-Test(rdm_rma, write_error)
-{
- int ret, max_retrans_val = 1;
-
- ret = gni_domain_ops[0]->set_val(&dom[0]->fid, GNI_MAX_RETRANSMITS,
- &max_retrans_val);
- cr_assert(!ret, "setval(GNI_MAX_RETRANSMITS)");
-
- ret = gni_domain_ops[1]->set_val(&dom[1]->fid, GNI_MAX_RETRANSMITS,
- &max_retrans_val);
- cr_assert(!ret, "setval(GNI_MAX_RETRANSMITS)");
- err_inject_enable();
-
- xfer_for_each_size(do_write_error, 8, BUF_SZ);
-}
-
-Test(dgram_rma, write_error)
-{
- int ret, max_retrans_val = 1;
-
- ret = gni_domain_ops[0]->set_val(&dom[0]->fid, GNI_MAX_RETRANSMITS,
- &max_retrans_val);
- cr_assert(!ret, "setval(GNI_MAX_RETRANSMITS)");
-
- ret = gni_domain_ops[1]->set_val(&dom[1]->fid, GNI_MAX_RETRANSMITS,
- &max_retrans_val);
- cr_assert(!ret, "setval(GNI_MAX_RETRANSMITS)");
- err_inject_enable();
-
- xfer_for_each_size(do_write_error, 8, BUF_SZ);
-}
-
-void do_read_error(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- struct fi_cq_err_entry err_cqe;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
- init_data(source, len, 0);
- init_data(target, len, 0xad);
- sz = fi_read(ep[0], source, len,
- loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
- (void *)READ_CTX);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, -FI_EAVAIL);
-
- ret = fi_cq_readerr(send_cq[0], &err_cqe, 0);
- cr_assert_eq(ret, 1);
-
- cr_assert((uint64_t)err_cqe.op_context == (uint64_t)READ_CTX,
- "Bad error context");
- cr_assert(err_cqe.flags == (FI_RMA | FI_READ));
- cr_assert(err_cqe.len == 0, "Bad error len");
- cr_assert(err_cqe.buf == 0, "Bad error buf");
- cr_assert(err_cqe.data == 0, "Bad error data");
- cr_assert(err_cqe.tag == 0, "Bad error tag");
- cr_assert(err_cqe.olen == 0, "Bad error olen");
- cr_assert(err_cqe.err == FI_ECANCELED, "Bad error errno");
- cr_assert(err_cqe.prov_errno == GNI_RC_TRANSACTION_ERROR,
- "Bad prov errno");
- cr_assert(err_cqe.err_data == NULL, "Bad error provider data");
-
- r_e[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-}
-
-Test(rdm_rma, read_error)
-{
- int ret, max_retrans_val = 1;
-
- ret = gni_domain_ops[0]->set_val(&dom[0]->fid, GNI_MAX_RETRANSMITS,
- &max_retrans_val);
- cr_assert(!ret, "setval(GNI_MAX_RETRANSMITS)");
-
- ret = gni_domain_ops[1]->set_val(&dom[1]->fid, GNI_MAX_RETRANSMITS,
- &max_retrans_val);
- cr_assert(!ret, "setval(GNI_MAX_RETRANSMITS)");
- err_inject_enable();
-
- xfer_for_each_size(do_read_error, 8, BUF_SZ);
-}
-
-void do_read_buf(void *s, void *t, int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
-#define READ_CTX 0x4e3dda1aULL
- init_data(s, len, 0);
- init_data(t, len, 0xad);
- sz = fi_read(ep[0], s, len, NULL, gni_addr[1], (uint64_t)t, mr_key[1],
- (void *)READ_CTX);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, (void *)READ_CTX, FI_RMA | FI_READ, 0);
-
- r[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got read context event!\n");
-
- cr_assert(check_data(s, t, len), "Data mismatch");
-}
-
-void do_read_alignment(int len)
-{
- int s_off, t_off, l_off;
-
- for (s_off = 0; s_off < 7; s_off++) {
- for (t_off = 0; t_off < 7; t_off++) {
- for (l_off = 0; l_off < 7; l_off++) {
- do_read_buf(source + s_off,
- target + t_off,
- len + l_off);
- }
- }
- }
-}
-
-Test(rdm_rma, read_alignment)
-{
- xfer_for_each_size(do_read_alignment, 1, (BUF_SZ - 1));
-}
-
-Test(rdm_rma, read_alignment_retrans)
-{
- err_inject_enable();
- xfer_for_each_size(do_read_alignment, 1, (BUF_SZ - 1));
-}
-
-Test(dgram_rma, read_alignment)
-{
- xfer_for_each_size(do_read_alignment, 1, (BUF_SZ - 1));
-}
-
-void do_write_buf(void *s, void *t, int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
-
- init_data(s, len, 0xab);
- init_data(t, len, 0);
- sz = fi_write(ep[0], s, len, NULL, gni_addr[1], (uint64_t)t, mr_key[1],
- t);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- if (dgm_fail) {
- cr_assert_eq(ret, -FI_EAVAIL);
- return;
- }
-
- cr_assert_eq(ret, 1);
- rdm_rma_check_tcqe(&cqe, t, FI_RMA | FI_WRITE, 0);
-
- w[0] = 1;
- rdm_rma_check_cntrs(w, r, w_e, r_e);
-
- dbg_printf("got write context event!\n");
-
- cr_assert(check_data(s, t, len), "Data mismatch");
-}
-
-void do_write_alignment(int len)
-{
- int s_off, t_off, l_off;
-
- for (s_off = 0; s_off < 7; s_off++) {
- for (t_off = 0; t_off < 7; t_off++) {
- for (l_off = 0; l_off < 7; l_off++) {
- do_write_buf(source + s_off,
- target + t_off,
- len + l_off);
- }
- }
- }
-}
-
-Test(rdm_rma, write_alignment)
-{
- xfer_for_each_size(do_write_alignment, 1, (BUF_SZ - 1));
-}
-
-Test(rdm_rma, write_alignment_retrans)
-{
- err_inject_enable();
- xfer_for_each_size(do_write_alignment, 1, (BUF_SZ - 1));
-}
-
-Test(dgram_rma, write_alignment)
-{
- xfer_for_each_size(do_write_alignment, 1, (BUF_SZ - 1));
-}
-
-Test(dgram_rma, write_alignment_retrans)
-{
- dgm_fail = 1;
- err_inject_enable();
- xfer_for_each_size(do_write_alignment, 1, (BUF_SZ - 1));
-}
diff --git a/prov/gni/test/rdm_fi_pcd_trecv_msg.c b/prov/gni/test/rdm_fi_pcd_trecv_msg.c
deleted file mode 100644
index 6694ae2..0000000
--- a/prov/gni/test/rdm_fi_pcd_trecv_msg.c
+++ /dev/null
@@ -1,1884 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <pthread.h>
-#include <sys/time.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-#include <rdma/fi_tagged.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include "gnix_vc.h"
-#include "gnix_cm_nic.h"
-#include "gnix_hashtable.h"
-#include "gnix_rma.h"
-
-#include <criterion/criterion.h>
-
-/* Both the send and recv paths use independent state machines within
- * each test to simulate the behavior you would expect in a client/server
- * or independent peer environment
- *
- * Below are the valid states, but state transitions may differ from test
- * to test. To simplify the reading of each test, it is recommended that
- * the state transitions in the test are written in order.
- *
- * Ex.
- * switch (send_state) {
- * case STATE_1:
- * case STATE_2:
- * case STATE_3:
- * ....
- * case STATE_DONE:
- * default:
- * }
- *
- * Normal switch semantics should be followed and no tag should be reused
- * within the same test. Duplicates will trigger an infinite loop.
- */
-
-#define stringify(x) helper(x)
-#define helper(x) #x
-
-#define STATE_DEF(x) [x] = stringify(x)
-
-/* sender states */
-enum send_state {
- S_STATE_SEND_MSG_1 = 0, /* send the first message */
- S_STATE_SEND_MSG_1_WAIT_CQ, /* wait for the first completion */
- S_STATE_SEND_MSG_2, /* send the second message */
- S_STATE_SEND_MSG_2_WAIT_CQ, /* wait for the second completion */
- S_STATE_DONE, /* sender is done */
- S_STATE_INVALID, /* invalid state used for test init */
-};
-
-__attribute__((unused))
-static char *send_state_strings[S_STATE_INVALID + 1] = {
- STATE_DEF(S_STATE_SEND_MSG_1),
- STATE_DEF(S_STATE_SEND_MSG_1_WAIT_CQ),
- STATE_DEF(S_STATE_SEND_MSG_2),
- STATE_DEF(S_STATE_SEND_MSG_2_WAIT_CQ),
- STATE_DEF(S_STATE_DONE),
- STATE_DEF(S_STATE_INVALID),
-};
-
-/* receiver states */
-enum recv_state {
- R_STATE_PEEK = 0, /* peek a message */
- R_STATE_PEEK_WAIT_CQ, /* wait for cqe from peek */
- /* wait for error cq */
- R_STATE_PEEK_WAIT_ERR_CQ,
- R_STATE_PEEK_CLAIM, /* peek|claim a message */
- R_STATE_PEEK_CLAIM_WAIT_CQ, /* wait for cqe from peek|claim */
- /* wait for error cq */
- R_STATE_PEEK_CLAIM_WAIT_ERR_CQ,
- R_STATE_PEEK_DISCARD, /* peek|discard a message */
- R_STATE_PEEK_DISCARD_WAIT_CQ, /* wait for cqe from peek|discard */
- /* wait for error cq */
- R_STATE_PEEK_DISCARD_WAIT_ERR_CQ,
- R_STATE_CLAIM, /* claim a message */
- R_STATE_CLAIM_WAIT_CQ, /* wait for cqe from claim */
- R_STATE_CLAIM_DISCARD, /* claim|discard a message */
- R_STATE_CLAIM_DISCARD_WAIT_CQ,
- R_STATE_RECV_MSG_1, /* recv first message */
- R_STATE_RECV_MSG_1_WAIT_CQ, /* wait for cqe from first recv message */
- R_STATE_RECV_MSG_2, /* recv second message */
- R_STATE_RECV_MSG_2_WAIT_CQ, /* wait for cqe from second recv message*/
- R_STATE_DONE, /* receiver is done */
- R_STATE_INVALID, /* invalid state used for test init */
-};
-
-__attribute__((unused))
-static char *recv_state_strings[R_STATE_INVALID + 1] = {
- STATE_DEF(R_STATE_PEEK),
- STATE_DEF(R_STATE_PEEK_WAIT_CQ),
- STATE_DEF(R_STATE_PEEK_WAIT_ERR_CQ),
- STATE_DEF(R_STATE_PEEK_CLAIM),
- STATE_DEF(R_STATE_PEEK_CLAIM_WAIT_CQ),
- STATE_DEF(R_STATE_PEEK_CLAIM_WAIT_ERR_CQ),
- STATE_DEF(R_STATE_PEEK_DISCARD),
- STATE_DEF(R_STATE_PEEK_DISCARD_WAIT_CQ),
- STATE_DEF(R_STATE_PEEK_DISCARD_WAIT_ERR_CQ),
- STATE_DEF(R_STATE_CLAIM),
- STATE_DEF(R_STATE_CLAIM_WAIT_CQ),
- STATE_DEF(R_STATE_CLAIM_DISCARD),
- STATE_DEF(R_STATE_RECV_MSG_1),
- STATE_DEF(R_STATE_RECV_MSG_1_WAIT_CQ),
- STATE_DEF(R_STATE_RECV_MSG_2),
- STATE_DEF(R_STATE_RECV_MSG_2_WAIT_CQ),
- STATE_DEF(R_STATE_DONE),
- STATE_DEF(R_STATE_INVALID),
-};
-
-/* poll the SCQ with a dummy CQE if true */
-#define SHOULD_BLIND_POLL_SCQ(state) \
- ((state) == S_STATE_DONE)
-
-/* poll the RCQ with a dummy CQE if true */
-#define SHOULD_BLIND_POLL_RCQ(state) \
- ((state) != R_STATE_PEEK_WAIT_CQ && \
- state != R_STATE_PEEK_CLAIM_WAIT_CQ && \
- state != R_STATE_PEEK_DISCARD_WAIT_CQ && \
- state != R_STATE_CLAIM_WAIT_CQ && \
- state != R_STATE_CLAIM_DISCARD_WAIT_CQ && \
- state != R_STATE_RECV_MSG_1_WAIT_CQ && \
- state != R_STATE_RECV_MSG_2_WAIT_CQ && \
- state != R_STATE_PEEK_WAIT_ERR_CQ && \
- state != R_STATE_PEEK_CLAIM_WAIT_ERR_CQ && \
- state != R_STATE_PEEK_DISCARD_WAIT_ERR_CQ)
-
-#define RAISE_UNREACHABLE_STATE cr_assert_eq(0, 1, "unreachable state")
-#define ASSERT_SEND_RECV_DONE \
- do { \
- if (!SEND_RECV_DONE) \
- dbg_printf("failed on test size %d\n", len); \
- if (s_state != S_STATE_DONE) \
- dbg_printf("failed to finish send side, " \
- "state=%s\n", \
- send_state_strings[s_state]); \
- if (r_state != R_STATE_DONE) \
- dbg_printf("failed to finish recv side, " \
- "state=%s\n", \
- recv_state_strings[r_state]); \
- cr_assert_eq(s_state, S_STATE_DONE); \
- cr_assert_eq(r_state, R_STATE_DONE); \
- } while (0)
-
-#define PROGRESS_CQS(cqs) \
- do { \
- struct fi_cq_tagged_entry trash; \
- int __events = 0; \
- if (SHOULD_BLIND_POLL_SCQ(s_state)) { \
- __events = fi_cq_read(cqs[0], &trash, 1); \
- if (__events == 1 && rate_limit()) \
- dbg_printf("found unexpected events: scq\n"); \
- } \
- if (SHOULD_BLIND_POLL_RCQ(r_state)) { \
- __events = fi_cq_read(cqs[1], &trash, 1); \
- if (__events == 1 && rate_limit()) \
- dbg_printf("found unexpected events: rcq\n"); \
- } \
- } while (0)
-
-#define __STATE_TRANSITION(state, next_state, labels) \
- do { \
- if ((state) != (next_state)) \
- dbg_printf(stringify(state) " transition: %s to %s\n", \
- labels[state], labels[next_state]); \
- (state) = (next_state); \
- } while (0)
-
-#define SEND_STATE_TRANSITION(next_state) \
- __STATE_TRANSITION(s_state, next_state, send_state_strings)
-
-#define RECV_STATE_TRANSITION(next_state) \
- __STATE_TRANSITION(r_state, next_state, recv_state_strings)
-
-#define __COND_STATE_TRANSITION(cond, state, true_state, false_state, labels) \
- __STATE_TRANSITION(state, ((cond) ? (true_state) : (false_state)), labels)
-
-#define COND_RECV_STATE_TRANSITION(actual, expected, true_state, false_state) \
- __COND_STATE_TRANSITION(((actual) == (expected)), \
- r_state, true_state, false_state, recv_state_strings)
-
-#define COND_SEND_STATE_TRANSITION(actual, expected, true_state, false_state) \
- __COND_STATE_TRANSITION(((actual) == (expected)), \
- s_state, true_state, false_state, send_state_strings)
-
-#define INIT_TEST_STATE(send_state, recv_state) \
- do { \
- SEND_STATE_TRANSITION(send_state); \
- RECV_STATE_TRANSITION(recv_state); \
- } while (0)
-
-#define SEND_RECV_DONE (r_state == R_STATE_DONE && s_state == S_STATE_DONE)
-
-#define TEST_TIME_LIMIT_EXPIRED (elapsed >= max_test_time)
-
-#if 1
-#define dbg_printf(...) do { } while (0)
-#else
-#define dbg_printf(...) \
- do { \
- fprintf(stderr, __VA_ARGS__); \
- fflush(stderr); \
- } while (0)
-#endif
-
-struct timeval begin, end;
-struct timeval loop_start, loop_end;
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom;
-static struct fid_ep *ep[2];
-static struct fid_av *av;
-static struct fi_info *hints;
-static struct fi_info *fi;
-static void *ep_name[2];
-static size_t gni_addr[2];
-static struct fid_cq *msg_cq[2];
-static struct fi_cq_attr cq_attr;
-
-#define BUF_SZ (64*1024)
-static char *target;
-static char *source;
-static struct fid_mr *rem_mr, *loc_mr;
-static uint64_t mr_key;
-static const int max_test_time = 5;
-
-/* test variables */
-static int elapsed;
-enum send_state s_state;
-enum recv_state r_state;
-struct fi_cq_tagged_entry d_peek_cqe;
-struct fi_cq_err_entry cqe_error;
-struct fi_msg_tagged peek_msg;
-struct iovec peek_iov;
-char *peek_buffer;
-
-
-static int elapsed_seconds(struct timeval *s, struct timeval *e)
-{
- /* rough estimate... I don't care that this is accurate */
- int seconds = e->tv_sec - s->tv_sec;
-
- if (!seconds)
- return seconds;
-
- if (e->tv_usec <= s->tv_usec)
- seconds -= 1;
-
- return seconds;
-}
-
-static int rate_limit(void)
-{
- static struct timeval _rl_begin, _rl_end;
- static int init, squelched, events;
- int ret = 0;
-
- if (!init)
- gettimeofday(&_rl_begin, NULL);
-
- gettimeofday(&_rl_end, NULL);
-
- if (events < ((elapsed_seconds(&_rl_begin, &_rl_end) + 1) * 20)) {
- ret = 1;
- events++;
- }
-
- if (ret && squelched) {
- dbg_printf("squelched %d similar events\n", squelched);
- squelched = 0;
- } else if (!ret) {
- squelched++;
- }
-
- return ret;
-}
-
-static inline void init_test_variables(void)
-{
- /* re-initialize all variables necessary here */
- elapsed = 0;
- r_state = R_STATE_INVALID;
- s_state = S_STATE_INVALID;
-
- memset(&d_peek_cqe, 0, sizeof(struct fi_cq_tagged_entry));
- memset(&peek_msg, 0, sizeof(struct fi_msg_tagged));
- memset(&peek_iov, 0, sizeof(struct iovec));
-}
-
-static void rdm_fi_pdc_setup(void)
-{
- int ret = 0;
- struct fi_av_attr attr;
- size_t addrlen = 0;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = 4;
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- ret = fi_domain(fab, fi, &dom, NULL);
- cr_assert(!ret, "fi_domain");
-
- attr.type = FI_AV_MAP;
- attr.count = 16;
-
- ret = fi_av_open(dom, &attr, &av, NULL);
- cr_assert(!ret, "fi_av_open");
-
- ret = fi_endpoint(dom, fi, &ep[0], NULL);
- cr_assert(!ret, "fi_endpoint");
-
- cq_attr.format = FI_CQ_FORMAT_TAGGED;
- cq_attr.size = 1024;
- cq_attr.wait_obj = 0;
-
- ret = fi_cq_open(dom, &cq_attr, &msg_cq[0], 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_cq_open(dom, &cq_attr, &msg_cq[1], 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_ep_bind(ep[0], &msg_cq[0]->fid, FI_SEND | FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
- cr_assert(addrlen > 0);
-
- ep_name[0] = malloc(addrlen);
- cr_assert(ep_name[0] != NULL);
-
- ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_endpoint(dom, fi, &ep[1], NULL);
- cr_assert(!ret, "fi_endpoint");
-
- ret = fi_ep_bind(ep[1], &msg_cq[1]->fid, FI_SEND | FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
-
- ep_name[1] = malloc(addrlen);
- cr_assert(ep_name[1] != NULL);
-
- ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_av_insert(av, ep_name[0], 1, &gni_addr[0], 0,
- NULL);
- cr_assert(ret == 1);
-
- ret = fi_av_insert(av, ep_name[1], 1, &gni_addr[1], 0,
- NULL);
- cr_assert(ret == 1);
-
- ret = fi_ep_bind(ep[0], &av->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_ep_bind(ep[1], &av->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_enable(ep[0]);
- cr_assert(!ret, "fi_ep_enable");
-
- ret = fi_enable(ep[1]);
- cr_assert(!ret, "fi_ep_enable");
-
- target = malloc(BUF_SZ*2);
- assert(target);
-
- source = malloc(BUF_SZ*2);
- assert(source);
-
- peek_buffer = calloc(BUF_SZ, sizeof(char));
- cr_assert(peek_buffer);
-
- ret = fi_mr_reg(dom, target, BUF_SZ*2,
- FI_REMOTE_WRITE, 0, 0, 0, &rem_mr, &target);
- cr_assert_eq(ret, 0);
-
- ret = fi_mr_reg(dom, source, BUF_SZ*2,
- FI_REMOTE_WRITE, 0, 0, 0, &loc_mr, &source);
- cr_assert_eq(ret, 0);
-
- mr_key = fi_mr_key(rem_mr);
-
- init_test_variables();
-}
-
-static void rdm_fi_pdc_teardown(void)
-{
- int ret = 0;
-
- fi_close(&loc_mr->fid);
- fi_close(&rem_mr->fid);
-
- free(target);
- free(source);
-
- ret = fi_close(&ep[0]->fid);
- cr_assert(!ret, "failure in closing ep.");
-
- ret = fi_close(&ep[1]->fid);
- cr_assert(!ret, "failure in closing ep.");
-
- ret = fi_close(&msg_cq[0]->fid);
- cr_assert(!ret, "failure in send cq.");
-
- ret = fi_close(&msg_cq[1]->fid);
- cr_assert(!ret, "failure in recv cq.");
-
- ret = fi_close(&av->fid);
- cr_assert(!ret, "failure in closing av.");
-
- ret = fi_close(&dom->fid);
- cr_assert(!ret, "failure in closing domain.");
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
- free(ep_name[0]);
- free(ep_name[1]);
- free(peek_buffer);
-}
-
-static void rdm_fi_pdc_init_data_range(
- char *buf,
- int start,
- int len,
- char seed)
-{
- int i;
-
- for (i = start; i < start + len; i++)
- buf[i] = seed;
-}
-
-static void rdm_fi_pdc_init_data(
- char *buf,
- int len,
- char seed)
-{
- rdm_fi_pdc_init_data_range(buf, 0, len, seed);
-}
-
-static int rdm_fi_pdc_check_data_range(
- char *src,
- char *dst,
- int len)
-{
- int i;
-
- for (i = 0; i < len; i++) {
- if (src[i] != dst[i]) {
- printf("data mismatch, elem: %d, exp: %x, act: %x\n",
- i, src[i], dst[i]);
- return 0;
- }
- }
-
- return 1;
-}
-
-static int rdm_fi_pdc_check_data_pattern(
- char *buf,
- char pattern,
- int len)
-{
- int i;
-
- for (i = 0; i < len; i++) {
- if (buf[i] != pattern) {
- printf("data mismatch, elem: %d, exp: %x, act: %x\n",
- i, pattern, buf[i]);
- return 0;
- }
- }
-
- return 1;
-}
-
-static int rdm_fi_pdc_check_data(
- char *buf1,
- char *buf2,
- int len)
-{
- return rdm_fi_pdc_check_data_range(buf1, buf2, len);
-}
-
-static void rdm_fi_pdc_xfer_for_each_size(
- void (*xfer)(int len),
- int slen,
- int elen)
-{
- int i;
-
- for (i = slen; i <= elen; i *= 2) {
- dbg_printf("running test on size %d\n", i);
- xfer(i);
- }
-}
-
-/*******************************************************************************
- * Test MSG functions
- ******************************************************************************/
-
-TestSuite(rdm_fi_pdc,
- .init = rdm_fi_pdc_setup,
- .fini = rdm_fi_pdc_teardown,
- .disabled = false);
-
-static void build_message(
- struct fi_msg_tagged *msg,
- struct iovec *iov,
- void *target,
- int len,
- void **rem_mr,
- size_t gni_addr,
- void *source,
- uint64_t tag,
- uint64_t ignore)
-{
- iov->iov_base = target;
- iov->iov_len = len;
-
- msg->msg_iov = iov;
- msg->desc = rem_mr;
- msg->iov_count = 1;
- msg->addr = gni_addr;
- msg->context = source;
- msg->data = (uint64_t) source;
- msg->tag = tag;
- msg->ignore = ignore;
-}
-
-static void build_peek_message(
- struct fi_msg_tagged *peek,
- struct fi_msg_tagged *msg)
-{
- /* copy contents */
- *peek = *msg;
-
- peek->msg_iov = &peek_iov;
-
- peek_iov.iov_base = peek_buffer;
- peek_iov.iov_len = msg->msg_iov[0].iov_len;
-}
-
-#define TSEND_FLAGS (FI_MSG | FI_SEND | FI_TAGGED)
-#define TRECV_FLAGS (FI_MSG | FI_RECV | FI_TAGGED)
-
-static void validate_cqe_contents(
- struct fi_cq_tagged_entry *entry,
- uint64_t flags,
- void *buf,
- size_t len,
- uint64_t tag,
- void *context)
-{
- cr_assert_eq(entry->op_context, context);
- cr_assert_eq(entry->flags, flags);
- cr_assert_eq(entry->data, 0);
-
- if (flags & FI_RECV) {
- if (!(flags & FI_DISCARD))
- cr_assert_eq(entry->len, len);
- else {
- cr_assert_eq(entry->len, 0);
- cr_assert_eq(entry->buf, NULL);
- }
- cr_assert_eq(entry->tag, tag);
- if (entry->buf != NULL)
- cr_assert_eq(entry->buf, buf);
- } else {
- cr_assert_eq(entry->len, 0);
- cr_assert_eq(entry->tag, 0);
- cr_assert_eq(entry->buf, 0);
- }
-}
-
-static void validate_cqe_with_message(
- struct fi_cq_tagged_entry *entry,
- struct fi_msg_tagged *msg,
- uint64_t flags)
-{
- validate_cqe_contents(entry, flags, msg->msg_iov[0].iov_base,
- msg->msg_iov[0].iov_len, msg->tag, msg->context);
-}
-
-
-static inline void map_src_cqes_to_src_context(
- struct fi_cq_tagged_entry *source_cqes,
- struct fi_cq_tagged_entry **mapping,
- void **context)
-{
- int i, j;
-
- /* map src cqes to src parameters */
- for (i = 0; i < 2; i++)
- for (j = 0; j < 2; j++)
- if (source_cqes[i].op_context == context[j])
- mapping[j] = &source_cqes[i];
-}
-
-static inline void start_test_timer(void)
-{
- gettimeofday(&loop_start, NULL);
-}
-
-static inline void update_test_timer(void)
-{
- gettimeofday(&loop_end, NULL);
- elapsed = elapsed_seconds(&loop_start, &loop_end);
-}
-
-/*
-ssize_t (*recvmsg)(struct fid_ep *ep, const struct fi_msg *msg,
- uint64_t flags);
- */
-Test(rdm_fi_pdc, peek_no_event)
-{
- /*
- * This test should do nothing but peek into EP to ensure that
- * no messages are there. This should be a simple test
- */
-
- int ret;
- struct fi_msg_tagged msg;
- struct iovec iov;
-
- build_message(&msg, &iov, target, 128, (void *) &rem_mr, gni_addr[0],
- source, 128, 0);
-
- ret = fi_trecvmsg(ep[1], &msg, FI_PEEK);
- cr_assert_eq(ret, FI_SUCCESS);
-
- ret = fi_cq_readerr(msg_cq[1], &cqe_error, 0);
- cr_assert_eq(ret, 1);
-
- cr_assert_eq(cqe_error.buf, msg.msg_iov[0].iov_base);
- cr_assert_eq(cqe_error.len, msg.msg_iov[0].iov_len);
- cr_assert_eq(cqe_error.err, FI_ENOMSG);
- cr_assert_eq(cqe_error.olen, msg.msg_iov[0].iov_len);
- cr_assert_eq(cqe_error.op_context, msg.context);
- cr_assert_eq(cqe_error.prov_errno, FI_ENOMSG);
- cr_assert_eq(cqe_error.tag, msg.tag);
-}
-
-static void pdc_peek_event_present_buffer_provided(int len)
-{
- /* PEEK then RECV with no buffer during peek test
- *
- * For each message size,
- * 1. send a message
- * 2. peek| the receiver to find the message
- * 3. receive the message
- *
- * Test validation is done at the end of the test by verifying the
- * contents of the CQEs after asserting that the send and recv
- * sides have finished. Buffer contents should match the
- * message contents.
- *
- * This is the special case where the application provides a buffer
- * during the peek for which some of the data can be written.
- *
- * An application may supply a buffer as part of the
- * peek operation. If given, the provider may return a copy
- * of the message data.
- *
- * Ideally, both cases should be tested, where the provider
- * returns a NULL pointer indicating that no data was available
- * yet even though the peek succeeded, and the case where some
- * of the data is copied back.
- */
- int ret;
- struct fi_msg_tagged msg;
- struct iovec iov;
- struct fi_cq_tagged_entry s_cqe;
- struct fi_cq_tagged_entry d_cqe;
-
- rdm_fi_pdc_init_data(source, len, 0xab);
- rdm_fi_pdc_init_data(target, len, 0);
-
- build_message(&msg, &iov, target, len, (void *) &rem_mr, gni_addr[0],
- source, len, 0);
-
- INIT_TEST_STATE(S_STATE_SEND_MSG_1, R_STATE_PEEK);
-
- /* we need to set up a peek buffer to ensure the contents of the peek
- * are copied correctly. In the event of a discard, the data can be
- * fetched with a peek, but the target buffer should remain untouched
- */
- build_peek_message(&peek_msg, &msg);
-
- start_test_timer();
- do {
- PROGRESS_CQS(msg_cq);
-
- switch (s_state) {
- case S_STATE_SEND_MSG_1:
- ret = fi_tsend(ep[0], source, len, loc_mr,
- gni_addr[1], len, target);
- cr_assert_eq(ret, FI_SUCCESS);
- SEND_STATE_TRANSITION(S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_SEND_MSG_1_WAIT_CQ:
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- COND_SEND_STATE_TRANSITION(ret, 1,
- S_STATE_DONE,
- S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- switch (r_state) {
- case R_STATE_PEEK:
- ret = fi_trecvmsg(ep[1], &peek_msg, FI_PEEK);
-
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_PEEK_WAIT_CQ,
- R_STATE_PEEK);
- break;
- case R_STATE_PEEK_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_peek_cqe, 1);
- if (ret == -FI_EAVAIL) {
- ret = fi_cq_readerr(msg_cq[1], &cqe_error, 0);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_PEEK,
- R_STATE_PEEK_WAIT_CQ);
- } else {
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_RECV_MSG_1,
- R_STATE_PEEK_WAIT_CQ);
- }
- break;
- case R_STATE_RECV_MSG_1:
- ret = fi_trecvmsg(ep[1], &msg, 0);
- cr_assert_eq(ret, FI_SUCCESS);
- RECV_STATE_TRANSITION(R_STATE_RECV_MSG_1_WAIT_CQ);
- break;
- case R_STATE_RECV_MSG_1_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_DONE,
- R_STATE_RECV_MSG_1_WAIT_CQ);
- break;
- case R_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- update_test_timer();
- } while (!TEST_TIME_LIMIT_EXPIRED && !SEND_RECV_DONE);
-
- ASSERT_SEND_RECV_DONE;
-
- /* validate the expected results */
- validate_cqe_contents(&s_cqe, TSEND_FLAGS, source, len, len, target);
- validate_cqe_with_message(&d_peek_cqe, &peek_msg,
- TRECV_FLAGS | FI_PEEK);
-
- /* if CQE provided a buffer back, the data was copied.
- * Check the data */
- if (d_peek_cqe.buf) {
- cr_assert(rdm_fi_pdc_check_data_pattern(peek_buffer, 0xab, len),
- "Data mismatch");
- }
-
- validate_cqe_with_message(&d_cqe, &msg, TRECV_FLAGS);
- cr_assert(rdm_fi_pdc_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_fi_pdc, peek_event_present_buff_provided)
-{
- rdm_fi_pdc_xfer_for_each_size(pdc_peek_event_present_buffer_provided,
- 1, BUF_SZ);
-}
-
-static void pdc_peek_event_present_no_buff_provided(int len)
-{
- /* PEEK then RECV with no buffer during peek test
- *
- * For each message size,
- * 1. send a message
- * 2. peek| the receiver to find the message
- * 3. receive the message
- *
- * Test validation is done at the end of the test by verifying the
- * contents of the CQEs after asserting that the send and recv
- * sides have finished. Buffer contents should match the
- * message contents.
- *
- * The CQE for the peek should have a NULL buffer field
- */
-
- int ret;
- struct fi_msg_tagged msg;
- struct iovec iov;
- struct fi_cq_tagged_entry s_cqe;
- struct fi_cq_tagged_entry d_cqe;
-
- rdm_fi_pdc_init_data(source, len, 0xab);
- rdm_fi_pdc_init_data(target, len, 0);
-
- build_message(&msg, &iov, target, len, (void *) &rem_mr,
- gni_addr[0], source, len, 0);
-
- INIT_TEST_STATE(S_STATE_SEND_MSG_1, R_STATE_PEEK);
-
- /* we need to set up a peek buffer to ensure the contents of the peek
- * are copied correctly. In the event of a discard, the data can be
- * fetched with a peek, but the target buffer should remain untouched
- */
- build_peek_message(&peek_msg, &msg);
-
- /* send in a null buffer to indicate that we don't want data back */
- peek_iov.iov_base = NULL;
-
- start_test_timer();
- do {
- PROGRESS_CQS(msg_cq);
-
- switch (s_state) {
- case S_STATE_SEND_MSG_1:
- ret = fi_tsend(ep[0], source, len, loc_mr,
- gni_addr[1], len, target);
- cr_assert_eq(ret, FI_SUCCESS);
- SEND_STATE_TRANSITION(S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_SEND_MSG_1_WAIT_CQ:
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- COND_SEND_STATE_TRANSITION(ret, 1,
- S_STATE_DONE,
- S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- switch (r_state) {
- case R_STATE_PEEK:
- ret = fi_trecvmsg(ep[1], &peek_msg, FI_PEEK);
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_PEEK_WAIT_CQ,
- R_STATE_PEEK);
- break;
- case R_STATE_PEEK_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_peek_cqe, 1);
- if (ret == -FI_EAVAIL) {
- ret = fi_cq_readerr(msg_cq[1], &cqe_error, 0);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_PEEK,
- R_STATE_PEEK_WAIT_CQ);
- } else {
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_RECV_MSG_1,
- R_STATE_PEEK_WAIT_CQ);
- }
- break;
- case R_STATE_RECV_MSG_1:
- ret = fi_trecvmsg(ep[1], &msg, 0);
- cr_assert_eq(ret, FI_SUCCESS);
- RECV_STATE_TRANSITION(R_STATE_RECV_MSG_1_WAIT_CQ);
- break;
- case R_STATE_RECV_MSG_1_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_DONE,
- R_STATE_RECV_MSG_1_WAIT_CQ);
- break;
- case R_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- update_test_timer();
- } while (!TEST_TIME_LIMIT_EXPIRED && !SEND_RECV_DONE);
-
- ASSERT_SEND_RECV_DONE;
-
- /* verify test execution correctness */
- validate_cqe_contents(&s_cqe, TSEND_FLAGS, source, len, len, target);
- validate_cqe_with_message(&d_cqe, &msg, TRECV_FLAGS);
-
- /* a pointer should never be returned */
- cr_assert_eq(d_peek_cqe.buf, NULL);
- validate_cqe_with_message(&d_peek_cqe, &peek_msg,
- TRECV_FLAGS | FI_PEEK);
- cr_assert(rdm_fi_pdc_check_data(source, target, len),
- "Data mismatch");
-}
-
-Test(rdm_fi_pdc, peek_event_present_no_buff_provided)
-{
- rdm_fi_pdc_xfer_for_each_size(pdc_peek_event_present_no_buff_provided,
- 1, BUF_SZ);
-}
-
-static void pdc_peek_claim_same_tag(int len)
-{
- /* PEEK|CLAIM then CLAIM using the same tags test
- *
- * For each message size,
- * 1. send two messages with identical tags but different
- * buffer parameters
- * 2. peek|claim the receiver to find and claim the first message
- * 3. receive the second message
- * 4. claim the first message
- *
- * Test validation is done at the end of the test by verifying the
- * contents of the CQEs after asserting that the send and recv
- * sides have finished. Buffer contents should match the
- * message contents.
- */
- int ret, i;
- struct fi_msg_tagged msg[2];
- struct iovec iov[2];
- struct fi_cq_tagged_entry s_cqe[2];
- struct fi_cq_tagged_entry d_cqe[2];
- struct fi_cq_tagged_entry *src_cqe[2] = {NULL, NULL};
- char *src_buf[2] = {source, source + len};
- char *dst_buf[2] = {target, target + len};
- void *src_context[2] = {target, target + len};
-
- /* initialize the initial data range on the source buffer to have
- * different data values for one message than for the other
- */
- rdm_fi_pdc_init_data_range(source, 0, len, 0xa5);
- rdm_fi_pdc_init_data_range(source, len, len, 0x5a);
- rdm_fi_pdc_init_data(target, len*2, 0);
-
- /* set up messages */
- for (i = 0; i < 2; i++) {
- build_message(&msg[i], &iov[i], dst_buf[i], len, (void *)
- &rem_mr, gni_addr[0], src_buf[i], len, 0);
- }
-
- /* we need to set up a peek buffer to ensure the contents of the peek
- * are copied correctly. In the event of a discard, the data can be
- * fetched with a peek, but the target buffer should remain untouched
- */
- build_peek_message(&peek_msg, &msg[0]);
-
- INIT_TEST_STATE(S_STATE_SEND_MSG_1, R_STATE_PEEK_CLAIM);
-
- start_test_timer();
- do {
- PROGRESS_CQS(msg_cq);
-
- switch (s_state) {
- case S_STATE_SEND_MSG_1:
- ret = fi_tsend(ep[0], src_buf[0], len, loc_mr,
- gni_addr[1], len, dst_buf[0]);
- cr_assert_eq(ret, FI_SUCCESS);
- SEND_STATE_TRANSITION(S_STATE_SEND_MSG_2);
- break;
- case S_STATE_SEND_MSG_2:
- ret = fi_tsend(ep[0], src_buf[1], len, loc_mr,
- gni_addr[1], len, dst_buf[1]);
- cr_assert_eq(ret, FI_SUCCESS);
- SEND_STATE_TRANSITION(S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_SEND_MSG_1_WAIT_CQ:
- ret = fi_cq_read(msg_cq[0], &s_cqe[0], 1);
- COND_SEND_STATE_TRANSITION(ret, 1,
- S_STATE_SEND_MSG_2_WAIT_CQ,
- S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_SEND_MSG_2_WAIT_CQ:
- ret = fi_cq_read(msg_cq[0], &s_cqe[1], 1);
- COND_SEND_STATE_TRANSITION(ret, 1,
- S_STATE_DONE,
- S_STATE_SEND_MSG_2_WAIT_CQ);
- break;
- case S_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- switch (r_state) {
- case R_STATE_PEEK_CLAIM:
- ret = fi_trecvmsg(ep[1], &peek_msg, FI_PEEK | FI_CLAIM);
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_PEEK_CLAIM_WAIT_CQ,
- R_STATE_PEEK_CLAIM);
- break;
- case R_STATE_PEEK_CLAIM_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_peek_cqe, 1);
- if (ret == -FI_EAVAIL) {
- ret = fi_cq_readerr(msg_cq[1], &cqe_error, 0);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_PEEK_CLAIM,
- R_STATE_PEEK_CLAIM_WAIT_CQ);
- } else {
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_RECV_MSG_2,
- R_STATE_PEEK_CLAIM_WAIT_CQ);
- }
- break;
- case R_STATE_RECV_MSG_2:
- ret = fi_trecvmsg(ep[1], &msg[1], 0);
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_RECV_MSG_2_WAIT_CQ,
- R_STATE_RECV_MSG_2);
- break;
- case R_STATE_RECV_MSG_2_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_cqe[1], 1);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_CLAIM,
- R_STATE_RECV_MSG_2_WAIT_CQ);
- break;
- case R_STATE_CLAIM:
- ret = fi_trecvmsg(ep[1], &msg[0], FI_CLAIM);
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_CLAIM_WAIT_CQ,
- R_STATE_CLAIM);
- break;
- case R_STATE_CLAIM_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_cqe[0], 1);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_DONE,
- R_STATE_CLAIM_WAIT_CQ);
- break;
- case R_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- update_test_timer();
- } while (!TEST_TIME_LIMIT_EXPIRED && !SEND_RECV_DONE);
-
- ASSERT_SEND_RECV_DONE;
-
- /* map src cqes to src parameters */
- map_src_cqes_to_src_context(s_cqe, src_cqe, src_context);
-
- /* verify test execution correctness */
- validate_cqe_contents(src_cqe[0], TSEND_FLAGS,
- src_buf[0], len, len, dst_buf[0]);
- validate_cqe_contents(src_cqe[1], TSEND_FLAGS,
- src_buf[1], len, len, dst_buf[1]);
- validate_cqe_with_message(&d_peek_cqe, &peek_msg,
- TRECV_FLAGS | FI_PEEK | FI_CLAIM);
- validate_cqe_with_message(&d_cqe[1], &msg[1], TRECV_FLAGS);
- validate_cqe_with_message(&d_cqe[0], &msg[0], TRECV_FLAGS | FI_CLAIM);
-
- /* if CQE provided a buffer back, the data was copied.
- * Check the data */
- if (d_peek_cqe.buf) {
- cr_assert(rdm_fi_pdc_check_data_pattern(peek_buffer, 0xa5, len),
- "Data mismatch");
- }
-
-
- cr_assert(rdm_fi_pdc_check_data(src_buf[0], dst_buf[0], len),
- "Data mismatch");
-
- cr_assert(rdm_fi_pdc_check_data(src_buf[1], dst_buf[1], len),
- "Data mismatch");
-}
-
-Test(rdm_fi_pdc, peek_claim_same_tag)
-{
- rdm_fi_pdc_xfer_for_each_size(pdc_peek_claim_same_tag, 1, BUF_SZ);
-}
-
-static void pdc_peek_claim_unique_tag(int len)
-{
- /* PEEK|CLAIM then CLAIM using unique tags test
- *
- * For each message size,
- * 1. send two messages with unique tags and parameters
- * 2. peek|claim the receiver to find and claim the first message
- * 3. receive the second message
- * 4. claim the first message
- *
- * Test validation is done at the end of the test by verifying the
- * contents of the CQEs after asserting that the send and recv
- * sides have finished. Buffer contents should match the
- * message contents
- */
-
- int ret, i;
- struct fi_msg_tagged msg[2];
- struct iovec iov[2];
- struct fi_cq_tagged_entry s_cqe[2];
- struct fi_cq_tagged_entry d_cqe[2];
- struct fi_cq_tagged_entry *src_cqe[2] = {NULL, NULL};
- char *src_buf[2] = {source, source + len};
- char *dst_buf[2] = {target, target + len};
- void *src_context[2] = {target, target + len};
-
- /* initialize the initial data range on the source buffer to have
- * different data vaules for one message than for the other
- */
- rdm_fi_pdc_init_data_range(source, 0, len, 0xa5);
- rdm_fi_pdc_init_data_range(source, len, len, 0x5a);
- rdm_fi_pdc_init_data(target, len*2, 0);
-
- /* set up messages */
- for (i = 0; i < 2; i++) {
- build_message(&msg[i], &iov[i], dst_buf[i], len,
- (void *) &rem_mr, gni_addr[0],
- src_buf[i], len + i, 0);
- }
-
- /* we need to set up a peek buffer to ensure the contents of the peek
- * are copied correctly. In the event of a discard, the data can be
- * fetched with a peek, but the target buffer should remain untouched
- */
- build_peek_message(&peek_msg, &msg[0]);
-
- INIT_TEST_STATE(S_STATE_SEND_MSG_1, R_STATE_PEEK_CLAIM);
-
- start_test_timer();
- do {
- PROGRESS_CQS(msg_cq);
-
- switch (s_state) {
- case S_STATE_SEND_MSG_1:
- ret = fi_tsend(ep[0], src_buf[0], len, loc_mr,
- gni_addr[1], len, dst_buf[0]);
- cr_assert_eq(ret, FI_SUCCESS);
- SEND_STATE_TRANSITION(S_STATE_SEND_MSG_2);
- break;
- case S_STATE_SEND_MSG_2:
- ret = fi_tsend(ep[0], src_buf[1], len, loc_mr,
- gni_addr[1], len + 1, dst_buf[1]);
- cr_assert_eq(ret, FI_SUCCESS);
- SEND_STATE_TRANSITION(S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_SEND_MSG_1_WAIT_CQ:
- ret = fi_cq_read(msg_cq[0], &s_cqe[0], 1);
- COND_SEND_STATE_TRANSITION(ret, 1,
- S_STATE_SEND_MSG_2_WAIT_CQ,
- S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_SEND_MSG_2_WAIT_CQ:
- ret = fi_cq_read(msg_cq[0], &s_cqe[1], 1);
- COND_SEND_STATE_TRANSITION(ret, 1,
- S_STATE_DONE,
- S_STATE_SEND_MSG_2_WAIT_CQ);
- break;
- case S_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- switch (r_state) {
- case R_STATE_PEEK_CLAIM:
- ret = fi_trecvmsg(ep[1], &peek_msg, FI_PEEK | FI_CLAIM);
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_PEEK_CLAIM_WAIT_CQ,
- R_STATE_PEEK_CLAIM);
- break;
- case R_STATE_PEEK_CLAIM_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_peek_cqe, 1);
- if (ret == -FI_EAVAIL) {
- ret = fi_cq_readerr(msg_cq[1], &cqe_error, 0);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_PEEK_CLAIM,
- R_STATE_PEEK_WAIT_CQ);
- } else {
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_RECV_MSG_2,
- R_STATE_PEEK_CLAIM_WAIT_CQ);
- }
- break;
- case R_STATE_RECV_MSG_2:
- ret = fi_trecvmsg(ep[1], &msg[1], 0);
- cr_assert_eq(ret, FI_SUCCESS);
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_RECV_MSG_2_WAIT_CQ,
- R_STATE_RECV_MSG_2);
- break;
- case R_STATE_RECV_MSG_2_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_cqe[1], 1);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_CLAIM,
- R_STATE_RECV_MSG_2_WAIT_CQ);
- break;
- case R_STATE_CLAIM:
- ret = fi_trecvmsg(ep[1], &msg[0], FI_CLAIM);
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_CLAIM_WAIT_CQ,
- R_STATE_CLAIM);
- break;
- case R_STATE_CLAIM_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_cqe[0], 1);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_DONE,
- R_STATE_CLAIM_WAIT_CQ);
- break;
- case R_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- update_test_timer();
- } while (!TEST_TIME_LIMIT_EXPIRED && !SEND_RECV_DONE);
-
- ASSERT_SEND_RECV_DONE;
-
- /* map src cqes to src parameters */
- map_src_cqes_to_src_context(s_cqe, src_cqe, src_context);
-
- /* verify test execution correctness */
- for (i = 0; i < 2; i++)
- validate_cqe_contents(src_cqe[i], TSEND_FLAGS,
- src_buf[i], len, len + i, dst_buf[i]);
-
- validate_cqe_with_message(&d_peek_cqe, &peek_msg,
- TRECV_FLAGS | FI_PEEK | FI_CLAIM);
- validate_cqe_with_message(&d_cqe[1], &msg[1], TRECV_FLAGS);
- validate_cqe_with_message(&d_cqe[0], &msg[0], TRECV_FLAGS | FI_CLAIM);
-
- /* if CQE provided a buffer back, the data was copied.
- * Check the data */
- if (d_peek_cqe.buf) {
- cr_assert(rdm_fi_pdc_check_data_pattern(peek_buffer, 0xa5, len),
- "Data mismatch");
- }
-
-
- cr_assert(rdm_fi_pdc_check_data(src_buf[0], dst_buf[0], len),
- "Data mismatch");
- cr_assert(rdm_fi_pdc_check_data(src_buf[1], dst_buf[1], len),
- "Data mismatch");
-}
-
-Test(rdm_fi_pdc, peek_claim_unique_tag)
-{
- rdm_fi_pdc_xfer_for_each_size(pdc_peek_claim_unique_tag, 1, BUF_SZ);
-}
-
-static void pdc_peek_discard(int len)
-{
- /* PEEK|DISCARD then PEEK test
- *
- * For each message size,
- * 1. send one messages
- * 2. peek|discard the receiver to find and discard message
- * 3. peek the message and fail to find it
- *
- * Test validation is done at the end of the test by verifying the
- * contents of the CQEs after asserting that the send and recv
- * sides have finished. Buffer contents should match the
- * message contents
- */
-
- int ret;
- struct fi_msg_tagged msg;
- struct iovec iov;
- struct fi_cq_tagged_entry s_cqe;
-
- rdm_fi_pdc_init_data(source, len, 0xab);
- rdm_fi_pdc_init_data(target, len, 0);
-
- build_message(&msg, &iov, target, len, (void *) &rem_mr,
- gni_addr[0], source, len, 0);
-
- INIT_TEST_STATE(S_STATE_SEND_MSG_1, R_STATE_PEEK_DISCARD);
-
- /* we need to set up a peek buffer to ensure the contents of the peek
- * are copied correctly. In the event of a discard, the data can be
- * fetched with a peek, but the target buffer should remain untouched
- */
- build_peek_message(&peek_msg, &msg);
-
- start_test_timer();
- do {
- PROGRESS_CQS(msg_cq);
-
- switch (s_state) {
- case S_STATE_SEND_MSG_1:
- ret = fi_tsend(ep[0], source, len, loc_mr,
- gni_addr[1], len, target);
- cr_assert_eq(ret, FI_SUCCESS);
- SEND_STATE_TRANSITION(S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_SEND_MSG_1_WAIT_CQ:
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- COND_SEND_STATE_TRANSITION(ret, 1,
- S_STATE_DONE,
- S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- switch (r_state) {
- case R_STATE_PEEK_DISCARD:
- ret = fi_trecvmsg(ep[1], &peek_msg, FI_PEEK | FI_DISCARD);
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_PEEK_DISCARD_WAIT_CQ,
- R_STATE_PEEK_DISCARD);
- break;
- case R_STATE_PEEK_DISCARD_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_peek_cqe, 1);
- if (ret == -FI_EAVAIL) {
- ret = fi_cq_readerr(msg_cq[1], &cqe_error, 0);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_PEEK_DISCARD,
- R_STATE_PEEK_DISCARD_WAIT_CQ);
- } else {
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_PEEK,
- R_STATE_PEEK_DISCARD_WAIT_CQ);
- }
- break;
- case R_STATE_PEEK:
- ret = fi_trecvmsg(ep[1], &msg, FI_PEEK);
- cr_assert_eq(ret, FI_SUCCESS);
- RECV_STATE_TRANSITION(R_STATE_PEEK_WAIT_ERR_CQ);
- break;
- case R_STATE_PEEK_WAIT_ERR_CQ:
- ret = fi_cq_readerr(msg_cq[1], &cqe_error, 0);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_DONE,
- R_STATE_PEEK_WAIT_ERR_CQ);
- break;
- case R_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- update_test_timer();
- } while (!TEST_TIME_LIMIT_EXPIRED && !SEND_RECV_DONE);
-
- ASSERT_SEND_RECV_DONE;
-
- /* verify test execution correctness */
- validate_cqe_contents(&s_cqe, TSEND_FLAGS, source, len, len, target);
- validate_cqe_with_message(&d_peek_cqe, &peek_msg,
- TRECV_FLAGS | FI_PEEK | FI_DISCARD);
-
- cr_assert(rdm_fi_pdc_check_data_pattern(target, 0, len),
- "Data matched");
-}
-
-Test(rdm_fi_pdc, peek_discard)
-{
- rdm_fi_pdc_xfer_for_each_size(pdc_peek_discard, 1, BUF_SZ);
-}
-
-static void pdc_peek_discard_unique_tags(int len)
-{
- /* PEEK|DISCARD then PEEK using unique tags test
- *
- * For each message size,
- * 1. send two messages with unique tags and parameters
- * 2. peek|discard the receiver to find and discard the first message
- * 3. receive the second message
- * 4. peek the first message and fail to find it
- *
- * Test validation is done at the end of the test by verifying the
- * contents of the CQEs after asserting that the send and recv
- * sides have finished. Buffer contents should match the
- * message contents
- */
- int ret, i;
- struct fi_msg_tagged msg[2];
- struct iovec iov[2];
- struct fi_cq_tagged_entry s_cqe[2];
- struct fi_cq_tagged_entry d_cqe[2];
- struct fi_cq_tagged_entry *src_cqe[2] = {NULL, NULL};
- char *src_buf[2] = {source, source + len};
- char *dst_buf[2] = {target, target + len};
- void *src_context[2] = {target, target + len};
-
- /* initialize the initial data range on the source buffer to have
- * different data vaules for one message than for the other
- */
- rdm_fi_pdc_init_data_range(source, 0, len, 0xa5);
- rdm_fi_pdc_init_data_range(source, len, len, 0x5a);
- rdm_fi_pdc_init_data(target, len*2, 0);
-
- /* set up messages */
- for (i = 0; i < 2; i++) {
- build_message(&msg[i], &iov[i], dst_buf[i], len,
- (void *) &rem_mr, gni_addr[0],
- src_buf[i], len + i, 0);
- }
-
- /* we need to set up a peek buffer to ensure the contents of the peek
- * are copied correctly. In the event of a discard, the data can be
- * fetched with a peek, but the target buffer should remain untouched
- */
- build_peek_message(&peek_msg, &msg[0]);
-
- INIT_TEST_STATE(S_STATE_SEND_MSG_1, R_STATE_PEEK_DISCARD);
-
- start_test_timer();
- do {
- PROGRESS_CQS(msg_cq);
-
- switch (s_state) {
- case S_STATE_SEND_MSG_1:
- ret = fi_tsend(ep[0], src_buf[0], len, loc_mr,
- gni_addr[1], len, dst_buf[0]);
- cr_assert_eq(ret, FI_SUCCESS);
- SEND_STATE_TRANSITION(S_STATE_SEND_MSG_2);
- break;
- case S_STATE_SEND_MSG_2:
- ret = fi_tsend(ep[0], src_buf[1], len, loc_mr,
- gni_addr[1], len + 1, dst_buf[1]);
- cr_assert_eq(ret, FI_SUCCESS);
- SEND_STATE_TRANSITION(S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_SEND_MSG_1_WAIT_CQ:
- ret = fi_cq_read(msg_cq[0], &s_cqe[0], 1);
- COND_SEND_STATE_TRANSITION(ret, 1,
- S_STATE_SEND_MSG_2_WAIT_CQ,
- S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_SEND_MSG_2_WAIT_CQ:
- ret = fi_cq_read(msg_cq[0], &s_cqe[1], 1);
- COND_SEND_STATE_TRANSITION(ret, 1,
- S_STATE_DONE,
- S_STATE_SEND_MSG_2_WAIT_CQ);
- break;
- case S_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- switch (r_state) {
- case R_STATE_PEEK_DISCARD:
- ret = fi_trecvmsg(ep[1], &peek_msg, FI_PEEK | FI_DISCARD);
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_PEEK_DISCARD_WAIT_CQ,
- R_STATE_PEEK_DISCARD);
- break;
- case R_STATE_PEEK_DISCARD_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_peek_cqe, 1);
- if (ret == -FI_EAVAIL) {
- ret = fi_cq_readerr(msg_cq[1], &cqe_error, 0);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_PEEK_DISCARD,
- R_STATE_PEEK_DISCARD_WAIT_CQ);
- } else {
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_RECV_MSG_2,
- R_STATE_PEEK_DISCARD_WAIT_CQ);
- }
- break;
- case R_STATE_RECV_MSG_2:
- ret = fi_trecvmsg(ep[1], &msg[1], 0);
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_RECV_MSG_2_WAIT_CQ,
- R_STATE_RECV_MSG_2);
- break;
- case R_STATE_RECV_MSG_2_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_cqe[1], 1);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_PEEK,
- R_STATE_RECV_MSG_2_WAIT_CQ);
- break;
- case R_STATE_PEEK:
- ret = fi_trecvmsg(ep[1], &msg[0], FI_PEEK);
- cr_assert_eq(ret, FI_SUCCESS);
- RECV_STATE_TRANSITION(R_STATE_PEEK_WAIT_ERR_CQ);
- break;
- case R_STATE_PEEK_WAIT_ERR_CQ:
- ret = fi_cq_readerr(msg_cq[1], &cqe_error, 0);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_DONE,
- R_STATE_PEEK_WAIT_ERR_CQ);
- break;
- case R_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- update_test_timer();
- } while (!TEST_TIME_LIMIT_EXPIRED && !SEND_RECV_DONE);
-
- ASSERT_SEND_RECV_DONE;
-
- /* map src cqes to src parameters */
- map_src_cqes_to_src_context(s_cqe, src_cqe, src_context);
-
- /* verify test execution correctness */
- for (i = 0; i < 2; i++)
- validate_cqe_contents(src_cqe[i], TSEND_FLAGS,
- src_buf[i], len, len + i, dst_buf[i]);
-
- validate_cqe_with_message(&d_cqe[1], &msg[1], TRECV_FLAGS);
- validate_cqe_with_message(&d_peek_cqe, &peek_msg,
- TRECV_FLAGS | FI_PEEK | FI_DISCARD);
-
- cr_assert(rdm_fi_pdc_check_data_pattern(dst_buf[0], 0, len),
- "Data mismatch");
- cr_assert(rdm_fi_pdc_check_data(src_buf[1], dst_buf[1], len),
- "Data mismatch");
-}
-
-Test(rdm_fi_pdc, peek_discard_unique_tags)
-{
- rdm_fi_pdc_xfer_for_each_size(pdc_peek_discard_unique_tags,
- 1, BUF_SZ);
-}
-
-static void pdc_peek_claim_then_claim_discard(int len)
-{
- /* PEEK|CLAIM then CLAIM|DISCARD test
- *
- * For each message size,
- * 1. send two messages with unique tags and parameters
- * 2. peek|claim the receiver to find and claim the first message
- * 3. receive the second message
- * 4. claim|discard the first message
- *
- * From the fi_tagged man page:
- * Claimed messages can only be retrieved using a subsequent,
- * paired receive operation with the FI_CLAIM flag set.
- *
- * (FI_DISCARD) flag may also be used in conjunction
- * with FI_CLAIM in order to retrieve and discard a message
- * previously claimed using an FI_PEEK + FI_CLAIM request.
- *
- * Test validation is done at the end of the test by verifying the
- * contents of the CQEs after asserting that the send and recv
- * sides have finished. Buffer contents should match the message
- * contents
- */
- int ret, i;
- struct fi_msg_tagged msg[2];
- struct iovec iov[2];
- struct fi_cq_tagged_entry s_cqe[2];
- struct fi_cq_tagged_entry d_cqe[2];
- struct fi_cq_tagged_entry *src_cqe[2] = {NULL, NULL};
- char *src_buf[2] = {source, source + len};
- char *dst_buf[2] = {target, target + len};
- void *src_context[2] = {target, target + len};
-
- /* initialize the initial data range on the source buffer to have
- * different data values for one message than for the other
- */
- rdm_fi_pdc_init_data_range(source, 0, len, 0xa5);
- rdm_fi_pdc_init_data_range(source, len, len, 0x5a);
- rdm_fi_pdc_init_data(target, len*2, 0);
-
- /* set up messages */
- for (i = 0; i < 2; i++) {
- build_message(&msg[i], &iov[i], dst_buf[i], len,
- (void *) &rem_mr, gni_addr[0],
- src_buf[i], len + i, 0);
- }
-
- /* we need to set up a peek buffer to ensure the contents of the peek
- * are copied correctly. In the event of a discard, the data can be
- * fetched with a peek, but the target buffer should remain untouched
- */
- build_peek_message(&peek_msg, &msg[0]);
-
- INIT_TEST_STATE(S_STATE_SEND_MSG_1, R_STATE_PEEK_CLAIM);
-
- start_test_timer();
- do {
- PROGRESS_CQS(msg_cq);
-
- switch (s_state) {
- case S_STATE_SEND_MSG_1:
- ret = fi_tsend(ep[0], src_buf[0], len, loc_mr,
- gni_addr[1], len, dst_buf[0]);
- cr_assert_eq(ret, FI_SUCCESS);
- SEND_STATE_TRANSITION(S_STATE_SEND_MSG_2);
- break;
- case S_STATE_SEND_MSG_2:
- ret = fi_tsend(ep[0], src_buf[1], len, loc_mr,
- gni_addr[1], len + 1, dst_buf[1]);
- cr_assert_eq(ret, FI_SUCCESS);
- SEND_STATE_TRANSITION(S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_SEND_MSG_1_WAIT_CQ:
- ret = fi_cq_read(msg_cq[0], &s_cqe[0], 1);
- COND_SEND_STATE_TRANSITION(ret, 1,
- S_STATE_SEND_MSG_2_WAIT_CQ,
- S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_SEND_MSG_2_WAIT_CQ:
- ret = fi_cq_read(msg_cq[0], &s_cqe[1], 1);
- COND_SEND_STATE_TRANSITION(ret, 1, S_STATE_DONE,
- S_STATE_SEND_MSG_2_WAIT_CQ);
- break;
- case S_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- switch (r_state) {
- case R_STATE_PEEK_CLAIM:
- ret = fi_trecvmsg(ep[1], &peek_msg, FI_PEEK | FI_CLAIM);
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_PEEK_CLAIM_WAIT_CQ,
- R_STATE_PEEK_CLAIM);
- break;
- case R_STATE_PEEK_CLAIM_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_peek_cqe, 1);
- if (ret == -FI_EAVAIL) {
- ret = fi_cq_readerr(msg_cq[1], &cqe_error, 0);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_PEEK_CLAIM,
- R_STATE_PEEK_CLAIM_WAIT_CQ);
- } else {
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_RECV_MSG_2,
- R_STATE_PEEK_CLAIM_WAIT_CQ);
- }
- break;
- case R_STATE_RECV_MSG_2:
- ret = fi_trecvmsg(ep[1], &msg[1], 0);
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_RECV_MSG_2_WAIT_CQ,
- R_STATE_RECV_MSG_2);
- break;
- case R_STATE_RECV_MSG_2_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_cqe[1], 1);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_CLAIM_DISCARD,
- R_STATE_RECV_MSG_2_WAIT_CQ);
- break;
- case R_STATE_CLAIM_DISCARD:
- ret = fi_trecvmsg(ep[1], &msg[0],
- FI_CLAIM | FI_DISCARD);
- cr_assert_eq(ret, FI_SUCCESS);
- RECV_STATE_TRANSITION(R_STATE_CLAIM_DISCARD_WAIT_CQ);
- break;
- case R_STATE_CLAIM_DISCARD_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_cqe[0], 1);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_DONE,
- R_STATE_CLAIM_DISCARD_WAIT_CQ);
- break;
- case R_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- update_test_timer();
- } while (!TEST_TIME_LIMIT_EXPIRED && !SEND_RECV_DONE);
-
- ASSERT_SEND_RECV_DONE;
-
- /* map src cqes to src parameters */
- map_src_cqes_to_src_context(s_cqe, src_cqe, src_context);
-
- /* verify test execution correctness */
- for (i = 0; i < 2; i++)
- validate_cqe_contents(src_cqe[i], TSEND_FLAGS,
- src_buf[i], len, len + i, dst_buf[i]);
-
- validate_cqe_with_message(&d_peek_cqe, &peek_msg,
- TRECV_FLAGS | FI_PEEK | FI_CLAIM);
-
- /* if CQE provided a buffer back, the data was copied.
- * Check the data */
- if (d_peek_cqe.buf) {
- cr_assert(rdm_fi_pdc_check_data_pattern(peek_buffer, 0xa5, len),
- "Data mismatch");
- }
-
- validate_cqe_with_message(&d_cqe[0], &msg[0],
- TRECV_FLAGS | FI_CLAIM | FI_DISCARD);
- validate_cqe_with_message(&d_cqe[1], &msg[1], TRECV_FLAGS);
-
- cr_assert(rdm_fi_pdc_check_data_pattern(dst_buf[0], 0, len),
- "Data mismatch");
- cr_assert(rdm_fi_pdc_check_data(src_buf[1], dst_buf[1], len),
- "Data mismatch");
-}
-
-Test(rdm_fi_pdc, peek_claim_then_claim_discard)
-{
- rdm_fi_pdc_xfer_for_each_size(pdc_peek_claim_then_claim_discard,
- 1, BUF_SZ);
-}
-
-static void pdc_peek_event_present_small_buffer_provided(int len)
-{
- /* Like pdc_peek_event_present_buffer_provided except uses an
- * undersized receive buffer with the FI_PEEK request. */
- int ret;
- struct fi_msg_tagged msg;
- struct iovec iov;
- struct fi_cq_tagged_entry s_cqe;
- struct fi_cq_tagged_entry d_cqe;
-
- rdm_fi_pdc_init_data(source, len, 0xab);
- rdm_fi_pdc_init_data(target, len, 0);
-
- build_message(&msg, &iov, target, len, (void *) &rem_mr, gni_addr[0],
- source, len, 0);
-
- INIT_TEST_STATE(S_STATE_SEND_MSG_1, R_STATE_PEEK);
-
- /* we need to set up a peek buffer to ensure the contents of the peek
- * are copied correctly. In the event of a discard, the data can be
- * fetched with a peek, but the target buffer should remain untouched
- */
- build_peek_message(&peek_msg, &msg);
- peek_iov.iov_len /= 2;
-
- start_test_timer();
- do {
- PROGRESS_CQS(msg_cq);
-
- switch (s_state) {
- case S_STATE_SEND_MSG_1:
- ret = fi_tsend(ep[0], source, len, loc_mr,
- gni_addr[1], len, target);
- cr_assert_eq(ret, FI_SUCCESS);
- SEND_STATE_TRANSITION(S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_SEND_MSG_1_WAIT_CQ:
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- COND_SEND_STATE_TRANSITION(ret, 1,
- S_STATE_DONE,
- S_STATE_SEND_MSG_1_WAIT_CQ);
- break;
- case S_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- switch (r_state) {
- case R_STATE_PEEK:
- ret = fi_trecvmsg(ep[1], &peek_msg, FI_PEEK);
-
- COND_RECV_STATE_TRANSITION(ret, FI_SUCCESS,
- R_STATE_PEEK_WAIT_CQ,
- R_STATE_PEEK);
- break;
- case R_STATE_PEEK_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_peek_cqe, 1);
- if (ret == -FI_EAVAIL) {
- ret = fi_cq_readerr(msg_cq[1], &cqe_error, 0);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_PEEK,
- R_STATE_PEEK_WAIT_CQ);
- } else {
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_RECV_MSG_1,
- R_STATE_PEEK_WAIT_CQ);
- }
- break;
- case R_STATE_RECV_MSG_1:
- ret = fi_trecvmsg(ep[1], &msg, 0);
- cr_assert_eq(ret, FI_SUCCESS);
- RECV_STATE_TRANSITION(R_STATE_RECV_MSG_1_WAIT_CQ);
- break;
- case R_STATE_RECV_MSG_1_WAIT_CQ:
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- COND_RECV_STATE_TRANSITION(ret, 1,
- R_STATE_DONE,
- R_STATE_RECV_MSG_1_WAIT_CQ);
- break;
- case R_STATE_DONE:
- break;
- default:
- RAISE_UNREACHABLE_STATE;
- break;
- }
-
- update_test_timer();
- } while (!TEST_TIME_LIMIT_EXPIRED && !SEND_RECV_DONE);
-
- ASSERT_SEND_RECV_DONE;
-
- /* validate the expected results */
- validate_cqe_contents(&s_cqe, TSEND_FLAGS, source, len, len, target);
- validate_cqe_contents(&d_peek_cqe, TRECV_FLAGS | FI_PEEK,
- peek_msg.msg_iov[0].iov_base, len, peek_msg.tag,
- peek_msg.context);
-
- /* if CQE provided a buffer back, the data was copied.
- * Check the data */
- if (d_peek_cqe.buf) {
- cr_assert(rdm_fi_pdc_check_data_pattern(peek_buffer, 0xab,
- peek_iov.iov_len),
- "Data mismatch");
- }
-
- validate_cqe_with_message(&d_cqe, &msg, TRECV_FLAGS);
- cr_assert(rdm_fi_pdc_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_fi_pdc, peek_event_present_small_buff_provided)
-{
- rdm_fi_pdc_xfer_for_each_size(
- pdc_peek_event_present_small_buffer_provided,
- 1, BUF_SZ);
-}
diff --git a/prov/gni/test/rdm_rx_overrun.c b/prov/gni/test/rdm_rx_overrun.c
deleted file mode 100644
index 87fe215..0000000
--- a/prov/gni/test/rdm_rx_overrun.c
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <pthread.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include "gnix_vc.h"
-#include "gnix_cm_nic.h"
-#include "gnix_hashtable.h"
-#include "gnix_rma.h"
-
-#include <criterion/criterion.h>
-
-#define NUM_EPS 61
-const int num_msgs = 10;
-
-/*
- * Note that even tho we will use a min RX CQ size of 1, ugni seems to
- * have an internal minimum that is around 511, so in order for this
- * test to exercise the overrun code, it must send at least 511
- * messages (i.e., NUM_EPS*num_msgs > 511)
- */
-const int min_rx_cq_size = 1;
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom[NUM_EPS];
-static struct fid_ep *ep[NUM_EPS];
-static struct fid_av *av[NUM_EPS];
-static struct fi_info *hints;
-static struct fi_info *fi;
-static void *ep_name[NUM_EPS];
-static fi_addr_t gni_addr[NUM_EPS];
-static struct fid_cq *msg_cq[NUM_EPS];
-static struct fi_cq_attr cq_attr;
-
-static int target[NUM_EPS];
-static int source[NUM_EPS];
-struct fid_mr *rem_mr[NUM_EPS], *loc_mr[NUM_EPS];
-static uint64_t mr_key[NUM_EPS];
-
-static void setup(void)
-{
- int i, j;
- int ret = 0;
- struct fi_av_attr attr;
- size_t addrlen = 0;
- struct fi_gni_ops_domain *gni_domain_ops;
- uint32_t rx_cq_size;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = 4;
- hints->domain_attr->data_progress = FI_PROGRESS_MANUAL;
-
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- attr.type = FI_AV_TABLE;
- attr.count = NUM_EPS;
-
- cq_attr.format = FI_CQ_FORMAT_CONTEXT;
- cq_attr.size = 1024;
- cq_attr.wait_obj = 0;
-
- for (i = 0; i < NUM_EPS; i++) {
- ret = fi_domain(fab, fi, &dom[i], NULL);
- cr_assert(!ret, "fi_domain");
-
- ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1, 0,
- (void **) &gni_domain_ops, NULL);
- cr_assert(ret == FI_SUCCESS, "fi_open_ops");
-
- rx_cq_size = min_rx_cq_size;
-
- ret = gni_domain_ops->set_val(&dom[i]->fid, GNI_RX_CQ_SIZE,
- &rx_cq_size);
- cr_assert(ret == FI_SUCCESS, "set_val");
-
- ret = fi_av_open(dom[i], &attr, &av[i], NULL);
- cr_assert(!ret, "fi_av_open");
-
- ret = fi_endpoint(dom[i], fi, &ep[i], NULL);
- cr_assert(!ret, "fi_endpoint");
- cr_assert(ep[i]);
- ret = fi_cq_open(dom[i], &cq_attr, &msg_cq[i], 0);
- cr_assert(!ret, "fi_cq_open");
- ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
- }
-
- ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
- cr_assert_eq(ret, -FI_ETOOSMALL);
- cr_assert(addrlen > 0);
-
- for (i = 0; i < NUM_EPS; i++) {
- ep_name[i] = malloc(addrlen);
- cr_assert(ep_name[i] != NULL);
- ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
- cr_assert(ret == FI_SUCCESS);
- for (j = 0; j < NUM_EPS; j++) {
- ret = fi_av_insert(av[j], ep_name[i],
- 1, &gni_addr[i], 0, NULL);
- cr_assert(ret == 1);
- }
- }
-
- for (i = 0; i < NUM_EPS; i++) {
- ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
- ret = fi_enable(ep[i]);
- cr_assert(!ret, "fi_ep_enable");
-
- ret = fi_mr_reg(dom[i], target, NUM_EPS*sizeof(int),
- FI_RECV, 0, 0, 0, &rem_mr[i], &target);
- cr_assert_eq(ret, 0);
-
- ret = fi_mr_reg(dom[i], source, NUM_EPS*sizeof(int),
- FI_SEND, 0, 0, 0, &loc_mr[i], &source);
- cr_assert_eq(ret, 0);
-
- mr_key[i] = fi_mr_key(rem_mr[i]);
- }
-}
-
-static void teardown(void)
-{
- int i;
- int ret = 0;
-
- for (i = 0; i < NUM_EPS; i++) {
- fi_close(&loc_mr[i]->fid);
- fi_close(&rem_mr[i]->fid);
-
- ret = fi_close(&ep[i]->fid);
- cr_assert(!ret, "failure in closing ep.");
- ret = fi_close(&msg_cq[i]->fid);
- cr_assert(!ret, "failure in msg cq.");
- free(ep_name[i]);
-
- ret = fi_close(&av[i]->fid);
- cr_assert(!ret, "failure in closing av.");
-
- ret = fi_close(&dom[i]->fid);
- cr_assert(!ret, "failure in closing domain.");
- }
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-}
-
-TestSuite(rdm_rx_overrun, .init = setup, .fini = teardown, .disabled = false);
-
-
-Test(rdm_rx_overrun, all_to_one)
-{
- int i, j;
- int source_done = 0, dest_done = 0;
- struct fi_cq_entry s_cqe, d_cqe;
- ssize_t sz;
- int ctx[NUM_EPS];
-
- for (i = 0; i < NUM_EPS; i++) {
- source[i] = i;
- target[i] = -1;
- ctx[i] = -1;
- }
-
- for (i = 1; i < NUM_EPS; i++) {
- for (j = 0; j < num_msgs; j++) {
- sz = fi_send(ep[i], &source[i], sizeof(int), loc_mr,
- gni_addr[0], ctx+i);
- cr_assert_eq(sz, 0);
- }
- }
-
- do {
- for (i = 1; i < NUM_EPS; i++) {
- for (j = 0; j < num_msgs; j++) {
- if (fi_cq_read(msg_cq[i], &s_cqe, 1) == 1) {
- cr_assert(((uint64_t) s_cqe.op_context
- >= (uint64_t) ctx) &&
- ((uint64_t) s_cqe.op_context
- <=
- (uint64_t) (ctx+NUM_EPS-1)));
- source_done += 1;
- }
- }
- }
- } while (source_done != num_msgs*(NUM_EPS-1));
-
- for (i = 1; i < NUM_EPS; i++) {
- for (j = 0; j < num_msgs; j++) {
- sz = fi_recv(ep[0], &target[i], sizeof(int), rem_mr,
- gni_addr[i], ctx+i);
- cr_assert_eq(sz, 0);
- }
- }
-
- do {
- for (i = 1; i < NUM_EPS; i++) {
- for (j = 0; j < num_msgs; j++) {
- if (fi_cq_read(msg_cq[0], &d_cqe, 1) == 1) {
- cr_assert(((uint64_t) d_cqe.op_context
- >= (uint64_t) ctx) &&
- ((uint64_t) d_cqe.op_context
- <=
- (uint64_t) (ctx+NUM_EPS-1)));
- dest_done += 1;
- }
- }
- }
- } while (dest_done != num_msgs*(NUM_EPS-1));
-
-
- /* good enough error checking (only checks the last send) */
- for (i = 1; i < NUM_EPS; i++) {
- cr_assert(target[i] < NUM_EPS);
- ctx[target[i]] = target[i];
- }
-
- for (i = 1; i < NUM_EPS; i++) {
- cr_assert(ctx[i] == i);
- }
-
-}
-
diff --git a/prov/gni/test/rdm_sr.c b/prov/gni/test/rdm_sr.c
deleted file mode 100644
index a362717..0000000
--- a/prov/gni/test/rdm_sr.c
+++ /dev/null
@@ -1,1587 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <pthread.h>
-#include <unistd.h>
-#include <limits.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include "gnix_vc.h"
-#include "gnix_cm_nic.h"
-#include "gnix_hashtable.h"
-#include "gnix_rma.h"
-
-#include <criterion/criterion.h>
-
-#if 1
-#define dbg_printf(...)
-#else
-#define dbg_printf(...) fprintf(stderr, __VA_ARGS__); fflush(stderr)
-#endif
-
-#define NUMEPS 2
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom[NUMEPS];
-struct fi_gni_ops_domain *gni_domain_ops[NUMEPS];
-static struct fid_ep *ep[NUMEPS];
-static struct fid_av *av[NUMEPS];
-void *ep_name[NUMEPS];
-fi_addr_t gni_addr[NUMEPS];
-static struct fid_cq *msg_cq[NUMEPS];
-static struct fi_info *fi[NUMEPS];
-static struct fi_cq_attr cq_attr;
-const char *cdm_id[NUMEPS] = { "5000", "5001" };
-struct fi_info *hints;
-static int using_bnd_ep = 0;
-
-#define BUF_SZ (1<<20)
-char *target;
-char *source;
-char *uc_target;
-char *uc_source;
-struct fid_mr *rem_mr[NUMEPS], *loc_mr[NUMEPS];
-uint64_t mr_key[NUMEPS];
-
-static struct fid_cntr *send_cntr[NUMEPS], *recv_cntr[NUMEPS];
-static struct fi_cntr_attr cntr_attr = {
- .events = FI_CNTR_EVENTS_COMP,
- .flags = 0
-};
-static uint64_t sends[NUMEPS] = {0}, recvs[NUMEPS] = {0},
- send_errs[NUMEPS] = {0}, recv_errs[NUMEPS] = {0};
-
-void rdm_sr_setup_common_eps(void)
-{
- int ret = 0, i = 0, j = 0;
- struct fi_av_attr attr;
- size_t addrlen = 0;
-
- attr.type = FI_AV_MAP;
- attr.count = NUMEPS;
-
- cq_attr.format = FI_CQ_FORMAT_TAGGED;
- cq_attr.size = 1024;
- cq_attr.wait_obj = 0;
-
- target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */
- assert(target);
-
- source = malloc(BUF_SZ);
- assert(source);
-
- uc_target = malloc(BUF_SZ);
- assert(uc_target);
-
- uc_source = malloc(BUF_SZ);
- assert(uc_source);
-
- ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- for (; i < NUMEPS; i++) {
- ret = fi_domain(fab, fi[i], dom + i, NULL);
- cr_assert(!ret, "fi_domain");
-
- ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
- 0, (void **) (gni_domain_ops + i), NULL);
-
- ret = fi_av_open(dom[i], &attr, av + i, NULL);
- cr_assert(!ret, "fi_av_open");
-
- ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
- cr_assert(!ret, "fi_endpoint");
-
- ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
- cr_assert(addrlen > 0);
-
- ep_name[i] = malloc(addrlen);
- cr_assert(ep_name[i] != NULL);
-
- ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
- cr_assert(ret == FI_SUCCESS);
- }
-
- for (i = 0; i < NUMEPS; i++) {
- /* Insert all gni addresses into each av */
- for (j = 0; j < NUMEPS; j++) {
- ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
- 0, NULL);
- cr_assert(ret == 1);
- }
-
- ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_enable(ep[i]);
- cr_assert(!ret, "fi_ep_enable");
-
- ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
- cr_assert(!ret, "fi_cntr_open");
-
- ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
- }
-}
-
-void rdm_sr_setup_common(void)
-{
- int ret = 0, i = 0;
-
- rdm_sr_setup_common_eps();
-
- for (i = 0; i < NUMEPS; i++) {
- ret = fi_mr_reg(dom[i], target, 3 * BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, rem_mr + i, &target);
- cr_assert_eq(ret, 0);
-
- ret = fi_mr_reg(dom[i], source, BUF_SZ,
- FI_REMOTE_WRITE, 0, 0, 0, loc_mr + i, &source);
- cr_assert_eq(ret, 0);
-
- mr_key[i] = fi_mr_key(rem_mr[i]);
- }
-}
-
-void rdm_sr_setup(bool is_noreg, enum fi_progress pm)
-{
- int ret = 0, i = 0;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = NUMEPS * 2;
- hints->domain_attr->control_progress = pm;
- hints->domain_attr->data_progress = pm;
- hints->mode = ~0;
- hints->caps = is_noreg ? hints->caps : FI_SOURCE | FI_MSG;
- hints->fabric_attr->name = strdup("gni");
-
- /* Get info about fabric services with the provided hints */
- for (; i < NUMEPS; i++) {
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi[i]);
- cr_assert(!ret, "fi_getinfo");
- }
-
- if (is_noreg)
- rdm_sr_setup_common_eps();
- else
- rdm_sr_setup_common();
-}
-
-static void rdm_sr_setup_reg(void) {
- rdm_sr_setup(false, FI_PROGRESS_AUTO);
-}
-
-static void rdm_sr_setup_noreg(void) {
- rdm_sr_setup(true, FI_PROGRESS_AUTO);
-}
-
-void rdm_sr_bnd_ep_setup(void)
-{
- int ret = 0, i = 0;
- char my_hostname[HOST_NAME_MAX];
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = NUMEPS * 2;
- hints->mode = ~0;
- hints->fabric_attr->name = strdup("gni");
-
- ret = gethostname(my_hostname, sizeof(my_hostname));
- cr_assert(!ret, "gethostname");
-
- for (; i < NUMEPS; i++) {
- ret = fi_getinfo(FI_VERSION(1, 0), my_hostname,
- cdm_id[i], 0, hints, fi + i);
- cr_assert(!ret, "fi_getinfo");
- }
-
- using_bnd_ep = 1;
-
- rdm_sr_setup_common();
-}
-
-static void rdm_sr_teardown_common(bool unreg)
-{
- int ret = 0, i = 0;
-
- free(uc_source);
- free(uc_target);
- free(target);
- free(source);
-
- for (; i < NUMEPS; i++) {
- fi_close(&recv_cntr[i]->fid);
- fi_close(&send_cntr[i]->fid);
-
- if (unreg) {
- fi_close(&loc_mr[i]->fid);
- fi_close(&rem_mr[i]->fid);
- }
-
- ret = fi_close(&ep[i]->fid);
- cr_assert(!ret, "failure in closing ep.");
-
- ret = fi_close(&msg_cq[i]->fid);
- cr_assert(!ret, "failure in send cq.");
-
- ret = fi_close(&av[i]->fid);
- cr_assert(!ret, "failure in closing av.");
-
- ret = fi_close(&dom[i]->fid);
- cr_assert(!ret, "failure in closing domain.");
-
- fi_freeinfo(fi[i]);
-
- free(ep_name[i]);
- }
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
-
- fi_freeinfo(hints);
-}
-
-static void rdm_sr_teardown(void)
-{
- rdm_sr_teardown_common(true);
-}
-
-static void rdm_sr_teardown_nounreg(void)
-{
- rdm_sr_teardown_common(false);
-}
-
-
-void rdm_sr_init_data(char *buf, int len, char seed)
-{
- int i;
-
- for (i = 0; i < len; i++) {
- buf[i] = seed++;
- }
-}
-
-int rdm_sr_check_data(char *buf1, char *buf2, int len)
-{
- int i;
-
- for (i = 0; i < len; i++) {
- if (buf1[i] != buf2[i]) {
- printf("data mismatch, elem: %d, exp: %hhx, act: %hhx\n"
- , i, buf1[i], buf2[i]);
- return 0;
- }
- }
-
- return 1;
-}
-
-void rdm_sr_xfer_for_each_size(void (*xfer)(int len), int slen, int elen)
-{
- int i;
-
- for (i = slen; i <= elen; i *= 2) {
- xfer(i);
- }
-}
-
-void rdm_sr_check_cqe(struct fi_cq_tagged_entry *cqe, void *ctx,
- uint64_t flags, void *addr, size_t len,
- uint64_t data)
-{
- cr_assert(cqe->op_context == ctx, "CQE Context mismatch");
- cr_assert(cqe->flags == flags, "CQE flags mismatch");
-
- if (flags & FI_RECV) {
- cr_assert(cqe->len == len, "CQE length mismatch");
-
- if (!(flags & FI_MULTI_RECV))
- cr_assert(cqe->buf == addr, "CQE address mismatch");
-
- if (flags & FI_REMOTE_CQ_DATA)
- cr_assert(cqe->data == data, "CQE data mismatch");
- } else {
- cr_assert(cqe->len == 0, "Invalid CQE length");
- cr_assert(cqe->buf == 0, "Invalid CQE address");
- cr_assert(cqe->data == 0, "Invalid CQE data");
- }
-
- cr_assert(cqe->tag == 0, "Invalid CQE tag");
-}
-
-void rdm_sr_check_cntrs(uint64_t s[], uint64_t r[], uint64_t s_e[],
- uint64_t r_e[])
-{
- int i = 0;
- for (; i < NUMEPS; i++) {
- sends[i] += s[i];
- recvs[i] += r[i];
- send_errs[i] += s_e[i];
- recv_errs[i] += r_e[i];
-
- cr_assert(fi_cntr_read(send_cntr[i]) == sends[i],
- "Bad send count");
- cr_assert(fi_cntr_read(recv_cntr[i]) == recvs[i],
- "Bad recv count");
- cr_assert(fi_cntr_readerr(send_cntr[i]) == send_errs[i],
- "Bad send err count");
- cr_assert(fi_cntr_readerr(recv_cntr[i]) == recv_errs[i],
- "Bad recv err count");
- }
-}
-
-void rdm_sr_err_inject_enable(void)
-{
- int ret, err_count_val = 1, i = 0;
-
- for (; i < NUMEPS; i++) {
- ret = gni_domain_ops[i]->set_val(&dom[i]->fid,
- GNI_ERR_INJECT_COUNT,
- &err_count_val);
- cr_assert(!ret, "setval(GNI_ERR_INJECT_COUNT)");
- }
-}
-
-void rdm_sr_lazy_dereg_disable(void)
-{
- int ret, lazy_dereg_val = 0, i = 0;
-
- for (; i < NUMEPS; i++) {
- ret = gni_domain_ops[i]->set_val(&dom[i]->fid,
- GNI_MR_CACHE_LAZY_DEREG,
- &lazy_dereg_val);
- cr_assert(!ret, "setval(GNI_MR_CACHE_LAZY_DEREG)");
- }
-}
-
-/*******************************************************************************
- * Test MSG functions
- ******************************************************************************/
-
-TestSuite(rdm_sr, .init = rdm_sr_setup_reg, .fini = rdm_sr_teardown,
- .disabled = false);
-
-TestSuite(rdm_sr_noreg, .init = rdm_sr_setup_noreg,
- .fini = rdm_sr_teardown_nounreg, .disabled = false);
-
-TestSuite(rdm_sr_bnd_ep, .init = rdm_sr_bnd_ep_setup, .fini = rdm_sr_teardown,
- .disabled = false);
-
-/*
- * ssize_t fi_send(struct fid_ep *ep, void *buf, size_t len,
- * void *desc, fi_addr_t dest_addr, void *context);
- *
- * ssize_t fi_recv(struct fid_ep *ep, void * buf, size_t len,
- * void *desc, fi_addr_t src_addr, void *context);
- */
-void do_send(int len)
-{
- int ret;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- ssize_t sz;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(source, len, 0xab);
- rdm_sr_init_data(target, len, 0);
-
- sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
- rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0);
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_sr, send)
-{
- rdm_sr_xfer_for_each_size(do_send, 1, BUF_SZ);
-}
-
-Test(rdm_sr, send_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_send, 1, BUF_SZ);
-}
-
-/*
-ssize_t fi_sendv(struct fid_ep *ep, const struct iovec *iov, void **desc,
- size_t count, fi_addr_t dest_addr, void *context);
-*/
-void do_sendv(int len)
-{
- int ret;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- ssize_t sz;
- struct iovec iov;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- iov.iov_base = source;
- iov.iov_len = len;
-
- rdm_sr_init_data(source, len, 0x25);
- rdm_sr_init_data(target, len, 0);
-
- sz = fi_sendv(ep[0], &iov, (void **)loc_mr, 1, gni_addr[1], target);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], target, len, rem_mr[0], gni_addr[0], source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
- rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0);
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_sr, sendv)
-{
- rdm_sr_xfer_for_each_size(do_sendv, 1, BUF_SZ);
-}
-
-Test(rdm_sr, sendv_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_sendv, 1, BUF_SZ);
-}
-
-/*
-ssize_t fi_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
- uint64_t flags);
-*/
-void do_sendmsg(int len)
-{
- int ret;
- ssize_t sz;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- struct fi_msg msg;
- struct iovec iov;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- iov.iov_base = source;
- iov.iov_len = len;
-
- msg.msg_iov = &iov;
- msg.desc = (void **)loc_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[1];
- msg.context = target;
- msg.data = (uint64_t)target;
-
- rdm_sr_init_data(source, len, 0xef);
- rdm_sr_init_data(target, len, 0);
-
- sz = fi_sendmsg(ep[0], &msg, 0);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], target, len, rem_mr[0], gni_addr[0], source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
- rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0);
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_sr, sendmsg)
-{
- rdm_sr_xfer_for_each_size(do_sendmsg, 1, BUF_SZ);
-}
-
-Test(rdm_sr, sendmsg_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_sendmsg, 1, BUF_SZ);
-}
-
-/*
-ssize_t fi_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
- uint64_t flags);
-*/
-
-void do_sendmsgdata(int len)
-{
- int ret;
- ssize_t sz;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- struct fi_msg msg;
- struct iovec iov;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- iov.iov_base = source;
- iov.iov_len = len;
-
- msg.msg_iov = &iov;
- msg.desc = (void **)loc_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[1];
- msg.context = target;
- msg.data = (uint64_t)source;
-
- rdm_sr_init_data(source, len, 0xef);
- rdm_sr_init_data(target, len, 0);
-
- sz = fi_sendmsg(ep[0], &msg, FI_REMOTE_CQ_DATA);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], target, len, rem_mr[0], gni_addr[0], source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
- rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV|FI_REMOTE_CQ_DATA),
- target, len, (uint64_t)source);
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_sr, sendmsgdata)
-{
- rdm_sr_xfer_for_each_size(do_sendmsgdata, 1, BUF_SZ);
-}
-
-Test(rdm_sr, sendmsgdata_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_sendmsgdata, 1, BUF_SZ);
-}
-
-/*
-ssize_t fi_inject(struct fid_ep *ep, void *buf, size_t len,
- fi_addr_t dest_addr);
-*/
-#define INJECT_SIZE 64
-void do_inject(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(source, len, 0x23);
- rdm_sr_init_data(target, len, 0);
-
- sz = fi_inject(ep[0], source, len, gni_addr[1]);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(msg_cq[1], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- /* Manually progress connection to domain 1 */
- fi_cq_read(msg_cq[0], &cqe, 1);
- }
-
- cr_assert_eq(ret, 1);
- rdm_sr_check_cqe(&cqe, source, (FI_MSG|FI_RECV),
- target, len, (uint64_t)source);
-
- dbg_printf("got recv context event!\n");
-
- /* do progress until send counter is updated */
- while (fi_cntr_read(send_cntr[0]) < 1) {
- pthread_yield();
- }
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- /* make sure inject does not generate a send competion */
- cr_assert_eq(fi_cq_read(msg_cq[0], &cqe, 1), -FI_EAGAIN);
-
- cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_sr, inject)
-{
- rdm_sr_xfer_for_each_size(do_inject, 1, INJECT_SIZE);
-}
-
-/*
- * this test attempts to demonstrate issue ofi-cray/libfabric-cray#559.
- * For domains with control_progress AUTO, this test should not hang.
- */
-Test(rdm_sr, inject_progress)
-{
- int ret, len = 64;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(source, len, 0x23);
- rdm_sr_init_data(target, len, 0);
-
- sz = fi_inject(ep[0], source, len, gni_addr[1]);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source);
- cr_assert_eq(sz, 0);
-
- /*
- * do progress until send counter is updated.
- * This works because we have FI_PROGRESS_AUTO for control progress
- */
- while (fi_cntr_read(send_cntr[0]) < 1) {
- pthread_yield();
- }
-
- while ((ret = fi_cq_read(msg_cq[1], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- rdm_sr_check_cqe(&cqe, source, (FI_MSG|FI_RECV),
- target, len, (uint64_t)source);
-
- dbg_printf("got recv context event!\n");
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- /* make sure inject does not generate a send competion */
- cr_assert_eq(fi_cq_read(msg_cq[0], &cqe, 1), -FI_EAGAIN);
-
- cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_sr, inject_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_inject, 1, INJECT_SIZE);
-}
-
-/*
-ssize_t fi_senddata(struct fid_ep *ep, void *buf, size_t len, void *desc,
- uint64_t data, fi_addr_t dest_addr, void *context);
-*/
-void do_senddata(int len)
-{
- int ret;
- ssize_t sz;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(source, len, 0xab);
- rdm_sr_init_data(target, len, 0);
-
- sz = fi_senddata(ep[0], source, len, loc_mr[0], (uint64_t)source,
- gni_addr[1], target);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], target, len, rem_mr[0], gni_addr[0], source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
- rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV|FI_REMOTE_CQ_DATA),
- target, len, (uint64_t)source);
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_sr, senddata)
-{
- rdm_sr_xfer_for_each_size(do_senddata, 1, BUF_SZ);
-}
-
-Test(rdm_sr, senddata_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_senddata, 1, BUF_SZ);
-}
-
-/*
-ssize_t fi_injectdata(struct fid_ep *ep, const void *buf, size_t len,
- uint64_t data, fi_addr_t dest_addr);
-*/
-void do_injectdata(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(source, len, 0xab);
- rdm_sr_init_data(target, len, 0);
-
- sz = fi_injectdata(ep[0], source, len, (uint64_t)source, gni_addr[1]);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], target, len, rem_mr[0], gni_addr[0], source);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(msg_cq[1], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- /* Manually progress connection to domain 1 */
- fi_cq_read(msg_cq[0], &cqe, 1);
- }
-
- rdm_sr_check_cqe(&cqe, source, (FI_MSG|FI_RECV|FI_REMOTE_CQ_DATA),
- target, len, (uint64_t)source);
-
- dbg_printf("got recv context event!\n");
-
- /* don't progress until send counter is updated */
- while (fi_cntr_read(send_cntr[0]) < 1) {
- pthread_yield();
- }
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- /* make sure inject does not generate a send competion */
- cr_assert_eq(fi_cq_read(msg_cq[0], &cqe, 1), -FI_EAGAIN);
-
- cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_sr, injectdata)
-{
- rdm_sr_xfer_for_each_size(do_injectdata, 1, INJECT_SIZE);
-}
-
-Test(rdm_sr, injectdata_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_injectdata, 1, INJECT_SIZE);
-}
-
-/*
-ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc,
- size_t count, fi_addr_t src_addr, void *context);
-*/
-void do_recvv(int len)
-{
- int ret;
- ssize_t sz;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- struct iovec iov;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(source, len, 0xab);
- rdm_sr_init_data(target, len, 0);
-
- sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
- cr_assert_eq(sz, 0);
-
- iov.iov_base = target;
- iov.iov_len = len;
-
- sz = fi_recvv(ep[1], &iov, (void **)rem_mr, 1, gni_addr[0], source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
- rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0);
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_sr, recvv)
-{
- rdm_sr_xfer_for_each_size(do_recvv, 1, BUF_SZ);
-}
-
-Test(rdm_sr, recvv_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_recvv, 1, BUF_SZ);
-}
-
-/*
-ssize_t (*recvmsg)(struct fid_ep *ep, const struct fi_msg *msg,
- uint64_t flags);
-*/
-void do_recvmsg(int len)
-{
- int ret;
- ssize_t sz;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- struct iovec iov;
- struct fi_msg msg;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(source, len, 0xab);
- rdm_sr_init_data(target, len, 0);
-
- sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
- cr_assert_eq(sz, 0);
-
- iov.iov_base = target;
- iov.iov_len = len;
-
- msg.msg_iov = &iov;
- msg.desc = (void **)rem_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[0];
- msg.context = source;
- msg.data = (uint64_t)source;
-
- sz = fi_recvmsg(ep[1], &msg, 0);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
- rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0);
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_sr, recvmsg)
-{
- rdm_sr_xfer_for_each_size(do_recvmsg, 1, BUF_SZ);
-}
-
-Test(rdm_sr, recvmsg_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_recvmsg, 1, BUF_SZ);
-}
-
-Test(rdm_sr_bnd_ep, recvmsg)
-{
- rdm_sr_xfer_for_each_size(do_recvmsg, 1, BUF_SZ);
-}
-
-void do_send_autoreg(int len)
-{
- int ret;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- ssize_t sz;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(source, len, 0xab);
- rdm_sr_init_data(target, len, 0);
-
- sz = fi_send(ep[0], source, len, NULL, gni_addr[1], target);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], target, len, NULL, gni_addr[0], source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
- rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0);
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_sr, send_autoreg)
-{
- rdm_sr_xfer_for_each_size(do_send_autoreg, 1, BUF_SZ);
-}
-
-Test(rdm_sr, send_autoreg_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_send_autoreg, 1, BUF_SZ);
-}
-
-void do_send_autoreg_uncached(int len)
-{
- int ret;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- ssize_t sz;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(uc_source, len, 0xab);
- rdm_sr_init_data(uc_target, len, 0);
-
- sz = fi_send(ep[0], uc_source, len, NULL, gni_addr[1], uc_target);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], uc_target, len, NULL, gni_addr[0], uc_source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- rdm_sr_check_cqe(&s_cqe, uc_target, (FI_MSG|FI_SEND), 0, 0, 0);
- rdm_sr_check_cqe(&d_cqe, uc_source, (FI_MSG|FI_RECV),
- uc_target, len, 0);
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_sr_check_data(uc_source, uc_target, len),
- "Data mismatch");
-}
-
-Test(rdm_sr, send_autoreg_uncached)
-{
- rdm_sr_xfer_for_each_size(do_send_autoreg_uncached, 1, BUF_SZ);
-}
-
-Test(rdm_sr, send_autoreg_uncached_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_send_autoreg_uncached, 1, BUF_SZ);
-}
-
-void do_send_err(int len)
-{
- int ret;
- struct fi_cq_tagged_entry s_cqe;
- struct fi_cq_err_entry err_cqe;
- ssize_t sz;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(source, len, 0xab);
- rdm_sr_init_data(target, len, 0);
-
- sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(msg_cq[0], &s_cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, -FI_EAVAIL);
-
- ret = fi_cq_readerr(msg_cq[0], &err_cqe, 0);
- cr_assert_eq(ret, 1);
-
- cr_assert((uint64_t)err_cqe.op_context == (uint64_t)target,
- "Bad error context");
- cr_assert(err_cqe.flags == (FI_MSG | FI_SEND));
- cr_assert(err_cqe.len == 0, "Bad error len");
- cr_assert(err_cqe.buf == 0, "Bad error buf");
- cr_assert(err_cqe.data == 0, "Bad error data");
- cr_assert(err_cqe.tag == 0, "Bad error tag");
- cr_assert(err_cqe.olen == 0, "Bad error olen");
- cr_assert(err_cqe.err == FI_ECANCELED, "Bad error errno");
- cr_assert(err_cqe.prov_errno == GNI_RC_TRANSACTION_ERROR,
- "Bad prov errno");
- cr_assert(err_cqe.err_data == NULL, "Bad error provider data");
-
- s_e[0] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-}
-
-Test(rdm_sr, send_err)
-{
- int ret, max_retrans_val = 0, i = 0; /* 0 to force SMSG failure */
-
- for (; i < NUMEPS; i++) {
- ret = gni_domain_ops[i]->set_val(&dom[i]->fid,
- GNI_MAX_RETRANSMITS,
- &max_retrans_val);
- cr_assert(!ret, "setval(GNI_MAX_RETRANSMITS)");
- }
- rdm_sr_err_inject_enable();
-
- rdm_sr_xfer_for_each_size(do_send_err, 1, BUF_SZ);
-}
-
-void do_send_autoreg_uncached_nolazydereg(int len)
-{
- int ret;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- ssize_t sz;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(uc_source, len, 0xab);
- rdm_sr_init_data(uc_target, len, 0);
-
- sz = fi_send(ep[0], uc_source, len, NULL, gni_addr[1], uc_target);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], uc_target, len, NULL, gni_addr[0], uc_source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- rdm_sr_check_cqe(&s_cqe, uc_target, (FI_MSG|FI_SEND), 0, 0, 0);
- rdm_sr_check_cqe(&d_cqe, uc_source, (FI_MSG|FI_RECV),
- uc_target, len, 0);
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_sr_check_data(uc_source, uc_target, len),
- "Data mismatch");
-}
-
-Test(rdm_sr_noreg, send_autoreg_uncached_nolazydereg)
-{
- rdm_sr_lazy_dereg_disable();
- rdm_sr_xfer_for_each_size(do_send_autoreg_uncached_nolazydereg,
- 1, BUF_SZ);
-}
-
-Test(rdm_sr, send_readfrom)
-{
- int ret;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- ssize_t sz;
- fi_addr_t src_addr;
- int len = 64;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(source, len, 0xab);
- rdm_sr_init_data(target, len, 0);
-
- sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], target, len, rem_mr[0], gni_addr[0], source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_readfrom(msg_cq[1], &d_cqe, 1, &src_addr);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
- rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0);
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- cr_assert(src_addr == gni_addr[0], "src_addr mismatch");
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
-}
-
-void do_send_buf(void *p, void *t, int len)
-{
- int ret;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- ssize_t sz;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(p, len, 0xab);
- rdm_sr_init_data(t, len, 0);
-
- sz = fi_send(ep[0], p, len, loc_mr[0], gni_addr[1], t);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], t, len, rem_mr[0], gni_addr[0], p);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- rdm_sr_check_cqe(&s_cqe, t, (FI_MSG|FI_SEND), 0, 0, 0);
- rdm_sr_check_cqe(&d_cqe, p, (FI_MSG|FI_RECV), t, len, 0);
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_sr_check_data(p, t, len), "Data mismatch");
-}
-
-void do_send_alignment(int len)
-{
- int s_off, t_off, l_off;
-
- for (s_off = 0; s_off < 7; s_off++) {
- for (t_off = 0; t_off < 7; t_off++) {
- for (l_off = 0; l_off < 7; l_off++) {
- do_send_buf(source + s_off,
- target + t_off,
- len + l_off);
- }
- }
- }
-}
-
-Test(rdm_sr, send_alignment)
-{
- rdm_sr_xfer_for_each_size(do_send_alignment, 1, (BUF_SZ-1));
-}
-
-Test(rdm_sr, send_alignment_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_send_alignment, 1, (BUF_SZ-1));
-}
-
-void do_sendrecv_buf(void *p, void *t, int send_len, int recv_len)
-{
- int ret;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- ssize_t sz;
- int xfer_len;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
-
- rdm_sr_init_data(p, send_len, 0xab);
- rdm_sr_init_data(t, recv_len, 0);
-
- sz = fi_send(ep[0], p, send_len, loc_mr[0], gni_addr[1], t);
- cr_assert_eq(sz, 0);
-
- sz = fi_recv(ep[1], t, recv_len, rem_mr[0], gni_addr[0], p);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- xfer_len = MIN(send_len, recv_len);
- rdm_sr_check_cqe(&s_cqe, t, (FI_MSG|FI_SEND), 0, 0, 0);
- rdm_sr_check_cqe(&d_cqe, p, (FI_MSG|FI_RECV), t, xfer_len, 0);
-
- s[0] = 1; r[1] = 1;
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_sr_check_data(p, t, xfer_len), "Data mismatch");
-}
-
-void do_sendrecv_alignment(int len)
-{
- int s_off, t_off, sl_off, rl_off;
-
- for (s_off = 0; s_off < 8; s_off++) {
- for (t_off = 0; t_off < 8; t_off++) {
- for (sl_off = -7; sl_off < 8; sl_off++) {
- for (rl_off = -7; rl_off < 8; rl_off++) {
- do_sendrecv_buf(source + s_off,
- target + t_off,
- len + sl_off,
- len + rl_off);
- }
- }
- }
- }
-}
-
-Test(rdm_sr, sendrecv_alignment)
-{
- rdm_sr_xfer_for_each_size(do_sendrecv_alignment, 8*1024, 16*1024);
-}
-
-Test(rdm_sr, sendrecv_alignment_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_sendrecv_alignment, 8*1024, 32*1024);
-}
-
-void do_multirecv(int len)
-{
- int i, ret;
- ssize_t sz;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- struct iovec iov;
- struct fi_msg msg;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
- int nrecvs = 3;
-
- rdm_sr_init_data(source, len, 0xab);
- rdm_sr_init_data(target, len, 0);
-
- /* Post receives first to force matching in SMSG callback. */
- iov.iov_base = target;
- iov.iov_len = len * nrecvs + 63;
-
- msg.msg_iov = &iov;
- msg.desc = (void **)rem_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[0];
- msg.context = source;
- msg.data = (uint64_t)source;
-
- sz = fi_recvmsg(ep[1], &msg, FI_MULTI_RECV);
- cr_assert_eq(sz, 0);
-
- for (i = 0; i < nrecvs; i++) {
- sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1],
- target);
- cr_assert_eq(sz, 0);
- }
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND),
- 0, 0, 0);
- s[0]++;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- rdm_sr_check_cqe(&d_cqe, source,
- (FI_MSG|FI_RECV|FI_MULTI_RECV),
- target + (r[1] * len), len, 0);
- cr_assert(rdm_sr_check_data(source, d_cqe.buf, len),
- "Data mismatch");
- r[1]++;
- }
- } while (s[0] < nrecvs || r[1] < nrecvs);
-
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-}
-
-Test(rdm_sr, multirecv)
-{
- rdm_sr_xfer_for_each_size(do_multirecv, 1, BUF_SZ);
-}
-
-Test(rdm_sr, multirecv_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_multirecv, 1, BUF_SZ);
-}
-
-void do_multirecv2(int len)
-{
- int i, ret;
- ssize_t sz;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- struct iovec iov;
- struct fi_msg msg;
- uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
- uint64_t r_e[NUMEPS] = {0};
- int nrecvs = 3;
-
- rdm_sr_init_data(source, len, 0xab);
- rdm_sr_init_data(target, len, 0);
-
- /* Post sends first to force matching in the _gnix_recv() path. */
- for (i = 0; i < nrecvs; i++) {
- sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1],
- target);
- cr_assert_eq(sz, 0);
- }
-
- /* Progress our sends. */
- for (i = 0; i < 10000; i++) {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND),
- 0, 0, 0);
- s[0]++;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- cr_assert_eq(ret, -FI_EAGAIN);
- }
-
- iov.iov_base = target;
- iov.iov_len = len * nrecvs + 63;
-
- msg.msg_iov = &iov;
- msg.desc = (void **)rem_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[0];
- msg.context = source;
- msg.data = (uint64_t)source;
-
- sz = fi_recvmsg(ep[1], &msg, FI_MULTI_RECV);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND),
- 0, 0, 0);
- s[0]++;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- rdm_sr_check_cqe(&d_cqe, source,
- (FI_MSG|FI_RECV|FI_MULTI_RECV),
- target + (r[1] * len), len, 0);
- cr_assert(rdm_sr_check_data(source, d_cqe.buf, len),
- "Data mismatch");
- r[1]++;
- }
- } while (s[0] < nrecvs || r[1] < nrecvs);
-
- rdm_sr_check_cntrs(s, r, s_e, r_e);
-
- dbg_printf("got context events!\n");
-}
-
-Test(rdm_sr, multirecv2)
-{
- rdm_sr_xfer_for_each_size(do_multirecv2, 1, BUF_SZ);
-}
-
-Test(rdm_sr, multirecv2_retrans)
-{
- rdm_sr_err_inject_enable();
- rdm_sr_xfer_for_each_size(do_multirecv2, 1, BUF_SZ);
-}
-
diff --git a/prov/gni/test/rdm_tagged_sr.c b/prov/gni/test/rdm_tagged_sr.c
deleted file mode 100644
index 43e3fbc..0000000
--- a/prov/gni/test/rdm_tagged_sr.c
+++ /dev/null
@@ -1,810 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <pthread.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-#include <rdma/fi_tagged.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include "gnix_vc.h"
-#include "gnix_cm_nic.h"
-#include "gnix_hashtable.h"
-#include "gnix_rma.h"
-
-#include <criterion/criterion.h>
-
-#if 1
-#define dbg_printf(...)
-#else
-#define dbg_printf(...) \
- do { \
- fprintf(stderr, __VA_ARGS__); \
- fflush(stderr); \
- } while(0)
-#endif
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom;
-static struct fid_ep *ep[2];
-static struct fid_av *av;
-static struct fi_info *hints;
-static struct fi_info *fi;
-void *ep_name[2];
-size_t gni_addr[2];
-static struct fid_cq *msg_cq[2];
-static struct fi_cq_attr cq_attr;
-
-#define BUF_SZ (64*1024)
-char *target;
-char *source;
-struct fid_mr *rem_mr, *loc_mr;
-uint64_t mr_key;
-
-static void setup_dom(enum fi_progress pm)
-{
- int ret;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->data_progress = pm;
-
- hints->domain_attr->cq_data_size = 4;
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- ret = fi_domain(fab, fi, &dom, NULL);
- cr_assert(!ret, "fi_domain");
-
-}
-
-static void setup_ep(void)
-{
- int ret;
- struct fi_av_attr attr;
- size_t addrlen = 0;
-
- attr.type = FI_AV_MAP;
- attr.count = 16;
-
- ret = fi_av_open(dom, &attr, &av, NULL);
- cr_assert(!ret, "fi_av_open");
-
- ret = fi_endpoint(dom, fi, &ep[0], NULL);
- cr_assert(!ret, "fi_endpoint");
-
- cq_attr.format = FI_CQ_FORMAT_TAGGED;
- cq_attr.size = 1024;
- cq_attr.wait_obj = 0;
-
- ret = fi_cq_open(dom, &cq_attr, &msg_cq[0], 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_cq_open(dom, &cq_attr, &msg_cq[1], 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_ep_bind(ep[0], &msg_cq[0]->fid, FI_SEND | FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
- cr_assert(addrlen > 0);
-
- ep_name[0] = malloc(addrlen);
- cr_assert(ep_name[0] != NULL);
-
- ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_endpoint(dom, fi, &ep[1], NULL);
- cr_assert(!ret, "fi_endpoint");
-
- ret = fi_ep_bind(ep[1], &msg_cq[1]->fid, FI_SEND | FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
-
- ep_name[1] = malloc(addrlen);
- cr_assert(ep_name[1] != NULL);
-
- ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_av_insert(av, ep_name[0], 1, &gni_addr[0], 0,
- NULL);
- cr_assert(ret == 1);
-
- ret = fi_av_insert(av, ep_name[1], 1, &gni_addr[1], 0,
- NULL);
- cr_assert(ret == 1);
-
- ret = fi_ep_bind(ep[0], &av->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_ep_bind(ep[1], &av->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_enable(ep[0]);
- cr_assert(!ret, "fi_ep_enable");
-
- ret = fi_enable(ep[1]);
- cr_assert(!ret, "fi_ep_enable");
-}
-
-static void setup_mr(void)
-{
- int ret;
-
- target = malloc(BUF_SZ);
- assert(target);
-
- source = malloc(BUF_SZ);
- assert(source);
-
- ret = fi_mr_reg(dom, target, BUF_SZ,
- FI_SEND | FI_RECV, 0, 0, 0, &rem_mr, &target);
- cr_assert_eq(ret, 0);
-
- ret = fi_mr_reg(dom, source, BUF_SZ,
- FI_SEND | FI_RECV, 0, 0, 0, &loc_mr, &source);
- cr_assert_eq(ret, 0);
-
- mr_key = fi_mr_key(rem_mr);
-}
-
-static void rdm_tagged_sr_setup(void)
-{
- /* Change this to FI_PROGRESS_AUTO when supported */
- setup_dom(FI_PROGRESS_MANUAL);
- setup_ep();
- setup_mr();
-}
-
-static void rdm_tagged_sr_teardown(void)
-{
- int ret = 0;
-
- fi_close(&loc_mr->fid);
- fi_close(&rem_mr->fid);
-
- free(target);
- free(source);
-
- ret = fi_close(&ep[0]->fid);
- cr_assert(!ret, "failure in closing ep.");
-
- ret = fi_close(&ep[1]->fid);
- cr_assert(!ret, "failure in closing ep.");
-
- ret = fi_close(&msg_cq[0]->fid);
- cr_assert(!ret, "failure in send cq.");
-
- ret = fi_close(&msg_cq[1]->fid);
- cr_assert(!ret, "failure in recv cq.");
-
- ret = fi_close(&av->fid);
- cr_assert(!ret, "failure in closing av.");
-
- ret = fi_close(&dom->fid);
- cr_assert(!ret, "failure in closing domain.");
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
- free(ep_name[0]);
- free(ep_name[1]);
-}
-
-void rdm_tagged_sr_init_data(char *buf, int len, char seed)
-{
- int i;
-
- for (i = 0; i < len; i++) {
- buf[i] = seed++;
- }
-}
-
-int rdm_tagged_sr_check_data(char *buf1, char *buf2, int len)
-{
- int i;
-
- for (i = 0; i < len; i++) {
- if (buf1[i] != buf2[i]) {
- printf("data mismatch, elem: %d, exp: %x, act: %x\n",
- i, buf1[i], buf2[i]);
- return 0;
- }
- }
-
- return 1;
-}
-
-void rdm_tagged_sr_xfer_for_each_size(void (*xfer)(int len), int slen, int elen)
-{
- int i;
-
- for (i = slen; i <= elen; i *= 2) {
- xfer(i);
- }
-}
-
-/*******************************************************************************
- * Test MSG functions
- ******************************************************************************/
-
-TestSuite(rdm_tagged_sr,
- .init = rdm_tagged_sr_setup,
- .fini = rdm_tagged_sr_teardown,
- .disabled = false);
-
-/*
- * ssize_t fi_tsend(struct fid_ep *ep, void *buf, size_t len,
- * void *desc, fi_addr_t dest_addr, uint64_t tag, void *context);
- *
- * ssize_t fi_trecv(struct fid_ep *ep, void * buf, size_t len,
- * void *desc, fi_addr_t src_addr, uint64_t tag, uint64_t ignore,
- * void *context);
- */
-void do_tsend(int len)
-{
- int ret;
- ssize_t sz;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
-
- rdm_tagged_sr_init_data(source, len, 0xab);
- rdm_tagged_sr_init_data(target, len, 0);
-
- sz = fi_tsend(ep[0], source, len, loc_mr, gni_addr[1], len, target);
- cr_assert_eq(sz, 0);
-
- sz = fi_trecv(ep[1], target, len, rem_mr, gni_addr[0], len, 0, source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- dbg_printf("got context events!\n");
-
-
- cr_assert(rdm_tagged_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_tagged_sr, tsend)
-{
- rdm_tagged_sr_xfer_for_each_size(do_tsend, 1, BUF_SZ);
-}
-
-/*
-ssize_t fi_tsendv(struct fid_ep *ep, const struct iovec *iov,
- void **desc, size_t count, fi_addr_t dest_addr, uint64_t tag,
- void *context);
- */
-void do_tsendv(int len)
-{
- int ret;
- ssize_t sz;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- struct iovec iov;
-
- iov.iov_base = source;
- iov.iov_len = len;
-
- rdm_tagged_sr_init_data(source, len, 0x25);
- rdm_tagged_sr_init_data(target, len, 0);
-
- sz = fi_tsendv(ep[0], &iov, (void **)&loc_mr, 1, gni_addr[1], len, target);
- cr_assert_eq(sz, 0);
-
- sz = fi_trecv(ep[1], target, len, rem_mr, gni_addr[0], len, 0, source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- dbg_printf("got recv context event!\n");
-
- cr_assert(rdm_tagged_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_tagged_sr, tsendv)
-{
- rdm_tagged_sr_xfer_for_each_size(do_tsendv, 1, BUF_SZ);
-}
-
-/*
-ssize_t fi_tsendmsg(struct fid_ep *ep, const struct fi_msg *msg,
- uint64_t flags);
-*/
-void do_tsendmsg(int len)
-{
- int ret;
- ssize_t sz;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- struct fi_msg_tagged msg;
- struct iovec iov;
-
- iov.iov_base = source;
- iov.iov_len = len;
-
- msg.msg_iov = &iov;
- msg.desc = (void **)&loc_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[1];
- msg.context = target;
- msg.data = (uint64_t)target;
- msg.tag = len;
- msg.ignore = 0;
-
- rdm_tagged_sr_init_data(source, len, 0xef);
- rdm_tagged_sr_init_data(target, len, 0);
-
- sz = fi_tsendmsg(ep[0], &msg, 0);
- cr_assert_eq(sz, 0);
-
- sz = fi_trecv(ep[1], target, len, rem_mr, gni_addr[0], len, 0, source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_tagged_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_tagged_sr, tsendmsg)
-{
- rdm_tagged_sr_xfer_for_each_size(do_tsendmsg, 1, BUF_SZ);
-}
-
-/*
-ssize_t fi_tinject(struct fid_ep *ep, void *buf, size_t len,
- fi_addr_t dest_addr);
-*/
-#define INJECT_SIZE 64
-void do_tinject(int len)
-{
- int ret;
- ssize_t sz;
- struct fi_cq_tagged_entry cqe;
-
- rdm_tagged_sr_init_data(source, len, 0x23);
- rdm_tagged_sr_init_data(target, len, 0);
-
- sz = fi_tinject(ep[0], source, len, gni_addr[1], len);
- cr_assert_eq(sz, 0);
-
- sz = fi_trecv(ep[1], target, len, rem_mr, gni_addr[0], len, 0, source);
- cr_assert_eq(sz, 0);
-
- while ((ret = fi_cq_read(msg_cq[1], &cqe, 1)) == -FI_EAGAIN) {
- pthread_yield();
- }
-
- cr_assert_eq(ret, 1);
- cr_assert_eq((uint64_t)cqe.op_context, (uint64_t)source);
-
- dbg_printf("got recv context event!\n");
-
- cr_assert(rdm_tagged_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_tagged_sr, tinject)
-{
- rdm_tagged_sr_xfer_for_each_size(do_tinject, 1, INJECT_SIZE);
-}
-
-/*
-ssize_t fi_tsenddata(struct fid_ep *ep, void *buf, size_t len,
- void *desc, uint64_t data, fi_addr_t dest_addr, void *context);
-*/
-void do_tsenddata(int len)
-{
- int ret;
- ssize_t sz;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
-
- rdm_tagged_sr_init_data(source, len, 0xab);
- rdm_tagged_sr_init_data(target, len, 0);
-
- sz = fi_tsenddata(ep[0], source, len, loc_mr, (uint64_t)source,
- gni_addr[1], len, target);
- cr_assert_eq(sz, 0);
-
- sz = fi_trecv(ep[1], target, len, rem_mr, gni_addr[0], len, 0, source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_tagged_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_tagged_sr, tsenddata)
-{
- rdm_tagged_sr_xfer_for_each_size(do_tsenddata, 1, BUF_SZ);
-}
-
-/*
-ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc,
- size_t count, fi_addr_t src_addr, void *context);
- */
-void do_trecvv(int len)
-{
- int ret;
- ssize_t sz;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- struct iovec iov;
-
- rdm_tagged_sr_init_data(source, len, 0xab);
- rdm_tagged_sr_init_data(target, len, 0);
-
- sz = fi_tsend(ep[0], source, len, loc_mr, gni_addr[1], len, target);
- cr_assert_eq(sz, 0);
-
- iov.iov_base = target;
- iov.iov_len = len;
-
- sz = fi_trecvv(ep[1], &iov, (void **)&rem_mr, 1, gni_addr[0], len, 0,
- source);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_tagged_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_tagged_sr, trecvv)
-{
- rdm_tagged_sr_xfer_for_each_size(do_trecvv, 1, BUF_SZ);
-}
-
-/*
-ssize_t (*recvmsg)(struct fid_ep *ep, const struct fi_msg *msg,
- uint64_t flags);
- */
-void do_trecvmsg(int len)
-{
- int ret;
- ssize_t sz;
- int source_done = 0, dest_done = 0;
- struct fi_cq_tagged_entry s_cqe, d_cqe;
- struct fi_msg_tagged msg;
- struct iovec iov;
-
- rdm_tagged_sr_init_data(source, len, 0xab);
- rdm_tagged_sr_init_data(target, len, 0);
-
- sz = fi_tsend(ep[0], source, len, loc_mr, gni_addr[1], len, target);
- cr_assert_eq(sz, 0);
-
- iov.iov_base = target;
- iov.iov_len = len;
-
- msg.msg_iov = &iov;
- msg.desc = (void **)&rem_mr;
- msg.iov_count = 1;
- msg.addr = gni_addr[0];
- msg.context = source;
- msg.data = (uint64_t)source;
- msg.tag = len;
- msg.ignore = 0;
-
- sz = fi_trecvmsg(ep[1], &msg, 0);
- cr_assert_eq(sz, 0);
-
- /* need to progress both CQs simultaneously for rendezvous */
- do {
- ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
- if (ret == 1) {
- source_done = 1;
- }
- ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
- if (ret == 1) {
- dest_done = 1;
- }
- } while (!(source_done && dest_done));
-
- dbg_printf("got context events!\n");
-
- cr_assert(rdm_tagged_sr_check_data(source, target, len), "Data mismatch");
-}
-
-Test(rdm_tagged_sr, trecvmsg)
-{
- rdm_tagged_sr_xfer_for_each_size(do_trecvmsg, 1, BUF_SZ);
-}
-
-
-Test(rdm_tagged_sr, multi_tsend_trecv) {
- int i, it, ridx, ret;
- const int iters = 37;
- const int num_msgs = 17;
- const int slen = 256;
- uint64_t tags[num_msgs];
- uint64_t rtag = 0x01000000;
- uint64_t ignore = 0xf0ffffff;
- char msg[num_msgs][slen];
- struct fi_cq_tagged_entry cqe;
-
- srand(time(NULL));
-
- for (it = 0; it < iters; it++) {
- for (i = 0; i < num_msgs; i++) {
- tags[i] = 0x01010abc + it*iters + i;
-
- sprintf(msg[i], "%d\n", i);
- ret = fi_tsend(ep[1], msg[i], strlen(msg[i]),
- NULL, gni_addr[0], tags[i], NULL);
- cr_assert(ret == FI_SUCCESS);
-
- do {
- ret = fi_cq_read(msg_cq[1], &cqe, 1);
- cr_assert((ret == 1) || (ret == -FI_EAGAIN));
- } while (ret == -FI_EAGAIN);
-
- cr_assert(cqe.len == 0);
- cr_assert(cqe.tag == 0);
- }
-
- for (i = 0; i < num_msgs; i++) {
- memset(target, 0, BUF_SZ);
- ret = fi_trecv(ep[0], target, BUF_SZ,
- fi_mr_desc(loc_mr),
- gni_addr[1], rtag, ignore, NULL);
- cr_assert(ret == FI_SUCCESS);
-
- do {
- ret = fi_cq_read(msg_cq[0], &cqe, 1);
- cr_assert((ret == 1) || (ret == -FI_EAGAIN));
- } while (ret == -FI_EAGAIN);
-
- cr_assert(rtag != cqe.tag);
-
- ret = sscanf(target, "%d", &ridx);
- cr_assert(ret == 1);
- cr_assert(cqe.len == strlen(msg[ridx]));
-
- /* zero out the tag for error checking below */
- tags[ridx] = 0;
- }
-
- /* Make sure we got everything */
- for (i = 0; i < num_msgs; i++)
- cr_assert(tags[i] == 0);
- }
-
-}
-
-static void do_tagged_sr_pipelined(void)
-{
- int i, it, s, ret;
- const int iters = 37;
- const int num_msgs = 61;
- const int msgs_per_stage = 17;
- const int num_stages = num_msgs/msgs_per_stage +
- (num_msgs%msgs_per_stage != 0);
- const int slen = 256;
- uint64_t tags[num_msgs];
- uint64_t rtag = 0x01000000;
- uint64_t ignore = 0xf0ffffff;
- char msg[num_msgs][slen];
- struct fi_cq_tagged_entry cqe;
-
- srand(time(NULL));
-
- for (it = 0; it < iters; it++) {
- dbg_printf("iter %d\n", it);
- for (s = 0; s < num_stages; s++) {
- dbg_printf("\tsending stage %d\n", s);
- for (i = s*msgs_per_stage;
- i < (s+1)*msgs_per_stage && i < num_msgs;
- i++) {
- tags[i] = 0x01010abc + it*iters + i;
-
- sprintf(msg[i], "%d\n", i%10);
- ret = fi_tsend(ep[1], msg[i], strlen(msg[i]),
- NULL, gni_addr[0], tags[i],
- NULL);
- cr_assert(ret == FI_SUCCESS);
- }
-
- for (i = s*msgs_per_stage;
- i < (s+1)*msgs_per_stage && i < num_msgs;
- i++) {
- do {
- ret = fi_cq_read(msg_cq[1], &cqe, 1);
- cr_assert((ret == 1) ||
- (ret == -FI_EAGAIN));
- } while (ret == -FI_EAGAIN);
-
- cr_assert(cqe.tag == 0);
- }
- cr_assert(cqe.len == 0);
- }
-
- for (s = 0; s < num_stages; s++) {
- dbg_printf("\treceiving stage %d\n", s);
- for (i = s*msgs_per_stage;
- i < (s+1)*msgs_per_stage && i < num_msgs;
- i++) {
- ret = fi_trecv(ep[0], &target[i], BUF_SZ,
- fi_mr_desc(loc_mr),
- gni_addr[1], rtag, ignore,
- NULL);
- cr_assert(ret == FI_SUCCESS);
- }
-
- for (i = s*msgs_per_stage;
- i < (s+1)*msgs_per_stage && i < num_msgs;
- i++) {
- do {
- ret = fi_cq_read(msg_cq[0], &cqe, 1);
- cr_assert((ret == 1) ||
- (ret == -FI_EAGAIN));
- } while (ret == -FI_EAGAIN);
-
- cr_assert(rtag != cqe.tag);
-
- cr_assert(ret == 1);
- cr_assert(cqe.len == 2);
-
- /* zero out the tag for error checking below */
- tags[cqe.tag - (0x01010abc + it*iters)] = 0;
- }
- }
-
- /* Make sure we got everything */
- for (i = 0; i < num_msgs; i++) {
- cr_assert(tags[i] == 0);
- }
- }
-
-}
-
-/* Add this test when FI_PROGRESS_AUTO is implemented */
-Test(rdm_tagged_sr, multi_tsend_trecv_pipelined, .disabled = true) {
- do_tagged_sr_pipelined();
-}
-
-/* Call fi_gni_domain_ops->set_val() with op and opval */
-static void progress_manual_dom_ops_setup(const dom_ops_val_t op,
- const uint32_t opval)
-{
- int ret;
- uint32_t val = opval;
- struct fi_gni_ops_domain *gni_domain_ops;
-
- setup_dom(FI_PROGRESS_MANUAL);
- ret = fi_open_ops(&dom->fid, FI_GNI_DOMAIN_OPS_1,
- 0, (void **) &gni_domain_ops, NULL);
- gni_domain_ops->set_val(&dom->fid, op, &val);
- cr_assert(ret == FI_SUCCESS, "fi_open_ops");
-
- setup_ep();
- setup_mr();
-}
-
-static void mbox_max_credit_setup(void)
-{
- /* Use this with manual progress */
- progress_manual_dom_ops_setup(GNI_MBOX_MAX_CREDIT,
- 122-1 /* 2*num_msgs-1 above */);
-}
-
-/* Suite of tests that should work with manual progress */
-TestSuite(rdm_tagged_sr_progress_manual,
- .init = mbox_max_credit_setup,
- .fini = rdm_tagged_sr_teardown,
- .disabled = false);
-
-Test(rdm_tagged_sr_progress_manual, multi_tsend_trecv_pipelined) {
- do_tagged_sr_pipelined();
-}
-
diff --git a/prov/gni/test/run_gnitest b/prov/gni/test/run_gnitest
deleted file mode 100755
index 4bf701f..0000000
--- a/prov/gni/test/run_gnitest
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/bin/bash
-#
-# Copyright (c) 2015 Cray Inc. All rights reserved.
-#
-# This software is available to you under a choice of one of two
-# licenses. You may choose to be licensed under the terms of the GNU
-# General Public License (GPL) Version 2, available from the file
-# COPYING in the main directory of this source tree, or the
-# BSD license below:
-#
-# Redistribution and use in source and binary forms, with or
-# without modification, are permitted provided that the following
-# conditions are met:
-#
-# - Redistributions of source code must retain the above
-# copyright notice, this list of conditions and the following
-# disclaimer.
-#
-# - Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following
-# disclaimer in the documentation and/or other materials
-# provided with the distribution.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-#
-
-#
-# Check for srun or aprun
-#
-srun=`command -v srun`
-if [ $? == 0 ]; then
- launcher="srun"
-else
- aprun=`command -v aprun`
- if [ $? == 0 ]; then
- launcher="aprun"
- else
- echo "Cannot find a supported job launcher (srun, aprun). Please load the appropriate module"
- exit -1
- fi
-fi
-
-gnitest_bin="$(cd "$(dirname "$0")" && pwd)/gnitest"
-if [ ! -f "$gnitest_bin" ]; then
- echo "Could not find gnitest executable"
- exit -1
-fi
-
-if [ $launcher = "srun" ]; then
- args="-N1 --exclusive --cpu_bind=none -t00:20:00 --ntasks=1"
-else
- args="-n1 -N1 -j0 -cc none -t1200"
-fi
-
-# pass all command line args to gnitest
-$launcher $args $gnitest_bin -j1 "$@"
diff --git a/prov/gni/test/tags.c b/prov/gni/test/tags.c
deleted file mode 100644
index e803206..0000000
--- a/prov/gni/test/tags.c
+++ /dev/null
@@ -1,1876 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-
-/*
- * Created on: July 17, 2015
- * Author: jswaro
- */
-#include <stdlib.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <sys/time.h>
-
-#include <rdma/fi_errno.h>
-#include <gnix_tags.h>
-#include <gnix.h>
-
-#include <criterion/criterion.h>
-#include <criterion/parameterized.h>
-
-static struct fi_info *hints;
-static struct fi_info *fi;
-
-#define DEFAULT_FORMAT 0x00000000FFFFFFFF
-#define SINGLE_FORMAT 0xFFFFFFFFFFFFFFFF
-#define DOUBLE_FORMAT 0xFFFFFFFF00000000
-#define TRIPLE_FORMAT 0xFFFF0000FFFFFFFF
-#define MULTI_FORMAT 0xFF00FFFF0000FFFF
-#define BAD_FORMAT 0xFFFFFFFFFFFFFF00
-#define SIMPLE_FORMAT 0x0000000000FF00FF
-
-#define TEST_TAG 0x00000000DEADBEEF
-
-enum {
- TEST_ORDER_INORDER = 0,
- TEST_ORDER_RANDOM,
- TEST_ORDER_REVERSE,
-};
-
-enum {
- TEST_OVERLAY_DEF = 0,
- TEST_OVERLAY_SINGLE,
- TEST_OVERLAY_DOUBLE,
- TEST_OVERLAY_TRIPLE,
- TEST_OVERLAY_MULTI,
- TEST_OVERLAY_BAD,
- TEST_OVERLAY_SIMPLE,
- TEST_OVERLAY_MAX,
-};
-
-struct __test_mask {
- uint64_t mask;
- uint64_t format;
- int type;
- int fields;
- int field_width[5];
- int reserved_bits;
-};
-
-struct gnix_fr_element {
- struct gnix_fab_req req;
- int claimed;
- uint64_t ignore;
- uint64_t addr_ignore;
- uint64_t peek_flags;
- uint64_t remove_flags;
- void *context;
-};
-
-struct __test_mask test_masks[TEST_OVERLAY_MAX] = {
- {
- .mask = DEFAULT_FORMAT,
- .format = DEFAULT_FORMAT,
- .type = TEST_OVERLAY_DEF,
- .fields = 1,
- .field_width = {32, 0, 0, 0, 0},
- .reserved_bits = 32,
- },
- {
- .mask = ~0,
- .format = SINGLE_FORMAT,
- .type = TEST_OVERLAY_SINGLE,
- .fields = 1,
- .field_width = {64, 0, 0, 0, 0},
- .reserved_bits = 0,
- },
- {
- .mask = ~0,
- .format = DOUBLE_FORMAT,
- .type = TEST_OVERLAY_DOUBLE,
- .fields = 2,
- .field_width = {32, 32, 0, 0, 0},
- .reserved_bits = 0,
- },
- {
- .mask = ~0,
- .format = TRIPLE_FORMAT,
- .type = TEST_OVERLAY_TRIPLE,
- .fields = 3,
- .field_width = {16, 16, 32, 0, 0},
- .reserved_bits = 0,
- },
- {
- .mask = ~0,
- .format = MULTI_FORMAT,
- .type = TEST_OVERLAY_MULTI,
- .fields = 5,
- .field_width = {8, 8, 16, 16, 16},
- .reserved_bits = 0,
- },
- {
- .mask = ~0,
- .format = BAD_FORMAT,
- .type = TEST_OVERLAY_BAD,
- .fields = 5,
- .field_width = {16, 16, 16, 8, 8},
- .reserved_bits = 0,
- },
- {
- .mask = 0x0000000000ffffff,
- .format = SIMPLE_FORMAT,
- .type = TEST_OVERLAY_SIMPLE,
- .fields = 3,
- .field_width = {8, 8, 8, 0, 0},
- .reserved_bits = 40,
- }
-};
-
-static struct gnix_tag_storage_attr default_list_attr = {
- .type = GNIX_TAG_LIST,
-};
-
-static struct gnix_tag_storage_attr default_hlist_attr = {
- .type = GNIX_TAG_HLIST,
-};
-
-static struct gnix_tag_storage_attr default_kdtree_attr = {
- .type = GNIX_TAG_KDTREE,
-};
-
-static struct gnix_tag_storage_attr default_auto_attr = {
- .type = GNIX_TAG_AUTOSELECT,
-};
-
-static struct gnix_fr_element default_reqs[8] = {
- {
- .req = {
- .msg = {
- .tag = 0x00005555,
- .ignore = 0x11111111
- },
- },
- .peek_flags = FI_PEEK,
- .remove_flags = 0,
- .context = NULL,
-
- },
- {
- .req = {
- .msg = {
- .tag = 0x0000AAAA,
- .ignore = 0x11111111
- },
- },
- .peek_flags = FI_PEEK,
- .remove_flags = 0,
- .context = NULL,
- },
- {
- .req = {
- .msg = {
- .tag = 0xAAAA5555,
- .ignore = 0x11111111
- },
- },
- .peek_flags = FI_PEEK,
- .remove_flags = 0,
- .context = NULL,
- },
- {
- .req = {
- .msg = {
- .tag = 0x5555AAAA,
- .ignore = 0x11111111
- },
- },
- .peek_flags = FI_PEEK,
- .remove_flags = 0,
- .context = NULL,
- },
- {
- .req = {
- .msg = {
- .tag = 0xAAAA5555,
- .ignore = 0x11111111
- },
- },
- .peek_flags = FI_PEEK,
- .remove_flags = 0,
- .context = NULL,
- },
- {
- .req = {
- .msg = {
- .tag = 0x00005555,
- .ignore = 0x11111111
- },
- },
- .peek_flags = FI_PEEK,
- .remove_flags = 0,
- .context = NULL,
- },
- {
- .req = {
- .msg = {
- .tag = 0x00005555,
- .ignore = 0x11111111
- },
- },
- .peek_flags = FI_PEEK,
- .remove_flags = 0,
- .context = NULL,
- },
- {
- .req = {
- .msg = {
- .tag = 0x0000AAAA,
- .ignore = 0x11111111
- },
- },
- .peek_flags = FI_PEEK,
- .remove_flags = 0,
- .context = NULL,
- },
-};
-
-static struct gnix_fr_element *make_evenly_distributed_tags(
- int requests,
- struct __test_mask *mask);
-
-static struct gnix_fr_element *make_random_tags(
- int requests,
- struct __test_mask *mask);
-
-struct ipr_test_params {
- int elements;
- struct gnix_fr_element *(*make_requests)(
- int,
- struct __test_mask *);
-};
-
-static struct ipr_test_params ipr_params[6] = {
- {
- .elements = 16,
- .make_requests = make_random_tags,
- },
- {
- .elements = 128,
- .make_requests = make_random_tags,
- },
- {
- .elements = 1024,
- .make_requests = make_random_tags,
- },
- {
- .elements = 16,
- .make_requests = make_evenly_distributed_tags,
- },
- {
- .elements = 128,
- .make_requests = make_evenly_distributed_tags,
- },
- {
- .elements = 1024,
- .make_requests = make_evenly_distributed_tags,
- },
-};
-
-static struct gnix_tag_storage *test_tag_storage;
-static int call_destruct;
-static int (*match_func)(
- struct dlist_entry *entry,
- const void *arg) = _gnix_match_posted_tag;
-
-
-static inline void reset_test_fr_metadata(struct gnix_fr_element *reqs,
- int requests)
-{
- int i;
-
- for (i = 0; i < requests; i++) {
- reqs[i].claimed = 0;
- reqs[i].req.msg.tle.context = NULL;
- }
-}
-static inline void reset_test_tag_storage(
- struct gnix_tag_storage *ts,
- struct gnix_tag_storage_attr *attr)
-{
- int ret;
-
- ret = _gnix_tag_storage_destroy(ts);
- cr_assert(ret == FI_SUCCESS,
- "failed to destroy tag storage on reset");
-
- ret = _gnix_tag_storage_init(ts, attr, match_func);
- cr_assert(ret == FI_SUCCESS,
- "failed to initialize tag storage on reset");
-}
-
-static void __gnix_tags_bare_test_setup(void)
-{
- int ret = 0;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = 4;
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert_eq(ret, FI_SUCCESS, "fi_getinfo");
-
- cr_assert(test_tag_storage == NULL,
- "test_tag_storage was not freed prior to setup");
- test_tag_storage = calloc(1, sizeof(*test_tag_storage));
- cr_assert(test_tag_storage != NULL,
- "could not allocate test_tag_storage");
-
- call_destruct = 0;
-
- srand(0xDEADBEEF);
-}
-
-static void __gnix_tags_bare_test_teardown(void)
-{
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-
- cr_assert(test_tag_storage != NULL,
- "test_tag_storage pointer "
- "deallocated or overwritten during test");
- free(test_tag_storage);
- test_tag_storage = NULL;
-}
-
-static void __gnix_tags_basic_test_setup(void)
-{
- __gnix_tags_bare_test_setup();
-}
-
-static void __gnix_tags_basic_test_teardown(void)
-{
- int ret;
-
- ret = _gnix_tag_storage_destroy(test_tag_storage);
- cr_assert(ret == FI_SUCCESS,
- "failed to destroy tag storage "
- "during basic teardown");
-
- __gnix_tags_bare_test_teardown();
-}
-
-static void __gnix_tags_basic_list_test_setup(void)
-{
- int ret;
-
- __gnix_tags_basic_test_setup();
-
- ret = _gnix_tag_storage_init(test_tag_storage, &default_list_attr,
- match_func);
- cr_assert(ret == FI_SUCCESS, "failed to initialize tag storage "
- "during basic list setup");
-}
-
-static void __gnix_tags_basic_hlist_test_setup(void)
-{
- int ret;
-
- __gnix_tags_basic_test_setup();
-
- ret = _gnix_tag_storage_init(test_tag_storage, &default_hlist_attr,
- match_func);
- cr_assert(ret == FI_SUCCESS, "failed to initialize tag storage "
- "during basic hlist setup");
-}
-
-static void __gnix_tags_basic_kdtree_test_setup(void)
-{
- int ret;
-
- __gnix_tags_basic_test_setup();
-
- ret = _gnix_tag_storage_init(test_tag_storage, &default_kdtree_attr,
- match_func);
- cr_assert(ret == FI_SUCCESS, "failed to initialize tag storage "
- "during basic kdtree setup");
-}
-
-static void __gnix_tags_basic_posted_list_test_setup(void)
-{
- match_func = _gnix_match_posted_tag;
- __gnix_tags_basic_list_test_setup();
-}
-
-static void __gnix_tags_basic_posted_hlist_test_setup(void)
-{
- match_func = _gnix_match_posted_tag;
- __gnix_tags_basic_hlist_test_setup();
-}
-
-static void __gnix_tags_basic_posted_kdtree_test_setup(void)
-{
- match_func = _gnix_match_posted_tag;
- __gnix_tags_basic_kdtree_test_setup();
-}
-
-static void __gnix_tags_basic_unexpected_list_test_setup(void)
-{
- match_func = _gnix_match_unexpected_tag;
- __gnix_tags_basic_list_test_setup();
-}
-
-static void __gnix_tags_basic_unexpected_hlist_test_setup(void)
-{
- match_func = _gnix_match_unexpected_tag;
- __gnix_tags_basic_hlist_test_setup();
-}
-
-static void __gnix_tags_basic_unexpected_kdtree_test_setup(void)
-{
- match_func = _gnix_match_unexpected_tag;
- __gnix_tags_basic_kdtree_test_setup();
-}
-
-__attribute__((unused))
-static void __gnix_tags_basic_auto_test_setup(void)
-{
- int ret;
-
- __gnix_tags_basic_test_setup();
-
- ret = _gnix_tag_storage_init(test_tag_storage, &default_auto_attr,
- match_func);
- cr_assert(ret == FI_SUCCESS,
- "failed to initialize tag storage during basic auto setup");
-}
-
-/* multi-mode tests */
-
-static inline uint64_t make_test_tag(struct __test_mask *mask, uint64_t val)
-{
- return val & mask->mask;
-}
-
-
-static struct gnix_fr_element *make_evenly_distributed_tags(
- int requests,
- struct __test_mask *mask)
-{
- int i, j;
- uint64_t offset, field_width, tmp;
- struct gnix_fr_element *reqs;
-
- reqs = calloc(requests, sizeof(*reqs));
- cr_assert(reqs);
-
- offset = 64 - mask->reserved_bits;
-
- for (i = 0; i < mask->fields; i++) {
- field_width = 1ull << mask->field_width[i];
- if (mask->field_width[i] == 64)
- field_width = ~0ull;
-
- offset -= mask->field_width[i];
- for (j = 0; j < requests; j++) {
- tmp = (j % field_width) << offset;
- reqs[j].req.msg.tag |= tmp;
- }
- }
-
- for (i = 0; i < requests; i++) {
- tmp = make_test_tag(mask, reqs[i].req.msg.tag);
- reqs[i].req.msg.ignore = 0;
- reqs[i].req.msg.tag = tmp;
- reqs[i].peek_flags = FI_PEEK;
- }
-
- return reqs;
-}
-
-static struct gnix_fr_element *make_random_tags(
- int requests,
- struct __test_mask *mask)
-{
- int i;
- uint64_t tag;
- struct gnix_fr_element *reqs;
-
- reqs = calloc(requests, sizeof(*reqs));
- cr_assert(reqs, "failed to allocate requests for random tag creation");
-
- for (i = 0; i < requests; i++) {
- tag = rand();
- tag <<= 32;
- tag += rand();
- reqs[i].req.msg.tag = make_test_tag(mask, tag);
- reqs[i].req.msg.ignore = 0;
- reqs[i].peek_flags = FI_PEEK;
- }
-
- return reqs;
-}
-
-static void print_request_lists(
- struct gnix_fr_element *reqs,
- int requests,
- int *correct_order,
- int *removal_order,
- int ordering_type)
-{
- int i;
- char *type;
- struct gnix_fr_element *current;
-
- if (ordering_type == TEST_ORDER_INORDER)
- type = "INORDER";
- else if (ordering_type == TEST_ORDER_RANDOM)
- type = "RANDOM";
- else if (ordering_type == TEST_ORDER_REVERSE)
- type = "REVERSE";
- else
- type = "UNKNOWN";
-
-
- fprintf(stderr, "FAILED %s\n", type);
- fprintf(stderr, "insertion order:\n");
- for (i = 0; i < requests; i++) {
- current = &reqs[i];
- fprintf(stderr, " req=%p req.msg.tag=0x%.16llx index=%i\n",
- ¤t->req,
- (unsigned long long int) current->req.msg.tag,
- i);
- }
-
- fprintf(stderr, "removal order:\n");
- for (i = 0; i < requests; i++) {
- current = &reqs[removal_order[i]];
- fprintf(stderr,
- " req=%p req.msg.tag=0x%.16llx ignore=0x%.16llx index=%i\n",
- ¤t->req,
- (unsigned long long int) current->req.msg.tag,
- (unsigned long long int) current->ignore,
- removal_order[i]);
- }
-
- fprintf(stderr, "correct order:\n");
- for (i = 0; i < requests; i++) {
- current = &reqs[correct_order[i]];
- fprintf(stderr, " req=%p req.msg.tag=0x%.16llx index=%i\n",
- ¤t->req,
- (unsigned long long int) current->req.msg.tag,
- correct_order[i]);
- }
-}
-
-static void multiple_insert_peek_remove_by_order(
- struct gnix_tag_storage *ts,
- struct __test_mask *mask,
- int requests,
- struct gnix_fr_element *reqs,
- int *correct_order,
- int *removal_order,
- int ordering_type)
-{
- int ret;
- int i, j, ignore_bits;
- struct gnix_fr_element *to_remove, *current;
- struct gnix_fab_req *correct, *found;
- int is_posted = ts->match_func == _gnix_match_posted_tag;
-
- /* clear claimed flags */
- for (i = 0; i < requests; i++)
- reqs[i].claimed = 0;
-
- /* establish correct removal order based on passed in removal order */
- for (i = 0; i < requests; i++) {
- to_remove = &reqs[removal_order[i]];
-
- for (j = 0; j < requests; j++) {
- current = &reqs[j];
-
- if (current->claimed)
- continue;
-
- /* when the tag store is a posted tag store,
- * always use the ignore bits from the stored request
- *
- * otherwise, we are attempting to remove from
- * a unexpected tag store and we should use the
- * provided ignore bits
- */
- if (is_posted)
- ignore_bits = current->req.msg.ignore;
- else
- ignore_bits = to_remove->ignore;
-
- /* applying peek flags - This assumes the appropriate
- * peek was performed prior to removal
- */
- if ((to_remove->peek_flags & FI_PEEK) &&
- (to_remove->peek_flags & FI_CLAIM)) {
- to_remove->req.msg.tle.context =
- to_remove->context;
- }
-
- if (_gnix_req_matches_params(&to_remove->req,
- current->req.msg.tag,
- ignore_bits,
- to_remove->remove_flags,
- to_remove->context,
- test_tag_storage->attr.use_src_addr_matching,
- &to_remove->req.addr, is_posted))
- break;
-
- to_remove->req.msg.tle.context = NULL;
- }
-
- cr_assert(j != requests,
- "failed to find a match for every entry");
- correct_order[i] = j;
- current->claimed = 1;
- }
-
- /* clear alterations to the req structure
- * during creation of correct list
- */
- for (i = 0; i < requests; i++)
- reqs[i].req.msg.tle.context = NULL;
-
- /* clear claimed flags */
- for (i = 0; i < requests; i++)
- reqs[i].claimed = 0;
-
- for (i = 0; i < requests; i++) {
- ret = _gnix_insert_tag(ts, reqs[i].req.msg.tag, &reqs[i].req,
- reqs[i].ignore);
- if (ret) {
- print_request_lists(reqs, requests, correct_order,
- removal_order, ordering_type);
- }
- cr_assert(ret == FI_SUCCESS,
- "failed to insert tag into storage");
- }
-
- for (i = 0; i < requests; i++) {
- to_remove = &reqs[removal_order[i]];
- correct = &reqs[correct_order[i]].req;
-
- found = _gnix_match_tag(ts,
- to_remove->req.msg.tag, to_remove->ignore,
- to_remove->peek_flags | FI_PEEK, to_remove->context,
- NULL);
- if (found != correct) {
- print_request_lists(reqs, requests, correct_order,
- removal_order, ordering_type);
- fprintf(stderr,
- "failed to find request, to_remove=%p "
- "correct=%p found=%p",
- &to_remove->req,
- found,
- correct);
- }
- cr_assert(found == correct,
- "failed to find tag in storage");
-
-
- found = _gnix_match_tag(ts,
- to_remove->req.msg.tag, to_remove->ignore,
- to_remove->remove_flags, to_remove->context,
- NULL);
- if (found != correct) {
- print_request_lists(reqs, requests, correct_order,
- removal_order, ordering_type);
- }
- cr_assert(found == correct,
- "failed to remove tag from storage");
- }
-}
-
-static void multiple_insert_peek_remove_inorder(
- struct gnix_tag_storage *ts,
- struct __test_mask *mask,
- int requests,
- struct gnix_fr_element *reqs,
- int *correct_order,
- int *removal_order)
-{
- int i;
-
- for (i = 0; i < requests; i++)
- removal_order[i] = i;
-
- multiple_insert_peek_remove_by_order(
- ts, mask, requests, reqs,
- correct_order, removal_order,
- TEST_ORDER_INORDER);
-}
-
-static void multiple_insert_peek_remove_reverse(
- struct gnix_tag_storage *ts,
- struct __test_mask *mask,
- int requests,
- struct gnix_fr_element *reqs,
- int *correct_order,
- int *removal_order)
-{
- int i;
-
- for (i = requests - 1; i >= 0; i--)
- removal_order[requests - 1 - i] = i;
-
- multiple_insert_peek_remove_by_order(ts,
- mask, requests, reqs,
- correct_order, removal_order,
- TEST_ORDER_REVERSE);
-}
-
-static void multiple_insert_peek_remove_random(
- struct gnix_tag_storage *ts,
- struct __test_mask *mask,
- int requests,
- struct gnix_fr_element *reqs,
- int *correct_order,
- int *removal_order)
-{
- int i, j, rand_index, rand_j;
-
- for (i = 0; i < requests; i++) {
- rand_index = rand() % requests;
- for (j = 0; j < requests; j++) {
- rand_j = (rand_index + j) % requests;
- if (!reqs[rand_j].claimed)
- break;
- }
-
- removal_order[i] = rand_j;
- reqs[rand_j].claimed = 1;
- }
-
- /* clear claimed flags */
- for (i = 0; i < requests; i++)
- reqs[i].claimed = 0;
-
- multiple_insert_peek_remove_by_order(ts,
- mask, requests, reqs,
- correct_order, removal_order,
- TEST_ORDER_RANDOM);
-}
-
-static inline void __test_multiple_ipr_reqs_by_function(
- int requests,
- struct gnix_tag_storage_attr *test_attr,
- void (*order_func)(
- struct gnix_tag_storage*,
- struct __test_mask*,
- int,
- struct gnix_fr_element*,
- int*,
- int*),
- struct gnix_fr_element *(make_requests)(
- int,
- struct __test_mask *),
- int *correct_order,
- int *removal_order)
-{
- int i;
- struct gnix_fr_element *reqs;
- struct gnix_tag_storage_attr attr;
-
- memcpy(&attr, test_attr, sizeof(struct gnix_tag_storage_attr));
-
- for (i = 0; i < TEST_OVERLAY_MAX; i++) {
- reqs = make_requests(requests, &test_masks[i]);
- cr_assert(reqs != NULL, "failed to make random tags");
-
- order_func(test_tag_storage, &test_masks[i], requests, reqs,
- correct_order, removal_order);
-
- /* make necessary alterations to the structure
- * for the next test mask
- */
- reset_test_tag_storage(test_tag_storage, &attr);
-
- free(reqs);
- }
-}
-
-static inline void __test_multiple_inorder_ipr_reqs(
- int requests,
- struct gnix_tag_storage_attr *test_attr,
- struct gnix_fr_element *(make_requests)(
- int,
- struct __test_mask *))
-
-{
- int *correct_order, *removal_order;
-
- correct_order = calloc(requests, sizeof(*correct_order));
- cr_assert(correct_order != NULL);
-
- removal_order = calloc(requests, sizeof(*removal_order));
- cr_assert(removal_order != NULL);
-
- __test_multiple_ipr_reqs_by_function(requests, test_attr,
- multiple_insert_peek_remove_inorder,
- make_requests, correct_order, removal_order);
-
- free(correct_order);
- free(removal_order);
-}
-
-static inline void __test_multiple_reverse_ipr_reqs(
- int requests,
- struct gnix_tag_storage_attr *test_attr,
- struct gnix_fr_element *(make_requests)(
- int,
- struct __test_mask *))
-
-{
- int *correct_order, *removal_order;
-
- correct_order = calloc(requests, sizeof(*correct_order));
- cr_assert(correct_order != NULL);
-
- removal_order = calloc(requests, sizeof(*removal_order));
- cr_assert(removal_order != NULL);
-
- __test_multiple_ipr_reqs_by_function(requests, test_attr,
- multiple_insert_peek_remove_reverse,
- make_requests, correct_order, removal_order);
-
- free(correct_order);
- free(removal_order);
-}
-
-static inline void __test_multiple_random_ipr_reqs(
- int requests,
- struct gnix_tag_storage_attr *test_attr,
- struct gnix_fr_element *(make_requests)(
- int,
- struct __test_mask *))
-
-{
- int *correct_order, *removal_order;
-
- correct_order = calloc(requests, sizeof(*correct_order));
- cr_assert(correct_order != NULL);
-
- removal_order = calloc(requests, sizeof(*removal_order));
- cr_assert(removal_order != NULL);
-
- __test_multiple_ipr_reqs_by_function(requests, test_attr,
- multiple_insert_peek_remove_random,
- make_requests, correct_order, removal_order);
-
- free(correct_order);
- free(removal_order);
-}
-
-static inline void __test_multiple_type_ipr_reqs(
- int requests,
- struct gnix_tag_storage_attr *test_attr,
- struct gnix_fr_element *(make_requests)(
- int,
- struct __test_mask *))
-{
- __test_multiple_inorder_ipr_reqs(requests, test_attr, make_requests);
-
- reset_test_tag_storage(test_tag_storage, test_attr);
-
- __test_multiple_reverse_ipr_reqs(requests, test_attr, make_requests);
-
- reset_test_tag_storage(test_tag_storage, test_attr);
-
- __test_multiple_random_ipr_reqs(requests, test_attr, make_requests);
-
- reset_test_tag_storage(test_tag_storage, test_attr);
-}
-
-static inline void __test_multiple_8_duplicate_tags(
- struct gnix_tag_storage_attr *test_attr)
-{
- int i;
- struct gnix_fr_element reqs[8];
- int requests = 8;
- int *correct_order, *removal_order;
-
- correct_order = calloc(8, sizeof(*correct_order));
- cr_assert(correct_order != NULL);
-
- removal_order = calloc(8, sizeof(*removal_order));
- cr_assert(removal_order != NULL);
-
- memcpy(reqs, default_reqs, sizeof(struct gnix_fr_element) * requests);
-
- for (i = 0; i < TEST_OVERLAY_MAX; i++) {
- multiple_insert_peek_remove_inorder(
- test_tag_storage, &test_masks[i],
- requests, reqs, correct_order,
- removal_order);
-
- reset_test_tag_storage(test_tag_storage, test_attr);
- reset_test_fr_metadata(reqs, requests);
-
- multiple_insert_peek_remove_reverse(
- test_tag_storage, &test_masks[i],
- requests, reqs, correct_order, removal_order);
-
- reset_test_tag_storage(test_tag_storage, test_attr);
- reset_test_fr_metadata(reqs, requests);
-
- multiple_insert_peek_remove_random(
- test_tag_storage, &test_masks[i],
- requests, reqs, correct_order, removal_order);
-
- reset_test_tag_storage(test_tag_storage, test_attr);
- reset_test_fr_metadata(reqs, requests);
- }
-
- free(correct_order);
- free(removal_order);
-}
-
-static inline void __test_not_found_empty(void)
-{
- struct gnix_fab_req *found;
-
- found = _gnix_match_tag(
- test_tag_storage, 0xCAFEBABE, 0, FI_PEEK, NULL, NULL);
- cr_assert(found == NULL,
- "something in this storage should not exist");
-}
-
-static inline void __test_not_found_non_empty(void)
-{
- int ret;
- int i, requests = 8;
- struct gnix_fab_req *correct, *found;
- struct gnix_fr_element *to_remove;
- struct gnix_fr_element reqs[8];
-
- memcpy(reqs, default_reqs, sizeof(struct gnix_fr_element) * 8);
-
- for (i = 0; i < requests; i++) {
- ret = _gnix_insert_tag(test_tag_storage,
- reqs[i].req.msg.tag, &reqs[i].req,
- reqs[i].ignore);
- cr_assert(ret == FI_SUCCESS,
- "failed to insert tag into storage");
- }
-
- found = _gnix_match_tag(test_tag_storage, 0xCAFEBABE, 0, FI_PEEK,
- NULL, NULL);
- cr_assert(found == NULL,
- "something in this storage should not exist");
-
- /* only doing in-order removal */
- for (i = 0; i < requests; i++) {
- to_remove = &reqs[i];
- correct = &reqs[i].req;
-
- found = _gnix_match_tag(
- test_tag_storage, to_remove->req.msg.tag,
- to_remove->ignore, to_remove->peek_flags,
- to_remove->context, NULL);
- cr_assert(found == correct,
- "failed to find tag in storage");
-
- found = _gnix_match_tag(test_tag_storage,
- to_remove->req.msg.tag, to_remove->ignore,
- to_remove->remove_flags, to_remove->context,
- NULL);
- cr_assert(found == correct,
- "failed to remove tag from storage");
- }
-}
-
-static inline void __test_ignore_mask_set(
- struct gnix_tag_storage_attr *test_attr)
-{
- int i;
- struct gnix_fr_element reqs[8];
- int requests = 8;
- int correct_order[8], removal_order[8];
-
- memcpy(reqs, default_reqs, sizeof(struct gnix_fr_element) * requests);
-
- for (i = requests - 1; i >= 0 ; i--)
- removal_order[i] = (requests - 1) - i;
-
- reqs[7].ignore = 0xffff;
- reqs[6].ignore = 0xffff0000;
- reqs[5].ignore = 0xffff0000;
- reqs[4].ignore = 0xffff0000;
- reqs[3].ignore = 0xffff0000;
- reqs[2].ignore = 0xffff0000;
- reqs[1].ignore = 0x0;
- reqs[0].ignore = 0xffffffff;
-
- for (i = 0; i < requests; i++)
- reqs[i].req.msg.ignore = reqs[i].ignore;
-
- multiple_insert_peek_remove_by_order(test_tag_storage,
- &test_masks[TEST_OVERLAY_DEF], requests, reqs,
- correct_order, removal_order, TEST_ORDER_REVERSE);
-}
-
-static inline void __test_claim_pass(
- struct gnix_tag_storage_attr *test_attr)
-{
- int ret;
- struct gnix_fr_element request = {
- .req = {
- .msg = {
- .tag = 0xA5A5A5A5,
- .ignore = 0xFFFFFFFF
- },
- },
- .peek_flags = FI_PEEK | FI_CLAIM,
- .remove_flags = FI_CLAIM,
- .context = (void *) 0xDEADBEEF,
- };
- struct gnix_fab_req *found;
-
- ret = _gnix_insert_tag(
- test_tag_storage, request.req.msg.tag,
- &request.req, request.ignore);
- cr_assert(ret == FI_SUCCESS, "failed to insert tag into storage");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag, request.ignore,
- request.peek_flags, request.context, NULL);
- cr_assert(found == &request.req, "failed to find tag in storage");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag, request.ignore,
- request.remove_flags, request.context, NULL);
- cr_assert(found == &request.req, "failed to remove tag from storage");
-}
-
-/* tests to ensure you cannot remove an unclaimed messge when FI_CLAIM is set */
-static inline void __test_fail_no_claimed_tags(
- struct gnix_tag_storage_attr *test_attr)
-{
- int ret;
- struct gnix_fab_req *found;
- struct gnix_fr_element request = {
- .req = {
- .msg = {
- .tag = 0xA5A5A5A5,
- .ignore = 0xFFFFFFFF
- },
- },
- .peek_flags = FI_PEEK,
- .remove_flags = FI_CLAIM,
- .context = (void *) 0xDEADBEEF,
- };
-
- ret = _gnix_insert_tag(
- test_tag_storage, request.req.msg.tag,
- &request.req, request.ignore);
- cr_assert(ret == FI_SUCCESS, "failed to insert tag into storage");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag, request.ignore,
- request.peek_flags, request.context, NULL);
- cr_assert(found == &request.req, "failed to find tag in storage");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag, request.ignore,
- request.remove_flags, request.context, NULL);
- cr_assert(found == NULL, "found an unexpected tag in remove");
-
- /* use the peek tags this time to remove the entry */
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag, request.ignore,
- 0, request.context, NULL);
- cr_assert(found == &request.req, "failed to find tag in storage");
-}
-
-/* test to ensure you cannot remove a message that has been claimed */
-static inline void __test_fail_all_claimed_tags(
- struct gnix_tag_storage_attr *test_attr)
-{
- int ret;
- struct gnix_fr_element request = {
- .req = {
- .msg = {
- .tag = 0xA5A5A5A5,
- .ignore = 0xFFFFFFFF
- },
- },
- .peek_flags = FI_PEEK | FI_CLAIM,
- .remove_flags = 0,
- .context = (void *) 0xDEADBEEF,
- };
- struct gnix_fab_req *found;
-
- ret = _gnix_insert_tag(
- test_tag_storage, request.req.msg.tag,
- &request.req, request.ignore);
- cr_assert(ret == FI_SUCCESS, "failed to insert tag into storage");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag, request.ignore,
- request.peek_flags, request.context, NULL);
- cr_assert(found == &request.req, "failed to find tag in storage");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag, request.ignore,
- request.remove_flags, request.context, NULL);
- cr_assert(found == NULL, "found an unexpected tag during remove");
-
- /* use the peek tags this time to remove the entry */
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag, request.ignore,
- FI_CLAIM, request.context, NULL);
- cr_assert(found == &request.req, "failed to find tag in storage");
-}
-
-static inline void __test_fail_peek_all_claimed(
- struct gnix_tag_storage_attr *test_attr)
-{
- int ret;
- struct gnix_fr_element request = {
- .req = {
- .msg = {
- .tag = 0xA5A5A5A5,
- .ignore = 0xFFFFFFFF
- },
- },
- .peek_flags = FI_PEEK,
- .remove_flags = FI_CLAIM,
- .context = (void *) 0xDEADBEEF,
- };
- struct gnix_fab_req *found;
-
- ret = _gnix_insert_tag(
- test_tag_storage, request.req.msg.tag,
- &request.req, request.ignore);
- cr_assert(ret == FI_SUCCESS, "failed to insert tag into storage");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag, request.ignore,
- request.peek_flags | FI_CLAIM, request.context, NULL);
- cr_assert(found == &request.req, "fail to claim tag");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag, request.ignore,
- request.peek_flags, request.context, NULL);
- cr_assert(found == NULL, "unexpectedly found a tag");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag, request.ignore,
- request.remove_flags, request.context, NULL);
- cr_assert(found == &request.req, "failed to remove tag from storage");
-}
-
-static inline void __test_src_addr_match(
- struct gnix_tag_storage_attr *test_attr)
-{
- int ret;
- struct gnix_fr_element request = {
- .req = {
- .msg = {
- .tag = 0xA5A5A5A5,
- },
- .addr = {
- .cdm_id = 1,
- .device_addr = 1,
- },
- },
- .peek_flags = FI_PEEK,
- };
- struct gnix_fab_req *found;
-
- ret = _gnix_insert_tag(
- test_tag_storage, request.req.msg.tag,
- &request.req, request.ignore);
- cr_assert(ret == FI_SUCCESS, "failed to insert tag into storage");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag,
- request.ignore, request.peek_flags,
- request.context, &request.req.addr);
- cr_assert(found == &request.req, "failed to find tag in storage");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag,
- request.ignore, request.remove_flags,
- request.context, &request.req.addr);
- cr_assert(found == &request.req, "failed to remove tag from storage");
-}
-
-/* tests to ensure you cannot remove an unclaimed messge when FI_CLAIM is set */
-static inline void __test_src_addr_fail_wrong_src_addr(
- struct gnix_tag_storage_attr *test_attr)
-{
- int ret;
- struct gnix_fab_req *found;
- struct gnix_fr_element request = {
- .req = {
- .msg = {
- .tag = 0xA5A5A5A5,
- },
- .addr = {
- .cdm_id = 1,
- .device_addr = 1,
- },
- },
- .ignore = 0,
- .addr_ignore = 0,
- .peek_flags = FI_PEEK,
- };
- struct gnix_address addr_to_find = {
- .cdm_id = 1,
- .device_addr = 2,
- };
-
- /* hack, don't actually do this */
- test_tag_storage->attr.use_src_addr_matching = 1;
-
- ret = _gnix_insert_tag(
- test_tag_storage, request.req.msg.tag,
- &request.req, request.ignore);
- cr_assert(ret == FI_SUCCESS, "failed to insert tag into storage");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag,
- request.ignore, request.peek_flags,
- request.context, &addr_to_find);
- cr_assert(found == NULL, "found unexpected tag");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag,
- request.ignore, request.peek_flags,
- request.context, &request.req.addr);
- cr_assert(found == &request.req, "failed to find tag in storage");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag,
- request.ignore, request.remove_flags,
- request.context, &request.req.addr);
- cr_assert(found == &request.req, "failed to find tag in storage");
-}
-
-/* test to ensure you cannot remove a message that has been claimed */
-static inline void __test_src_addr_match_unspec(
- struct gnix_tag_storage_attr *test_attr)
-{
- int ret;
- struct gnix_fr_element request = {
- .req = {
- .msg = {
- .tag = 0xA5A5A5A5,
- },
- .addr = {
- .cdm_id = 1,
- .device_addr = 1,
- },
- },
- .peek_flags = FI_PEEK,
- .ignore = 0,
- .addr_ignore = 0,
- };
- struct gnix_fab_req *found;
- struct gnix_address to_find = {
- .cdm_id = -1,
- .device_addr = -1,
- };
-
-
- if (test_tag_storage->match_func == _gnix_match_posted_tag) {
- /* swap addresses because posted tag receives check
- * the address in the request for unspec rather than
- * looking at the address in the parameters
- */
- request.req.addr.cdm_id = -1;
- request.req.addr.device_addr = -1;
-
- to_find.cdm_id = 1;
- to_find.device_addr = 1;
- }
-
-
-
- /* hack, don't actually do this */
- test_tag_storage->attr.use_src_addr_matching = 1;
-
- ret = _gnix_insert_tag(
- test_tag_storage, request.req.msg.tag,
- &request.req, request.ignore);
- cr_assert(ret == FI_SUCCESS, "failed to insert tag into storage");
-
-
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag,
- request.ignore, request.peek_flags,
- request.context, &to_find);
- cr_assert(found == &request.req, "failed to find tag in storage");
-
- found = _gnix_match_tag(
- test_tag_storage, request.req.msg.tag,
- request.ignore, request.remove_flags,
- request.context, &to_find);
- cr_assert(found == &request.req, "failed to find tag in storage");
-}
-
-static void single_insert_peek_remove(
- struct gnix_tag_storage *ts,
- struct __test_mask *mask,
- struct gnix_fr_element *reqs)
-{
- int correct_order, removal_order;
-
- multiple_insert_peek_remove_inorder(ts, mask, 1, reqs, &correct_order,
- &removal_order);
-}
-
-/*
- * Basic functionality tests for the gnix_bitmap_t object
- */
-
-TestSuite(gnix_tags_bare,
- .init = __gnix_tags_bare_test_setup,
- .fini = __gnix_tags_bare_test_teardown);
-
-TestSuite(gnix_tags_basic_posted_list,
- .init = __gnix_tags_basic_posted_list_test_setup,
- .fini = __gnix_tags_basic_test_teardown);
-
-TestSuite(gnix_tags_basic_posted_hlist,
- .init = __gnix_tags_basic_posted_hlist_test_setup,
- .fini = __gnix_tags_basic_test_teardown,
- .disabled = true);
-
-TestSuite(gnix_tags_basic_posted_kdtree,
- .init = __gnix_tags_basic_posted_kdtree_test_setup,
- .fini = __gnix_tags_basic_test_teardown,
- .disabled = true);
-
-TestSuite(gnix_tags_basic_unexpected_list,
- .init = __gnix_tags_basic_unexpected_list_test_setup,
- .fini = __gnix_tags_basic_test_teardown);
-
-TestSuite(gnix_tags_basic_unexpected_hlist,
- .init = __gnix_tags_basic_unexpected_hlist_test_setup,
- .fini = __gnix_tags_basic_test_teardown,
- .disabled = true);
-
-TestSuite(gnix_tags_basic_unexpected_kdtree,
- .init = __gnix_tags_basic_unexpected_kdtree_test_setup,
- .fini = __gnix_tags_basic_test_teardown,
- .disabled = true);
-
-Test(gnix_tags_bare, uninitialized)
-{
- cr_assert(test_tag_storage->gen == 0);
- cr_assert(test_tag_storage->attr.type == GNIX_TAG_AUTOSELECT);
- cr_assert(test_tag_storage->state == GNIX_TS_STATE_UNINITIALIZED);
-}
-
-Test(gnix_tags_bare, initialize_list)
-{
- int ret;
-
- ret = _gnix_tag_storage_init(test_tag_storage, &default_list_attr,
- match_func);
- cr_assert(ret == FI_SUCCESS);
-
- ret = _gnix_tag_storage_destroy(test_tag_storage);
- cr_assert(ret == FI_SUCCESS);
-}
-
-
-Test(gnix_tags_bare, initialize_hlist)
-{
- int ret;
-
- ret = _gnix_tag_storage_init(test_tag_storage, &default_hlist_attr,
- match_func);
- cr_assert(ret == FI_SUCCESS);
-
- ret = _gnix_tag_storage_destroy(test_tag_storage);
- cr_assert(ret == FI_SUCCESS);
-}
-
-Test(gnix_tags_bare, initialize_kdtree)
-{
- int ret;
-
- ret = _gnix_tag_storage_init(test_tag_storage, &default_kdtree_attr,
- match_func);
- cr_assert(ret == -FI_ENOSYS);
-
- //ret = _gnix_tag_storage_destroy(test_tag_storage);
- //cr_assert(ret == FI_SUCCESS);
-}
-
-
-Test(gnix_tags_bare, initialize_auto)
-{
- int ret;
-
- ret = _gnix_tag_storage_init(test_tag_storage, &default_auto_attr,
- match_func);
- cr_assert(ret == FI_SUCCESS);
-
- ret = _gnix_tag_storage_destroy(test_tag_storage);
- cr_assert(ret == FI_SUCCESS);
-}
-
-Test(gnix_tags_bare, already_initialized)
-{
- int ret;
-
- ret = _gnix_tag_storage_init(test_tag_storage, &default_list_attr,
- match_func);
- cr_assert(ret == FI_SUCCESS);
-
- ret = _gnix_tag_storage_init(test_tag_storage, &default_list_attr,
- match_func);
- cr_assert(ret == -FI_EINVAL);
-
- ret = _gnix_tag_storage_destroy(test_tag_storage);
- cr_assert(ret == FI_SUCCESS);
-}
-
-Test(gnix_tags_basic_posted_list, single_insert_remove)
-{
- int i;
- struct gnix_fr_element request = {
- .req = {
- .msg = {
- .tag = 0xA5A5A5A5,
- .ignore = 0xFFFFFFFF
- },
- },
- .peek_flags = FI_PEEK,
- };
- struct gnix_tag_storage_attr attr;
-
- memcpy(&attr, &default_list_attr, sizeof(struct gnix_tag_storage_attr));
-
- for (i = 0; i < TEST_OVERLAY_MAX; i++) {
-
- single_insert_peek_remove(test_tag_storage,
- &test_masks[i], &request);
-
- /* make necessary alterations to the structure
- * for the next test mask
- */
- reset_test_tag_storage(test_tag_storage, &attr);
- }
-}
-
-ParameterizedTestParameters(gnix_tags_basic_posted_list, multiple_ipr_tags)
-{
- size_t nb_params = sizeof (ipr_params) / sizeof (struct ipr_test_params);
- return cr_make_param_array(struct ipr_test_params, ipr_params, nb_params);
-}
-
-ParameterizedTest(struct ipr_test_params *params,
- gnix_tags_basic_posted_list, multiple_ipr_tags)
-{
- __test_multiple_type_ipr_reqs(params->elements, &default_list_attr,
- params->make_requests);
-}
-
-
-Test(gnix_tags_basic_posted_list, multiple_8_duplicate_tags)
-{
- __test_multiple_8_duplicate_tags(&default_list_attr);
-}
-
-Test(gnix_tags_basic_posted_list, not_found_non_empty)
-{
- __test_not_found_non_empty();
-}
-
-Test(gnix_tags_basic_posted_list, not_found_empty)
-{
- __test_not_found_empty();
-}
-
-Test(gnix_tags_basic_posted_list, ignore_mask_set)
-{
- __test_ignore_mask_set(&default_list_attr);
-}
-
-Test(gnix_tags_basic_posted_list, fi_claim_pass)
-{
- __test_claim_pass(&default_list_attr);
-}
-
-Test(gnix_tags_basic_posted_list, fi_claim_fail_no_claimed_tags)
-{
- __test_fail_no_claimed_tags(&default_list_attr);
-}
-
-Test(gnix_tags_basic_posted_list, fi_claim_fail_all_claimed_tags)
-{
- __test_fail_all_claimed_tags(&default_list_attr);
-}
-
-Test(gnix_tags_basic_posted_list, fi_claim_fail_peek_all_claimed)
-{
- __test_fail_peek_all_claimed(&default_list_attr);
-}
-
-/* unexpected list src address matching tests */
-Test(gnix_tags_basic_posted_list, src_addr_match_success)
-{
- __test_src_addr_match(&default_list_attr);
-}
-
-Test(gnix_tags_basic_posted_list, src_addr_no_match_wrong_addr)
-{
- __test_src_addr_fail_wrong_src_addr(&default_list_attr);
-}
-
-Test(gnix_tags_basic_posted_list, src_addr_match_unspec)
-{
- __test_src_addr_match_unspec(&default_list_attr);
-}
-
-
-/*
- * unexpected list tests
- */
-
-Test(gnix_tags_basic_unexpected_list, single_insert_remove)
-{
- int i;
- struct gnix_fr_element request = {
- .req = {
- .msg = {
- .tag = 0xA5A5A5A5,
- .ignore = 0xFFFFFFFF
- },
- },
- .peek_flags = FI_PEEK,
- };
- struct gnix_tag_storage_attr attr;
-
- memcpy(&attr, &default_list_attr, sizeof(struct gnix_tag_storage_attr));
-
- for (i = 0; i < TEST_OVERLAY_MAX; i++) {
-
- single_insert_peek_remove(test_tag_storage,
- &test_masks[i], &request);
-
- /* make necessary alterations to the structure
- * for the next test mask
- */
- reset_test_tag_storage(test_tag_storage, &attr);
- }
-}
-
-ParameterizedTestParameters(gnix_tags_basic_unexpected_list, multiple_ipr_tags)
-{
- size_t nb_params = sizeof (ipr_params) / sizeof (struct ipr_test_params);
- return cr_make_param_array(struct ipr_test_params, ipr_params, nb_params);
-}
-
-ParameterizedTest(struct ipr_test_params *params,
- gnix_tags_basic_unexpected_list, multiple_ipr_tags)
-{
- __test_multiple_type_ipr_reqs(params->elements, &default_list_attr,
- params->make_requests);
-}
-
-Test(gnix_tags_basic_unexpected_list, multiple_8_duplicate_tags)
-{
- __test_multiple_8_duplicate_tags(&default_list_attr);
-}
-
-Test(gnix_tags_basic_unexpected_list, not_found_non_empty)
-{
- __test_not_found_non_empty();
-}
-
-Test(gnix_tags_basic_unexpected_list, not_found_empty)
-{
- __test_not_found_empty();
-}
-
-Test(gnix_tags_basic_unexpected_list, ignore_mask_set)
-{
- __test_ignore_mask_set(&default_list_attr);
-}
-
-Test(gnix_tags_basic_unexpected_list, fi_claim_pass)
-{
- __test_claim_pass(&default_list_attr);
-}
-
-Test(gnix_tags_basic_unexpected_list, fi_claim_fail_no_claimed_tags)
-{
- __test_fail_no_claimed_tags(&default_list_attr);
-}
-
-Test(gnix_tags_basic_unexpected_list, fi_claim_fail_all_claimed_tags)
-{
- __test_fail_all_claimed_tags(&default_list_attr);
-}
-
-Test(gnix_tags_basic_unexpected_list, fi_claim_fail_peek_all_claimed)
-{
- __test_fail_peek_all_claimed(&default_list_attr);
-}
-
-/* unexpected list src address matching tests */
-Test(gnix_tags_basic_unexpected_list, src_addr_match_success)
-{
- __test_src_addr_match(&default_list_attr);
-}
-
-Test(gnix_tags_basic_unexpected_list, src_addr_no_match_wrong_addr)
-{
- __test_src_addr_fail_wrong_src_addr(&default_list_attr);
-}
-
-Test(gnix_tags_basic_unexpected_list, src_addr_match_unspec)
-{
- __test_src_addr_match_unspec(&default_list_attr);
-}
-
-/*
- * hlist tests
- */
-
-Test(gnix_tags_basic_posted_hlist, single_insert_remove)
-{
- int i;
- struct gnix_fr_element request = {
- .req = {
- .msg = {
- .tag = 0xA5A5A5A5,
- .ignore = 0xFFFFFFFF
- },
- },
- .peek_flags = FI_PEEK,
- };
- struct gnix_tag_storage_attr attr;
-
- memcpy(&attr, &default_hlist_attr, sizeof(struct gnix_tag_storage_attr));
-
- for (i = 0; i < TEST_OVERLAY_MAX; i++) {
-
- single_insert_peek_remove(test_tag_storage,
- &test_masks[i], &request);
-
- /* make necessary alterations to the structure
- * for the next test mask
- */
- reset_test_tag_storage(test_tag_storage, &attr);
- }
-}
-
-ParameterizedTestParameters(gnix_tags_basic_posted_hlist, multiple_ipr_tags)
-{
- size_t nb_params = sizeof (ipr_params) / sizeof (struct ipr_test_params);
- return cr_make_param_array(struct ipr_test_params, ipr_params, nb_params);
-}
-
-ParameterizedTest(struct ipr_test_params *params,
- gnix_tags_basic_posted_hlist, multiple_ipr_tags)
-{
- __test_multiple_type_ipr_reqs(params->elements, &default_hlist_attr,
- params->make_requests);
-}
-
-
-Test(gnix_tags_basic_posted_hlist, multiple_8_duplicate_tags)
-{
- __test_multiple_8_duplicate_tags(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_posted_hlist, not_found_non_empty)
-{
- __test_not_found_non_empty();
-}
-
-Test(gnix_tags_basic_posted_hlist, not_found_empty)
-{
- __test_not_found_empty();
-}
-
-Test(gnix_tags_basic_posted_hlist, ignore_mask_set)
-{
- __test_ignore_mask_set(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_posted_hlist, fi_claim_pass)
-{
- __test_claim_pass(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_posted_hlist, fi_claim_fail_no_claimed_tags)
-{
- __test_fail_no_claimed_tags(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_posted_hlist, fi_claim_fail_all_claimed_tags)
-{
- __test_fail_all_claimed_tags(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_posted_hlist, fi_claim_fail_peek_all_claimed)
-{
- __test_fail_peek_all_claimed(&default_hlist_attr);
-}
-
-/* unexpected list src address matching tests */
-Test(gnix_tags_basic_posted_hlist, src_addr_match_success)
-{
- __test_src_addr_match(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_posted_hlist, src_addr_no_match_wrong_addr)
-{
- __test_src_addr_fail_wrong_src_addr(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_posted_hlist, src_addr_match_unspec)
-{
- __test_src_addr_match_unspec(&default_hlist_attr);
-}
-
-
-/*
- * unexpected hlist tests
- */
-
-Test(gnix_tags_basic_unexpected_hlist, single_insert_remove)
-{
- int i;
- struct gnix_fr_element request = {
- .req = {
- .msg = {
- .tag = 0xA5A5A5A5,
- .ignore = 0xFFFFFFFF
- },
- },
- .peek_flags = FI_PEEK,
- };
- struct gnix_tag_storage_attr attr;
-
- memcpy(&attr, &default_hlist_attr, sizeof(struct gnix_tag_storage_attr));
-
- for (i = 0; i < TEST_OVERLAY_MAX; i++) {
-
- single_insert_peek_remove(test_tag_storage,
- &test_masks[i], &request);
-
- /* make necessary alterations to the structure
- * for the next test mask
- */
- reset_test_tag_storage(test_tag_storage, &attr);
- }
-}
-
-ParameterizedTestParameters(gnix_tags_basic_unexpected_hlist, multiple_ipr_tags)
-{
- size_t nb_params = sizeof (ipr_params) / sizeof (struct ipr_test_params);
- return cr_make_param_array(struct ipr_test_params, ipr_params, nb_params);
-}
-
-ParameterizedTest(struct ipr_test_params *params,
- gnix_tags_basic_unexpected_hlist, multiple_ipr_tags)
-{
- __test_multiple_type_ipr_reqs(params->elements, &default_hlist_attr,
- params->make_requests);
-}
-
-Test(gnix_tags_basic_unexpected_hlist, multiple_8_duplicate_tags)
-{
- __test_multiple_8_duplicate_tags(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_unexpected_hlist, not_found_non_empty)
-{
- __test_not_found_non_empty();
-}
-
-Test(gnix_tags_basic_unexpected_hlist, not_found_empty)
-{
- __test_not_found_empty();
-}
-
-Test(gnix_tags_basic_unexpected_hlist, ignore_mask_set)
-{
- __test_ignore_mask_set(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_unexpected_hlist, fi_claim_pass)
-{
- __test_claim_pass(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_unexpected_hlist, fi_claim_fail_no_claimed_tags)
-{
- __test_fail_no_claimed_tags(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_unexpected_hlist, fi_claim_fail_all_claimed_tags)
-{
- __test_fail_all_claimed_tags(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_unexpected_hlist, fi_claim_fail_peek_all_claimed)
-{
- __test_fail_peek_all_claimed(&default_hlist_attr);
-}
-
-/* unexpected list src address matching tests */
-Test(gnix_tags_basic_unexpected_hlist, src_addr_match_success)
-{
- __test_src_addr_match(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_unexpected_hlist, src_addr_no_match_wrong_addr)
-{
- __test_src_addr_fail_wrong_src_addr(&default_hlist_attr);
-}
-
-Test(gnix_tags_basic_unexpected_hlist, src_addr_match_unspec)
-{
- __test_src_addr_match_unspec(&default_hlist_attr);
-}
-
diff --git a/prov/gni/test/utils.c b/prov/gni/test/utils.c
deleted file mode 100644
index f792f93..0000000
--- a/prov/gni/test/utils.c
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Copyright (c) 2015 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <linux/limits.h>
-#include <sys/syscall.h>
-#include <fcntl.h>
-#include <unistd.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include <criterion/criterion.h>
-#include "gnix_util.h"
-#include "gnix.h"
-
-struct gnix_reference_tester {
- struct gnix_reference ref_cnt;
- int destructed;
-};
-
-Test(utils, proc)
-{
- int rc;
-
- rc = _gnix_task_is_not_app();
- cr_expect(rc == 0);
-
- /* *_unassigned_cpus flags don't work on tiger */
- rc = _gnix_job_enable_unassigned_cpus();
- cr_expect(rc != 0);
-
- rc = _gnix_job_disable_unassigned_cpus();
- cr_expect(rc != 0);
-
- rc = _gnix_job_enable_affinity_apply();
- cr_expect(rc == 0);
-
- rc = _gnix_job_disable_affinity_apply();
- cr_expect(rc == 0);
-
-}
-
-Test(utils, alps)
-{
- int rc;
- uint8_t ptag;
- uint32_t cookie, fmas, cqs, npes, npr;
- void *addr = NULL;
-
- _gnix_alps_cleanup();
-
- rc = gnixu_get_rdma_credentials(addr, &ptag, &cookie);
- cr_expect(!rc);
-
- rc = _gnix_job_fma_limit(0, ptag, &fmas);
- cr_expect(!rc);
-
- rc = _gnix_job_cq_limit(0, ptag, &cqs);
- cr_expect(!rc);
-
- rc = _gnix_pes_on_node(&npes);
- cr_expect(!rc);
-
- rc = _gnix_nics_per_rank(&npr);
- cr_expect(!rc);
-
- cqs /= GNIX_CQS_PER_EP;
- cr_expect(((fmas > cqs ? cqs : fmas) / npes) == npr);
-
- _gnix_alps_cleanup();
-}
-
-static void test_destruct(void *obj)
-{
- struct gnix_reference_tester *t = (struct gnix_reference_tester *) obj;
-
- t->destructed = 1;
-}
-
-Test(utils, references)
-{
- int refs;
- struct gnix_reference_tester test;
-
- /* initialize test structure */
- _gnix_ref_init(&test.ref_cnt, 1, test_destruct);
- test.destructed = 0;
-
- /* check for validity */
- cr_assert(atomic_get(&test.ref_cnt.references) == 1);
- cr_assert(test.destructed == 0);
-
- /* increment refs and check */
- refs = _gnix_ref_get(&test);
- cr_assert(refs == 2);
- cr_assert(atomic_get(&test.ref_cnt.references) == 2);
- cr_assert(test.destructed == 0);
-
- /* decrement refs and check */
- refs = _gnix_ref_put(&test);
- cr_assert(refs == 1);
- cr_assert(atomic_get(&test.ref_cnt.references) == 1);
- cr_assert(test.destructed == 0);
-
- /* decrement and destruct, check for validity */
- refs = _gnix_ref_put(&test);
- cr_assert(refs == 0);
- cr_assert(atomic_get(&test.ref_cnt.references) == 0);
- cr_assert(test.destructed == 1);
-}
-
diff --git a/prov/gni/test/vc.c b/prov/gni/test/vc.c
deleted file mode 100644
index ee5bdba..0000000
--- a/prov/gni/test/vc.c
+++ /dev/null
@@ -1,388 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <getopt.h>
-#include <poll.h>
-#include <time.h>
-#include <string.h>
-#include <pthread.h>
-
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_errno.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <inttypes.h>
-
-#include "gnix_vc.h"
-#include "gnix_nic.h"
-#include "gnix_cm_nic.h"
-#include "gnix_hashtable.h"
-#include "gnix_av.h"
-
-#include <criterion/criterion.h>
-
-static struct fid_fabric *fab;
-static struct fid_domain *dom;
-static struct fid_ep *ep[2];
-static struct fid_av *av;
-static struct fi_info *hints;
-static struct fi_info *fi;
-static struct fid_cq *cq;
-static struct fi_cq_attr cq_attr;
-void *ep_name[2];
-fi_addr_t gni_addr[2];
-struct gnix_av_addr_entry * gnix_addr[2];
-size_t gnix_addrlen[2];
-
-static void vc_setup_common(void);
-
-static void vc_setup_auto(void)
-{
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = 4;
- hints->domain_attr->control_progress = FI_PROGRESS_AUTO;
- hints->mode = ~0;
-
- vc_setup_common();
-}
-
-static void vc_setup_manual(void)
-{
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->domain_attr->cq_data_size = 4;
- hints->domain_attr->control_progress = FI_PROGRESS_MANUAL;
- hints->mode = ~0;
- vc_setup_common();
-}
-
-static void vc_setup_common(void)
-{
- int ret = 0;
- struct fi_av_attr attr;
- size_t addrlen = 0;
- struct gnix_fid_av *gnix_av;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-
- ret = fi_domain(fab, fi, &dom, NULL);
- cr_assert(!ret, "fi_domain");
-
- attr.type = FI_AV_MAP;
- attr.count = 16;
-
- ret = fi_av_open(dom, &attr, &av, NULL);
- cr_assert(!ret, "fi_av_open");
-
- gnix_av = container_of(av, struct gnix_fid_av, av_fid);
-
- ret = fi_endpoint(dom, fi, &ep[0], NULL);
- cr_assert(!ret, "fi_endpoint");
-
- ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
- cr_assert(addrlen > 0);
-
- ep_name[0] = malloc(addrlen);
- cr_assert(ep_name[0] != NULL);
-
- ep_name[1] = malloc(addrlen);
- cr_assert(ep_name[1] != NULL);
-
- ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_endpoint(dom, fi, &ep[1], NULL);
- cr_assert(!ret, "fi_endpoint");
-
- ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_av_insert(av, ep_name[0], 1, &gni_addr[0], 0,
- NULL);
- cr_assert(ret == 1);
-
- ret = _gnix_av_lookup(gnix_av, gni_addr[0], &gnix_addr[0]);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_av_insert(av, ep_name[1], 1, &gni_addr[1], 0,
- NULL);
- cr_assert(ret == 1);
-
- ret = _gnix_av_lookup(gnix_av, gni_addr[1], &gnix_addr[1]);
- cr_assert(ret == FI_SUCCESS);
-
- ret = fi_ep_bind(ep[0], &av->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
- cq_attr.format = FI_CQ_FORMAT_TAGGED;
- cq_attr.size = 1024;
- cq_attr.wait_obj = 0;
-
- ret = fi_cq_open(dom, &cq_attr, &cq, 0);
- cr_assert(!ret, "fi_cq_open");
-
- ret = fi_ep_bind(ep[0], &cq->fid, FI_SEND | FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_enable(ep[0]);
- cr_assert(!ret, "fi_enable");
-
- ret = fi_ep_bind(ep[1], &cq->fid, FI_SEND | FI_RECV);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_ep_bind(ep[1], &av->fid, 0);
- cr_assert(!ret, "fi_ep_bind");
-
- ret = fi_enable(ep[1]);
- cr_assert(!ret, "fi_ep_enable");
-}
-
-void vc_teardown(void)
-{
- int ret = 0;
-
- ret = fi_close(&ep[0]->fid);
- cr_assert(!ret, "failure in closing ep.");
-
- ret = fi_close(&ep[1]->fid);
- cr_assert(!ret, "failure in closing ep.");
-
- ret = fi_close(&cq->fid);
- cr_assert(!ret, "failure in closing cq.");
-
- ret = fi_close(&av->fid);
- cr_assert(!ret, "failure in closing av.");
-
- ret = fi_close(&dom->fid);
- cr_assert(!ret, "failure in closing domain.");
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
- free(ep_name[0]);
- free(ep_name[1]);
-}
-
-/*******************************************************************************
- * Test vc functions.
- ******************************************************************************/
-
-TestSuite(vc_management_auto, .init = vc_setup_auto, .fini = vc_teardown,
- .disabled = false);
-
-TestSuite(vc_management_manual, .init = vc_setup_manual, .fini = vc_teardown,
- .disabled = false);
-
-Test(vc_management_auto, vc_alloc_simple)
-{
- int ret;
- struct gnix_vc *vc[2];
- struct gnix_fid_ep *ep_priv;
-
- ep_priv = container_of(ep[0], struct gnix_fid_ep, ep_fid);
-
- ret = _gnix_vc_alloc(ep_priv, gnix_addr[0], &vc[0]);
- cr_assert_eq(ret, FI_SUCCESS);
-
- ret = _gnix_vc_alloc(ep_priv, gnix_addr[1], &vc[1]);
- cr_assert_eq(ret, FI_SUCCESS);
-
- /*
- * vc_id's have to be different since the
- * vc's were allocated using the same ep.
- */
- cr_assert_neq(vc[0]->vc_id, vc[1]->vc_id);
-
- ret = _gnix_vc_destroy(vc[0]);
- cr_assert_eq(ret, FI_SUCCESS);
-
- ret = _gnix_vc_destroy(vc[1]);
- cr_assert_eq(ret, FI_SUCCESS);
-}
-
-Test(vc_management_auto, vc_lookup_by_id)
-{
- int ret;
- struct gnix_vc *vc[2], *vc_chk;
- struct gnix_fid_ep *ep_priv;
-
- ep_priv = container_of(ep[0], struct gnix_fid_ep, ep_fid);
-
- ret = _gnix_vc_alloc(ep_priv, gnix_addr[0], &vc[0]);
- cr_assert_eq(ret, FI_SUCCESS);
-
- ret = _gnix_vc_alloc(ep_priv, gnix_addr[1], &vc[1]);
- cr_assert_eq(ret, FI_SUCCESS);
-
- vc_chk = __gnix_nic_elem_by_rem_id(ep_priv->nic, vc[0]->vc_id);
- cr_assert_eq(vc_chk, vc[0]);
-
- vc_chk = __gnix_nic_elem_by_rem_id(ep_priv->nic, vc[1]->vc_id);
- cr_assert_eq(vc_chk, vc[1]);
-
- ret = _gnix_vc_destroy(vc[0]);
- cr_assert_eq(ret, FI_SUCCESS);
-
- ret = _gnix_vc_destroy(vc[1]);
- cr_assert_eq(ret, FI_SUCCESS);
-
-}
-
-Test(vc_management_auto, vc_connect)
-{
- int ret;
- struct gnix_vc *vc_conn;
- struct gnix_fid_ep *ep_priv[2];
- gnix_ht_key_t key;
- enum gnix_vc_conn_state state;
-
- ep_priv[0] = container_of(ep[0], struct gnix_fid_ep, ep_fid);
-
- ep_priv[1] = container_of(ep[1], struct gnix_fid_ep, ep_fid);
-
- ret = _gnix_vc_alloc(ep_priv[0], gnix_addr[1], &vc_conn);
- cr_assert_eq(ret, FI_SUCCESS);
-
- memcpy(&key, &gni_addr[1],
- sizeof(gnix_ht_key_t));
-
- ret = _gnix_ht_insert(ep_priv[0]->vc_ht, key, vc_conn);
- cr_assert_eq(ret, FI_SUCCESS);
- vc_conn->modes |= GNIX_VC_MODE_IN_HT;
-
- ret = _gnix_vc_connect(vc_conn);
- cr_assert_eq(ret, FI_SUCCESS);
-
- /*
- * since we asked for FI_PROGRESS_AUTO for control
- * we can just spin here. add a yield in case the
- * test is only being run on one cpu.
- */
-
- state = GNIX_VC_CONN_NONE;
- while (state != GNIX_VC_CONNECTED) {
- pthread_yield();
- state = _gnix_vc_state(vc_conn);
- }
-
- ret = _gnix_vc_disconnect(vc_conn);
- cr_assert_eq(ret, FI_SUCCESS);
-
- /* VC is destroyed by the EP */
-}
-
-Test(vc_management_auto, vc_connect2)
-{
- int ret;
- struct gnix_vc *vc_conn0, *vc_conn1;
- struct gnix_fid_ep *ep_priv[2];
- gnix_ht_key_t key;
- enum gnix_vc_conn_state state;
-
- ep_priv[0] = container_of(ep[0], struct gnix_fid_ep, ep_fid);
- ep_priv[1] = container_of(ep[1], struct gnix_fid_ep, ep_fid);
-
- ret = _gnix_vc_alloc(ep_priv[0], gnix_addr[1], &vc_conn0);
- cr_assert_eq(ret, FI_SUCCESS);
-
- memcpy(&key, &gni_addr[1],
- sizeof(gnix_ht_key_t));
-
- ret = _gnix_ht_insert(ep_priv[0]->vc_ht, key, vc_conn0);
- cr_assert_eq(ret, FI_SUCCESS);
-
- vc_conn0->modes |= GNIX_VC_MODE_IN_HT;
-
- ret = _gnix_vc_alloc(ep_priv[1], gnix_addr[0], &vc_conn1);
- cr_assert_eq(ret, FI_SUCCESS);
-
- memcpy(&key, &gni_addr[0],
- sizeof(gnix_ht_key_t));
-
- ret = _gnix_ht_insert(ep_priv[1]->vc_ht, key, vc_conn1);
- cr_assert_eq(ret, FI_SUCCESS);
-
- vc_conn1->modes |= GNIX_VC_MODE_IN_HT;
-
- ret = _gnix_vc_connect(vc_conn0);
- cr_assert_eq(ret, FI_SUCCESS);
-
- ret = _gnix_vc_connect(vc_conn1);
- cr_assert_eq(ret, FI_SUCCESS);
-
- /*
- * since we asked for FI_PROGRESS_AUTO for control
- * we can just spin here. add a yield in case the
- * test is only being run on one cpu.
- */
-
- state = GNIX_VC_CONN_NONE;
- while (state != GNIX_VC_CONNECTED) {
- pthread_yield();
- state = _gnix_vc_state(vc_conn0);
- }
-
- state = GNIX_VC_CONN_NONE;
- while (state != GNIX_VC_CONNECTED) {
- pthread_yield();
- state = _gnix_vc_state(vc_conn1);
- }
-
- ret = _gnix_vc_disconnect(vc_conn0);
- cr_assert_eq(ret, FI_SUCCESS);
-
- ret = _gnix_vc_disconnect(vc_conn1);
- cr_assert_eq(ret, FI_SUCCESS);
-
- /* VC is destroyed by the EP */
-}
diff --git a/prov/gni/test/vector.c b/prov/gni/test/vector.c
deleted file mode 100644
index fe13a6a..0000000
--- a/prov/gni/test/vector.c
+++ /dev/null
@@ -1,358 +0,0 @@
-/*
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All
- * rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "gnix_vector.h"
-#include <unistd.h>
-#include <criterion/criterion.h>
-#include <stdlib.h>
-
-gnix_vector_t vec;
-gnix_vec_attr_t attr;
-
-#define VEC_MAX (1024)
-#define VEC_INIT 128
-
-void vector_setup_lockless(void)
-{
- int ret;
-
- attr.vec_increase_step = 2;
- attr.vec_increase_type = GNIX_VEC_INCREASE_MULT;
- attr.vec_initial_size = VEC_INIT;
- attr.vec_internal_locking = GNIX_VEC_UNLOCKED;
- attr.vec_maximum_size = VEC_MAX;
-
- ret = _gnix_vec_init(&vec, &attr);
- cr_assert(!ret, "_gnix_vec_init");
-}
-
-void vector_setup_locked()
-{
- int ret;
-
- attr.vec_increase_step = 2;
- attr.vec_increase_type = GNIX_VEC_INCREASE_ADD;
- attr.vec_initial_size = VEC_INIT;
- attr.vec_internal_locking = GNIX_VEC_LOCKED;
- attr.vec_maximum_size = VEC_MAX;
-
- ret = _gnix_vec_init(&vec, &attr);
- cr_assert(!ret, "_gnix_vec_init");
-}
-
-void do_invalid_ops_params()
-{
- int ret;
- void *tmp;
-
- /* ret = _gnix_vec_insert_first(&vec, NULL); */
- /* cr_assert_eq(ret, -FI_EINVAL); */
-
- /* ret = _gnix_vec_remove_first(&vec); */
- /* cr_assert_eq(ret, -FI_EINVAL); */
-
- /* ret = _gnix_vec_first(&vec, &tmp); */
- /* cr_assert_eq(ret, -FI_EINVAL); */
-
- ret = _gnix_vec_insert_first(NULL, NULL);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_vec_remove_first(NULL);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_vec_first(NULL, &tmp);
- cr_assert_eq(ret, -FI_EINVAL);
-}
-
-void vector_teardown(void)
-{
- int ret;
-
- ret = _gnix_vec_close(&vec);
- cr_assert(!ret, "_gnix_vec_close");
-
- do_invalid_ops_params();
-
- /* ret = _gnix_vec_close(&vec); */
- /* cr_assert_eq(ret, -FI_EINVAL); */
-
- ret = _gnix_vec_close(NULL);
- cr_assert_eq(ret, -FI_EINVAL);
-}
-
-/* Test invalid parameters for setup */
-void vector_setup_error(void)
-{
- int ret;
-
- attr.vec_increase_step = 2;
- attr.vec_increase_type = GNIX_VEC_INCREASE_MULT;
- attr.vec_initial_size = VEC_INIT;
- attr.vec_internal_locking = GNIX_VEC_UNLOCKED;
- attr.vec_maximum_size = VEC_MAX;
-
- ret = _gnix_vec_init(NULL, NULL);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_vec_init(NULL, &attr);
- cr_assert_eq(ret, -FI_EINVAL);
-
- ret = _gnix_vec_init(&vec, NULL);
- cr_assert_eq(ret, -FI_EINVAL);
-
- attr.vec_initial_size = 256;
- attr.vec_maximum_size = 128;
-
- ret = _gnix_vec_init(&vec, &attr);
- cr_assert_eq(ret, -FI_EINVAL);
-
- /* attr.vec_initial_size = 64; */
- /* attr.vec_maximum_size = 128; */
- /* vec.state = GNIX_VEC_STATE_READY; */
- /* ret = _gnix_vec_init(&vec, &attr); */
-
- vector_setup_lockless();
-}
-
-void do_insert_first()
-{
- int ret;
- void *tmp = malloc(sizeof(gnix_vec_entry_t));
- cr_assert(tmp, "do_insert_first");
-
- ret = vec.ops->insert_first(&vec, tmp);
- cr_assert(!ret, "_gnix_vec_insert_first");
-
- ret = vec.ops->insert_first(&vec, tmp);
- cr_assert_eq(ret, -FI_ECANCELED);
-}
-
-void do_insert_last()
-{
- int ret;
- void *tmp = malloc(sizeof(gnix_vec_entry_t));
- cr_assert(tmp, "do_insert_last");
-
- ret = vec.ops->insert_last(&vec, tmp);
- cr_assert(!ret, "_gnix_vec_insert_last");
-
- ret = vec.ops->insert_last(&vec, tmp);
- cr_assert_eq(ret, -FI_ECANCELED);
-}
-
-void do_fill_insert_at()
-{
- int i, ret;
- void *tmp;
-
- for (i = 0; i < vec.attr.cur_size; i++) {
- tmp = malloc(sizeof(gnix_vec_entry_t));
- cr_assert(tmp, "do_insert_at");
-
- ret = vec.ops->insert_at(&vec, tmp, i);
- cr_assert(!ret, "_gnix_vec_insert_at");
- }
-
- /* Test grow. */
- tmp = malloc(sizeof(gnix_vec_entry_t));
- cr_assert(tmp, "do_insert_at");
-
- ret = vec.ops->insert_at(&vec, tmp, VEC_MAX-1);
- cr_assert(!ret, "_gnix_vec_insert_at");
- cr_assert_eq(vec.attr.cur_size, VEC_MAX);
-
- ret = vec.ops->insert_at(&vec, tmp, VEC_MAX-1);
- cr_assert_eq(ret, -FI_ECANCELED);
-
- for (; i < vec.attr.cur_size - 1; i++) {
- tmp = malloc(sizeof(gnix_vec_entry_t));
- cr_assert(tmp, "do_insert_at");
-
- ret = vec.ops->insert_at(&vec, tmp, i);
- cr_assert(!ret, "_gnix_vec_insert_at");
- }
-
- ret = vec.ops->insert_at(&vec, tmp, VEC_MAX);
- cr_assert_eq(ret, -FI_EINVAL);
-}
-
-void do_remove_first()
-{
- int ret;
-
- ret = vec.ops->remove_first(&vec);
- cr_assert(!ret, "_gnix_vec_remove_first");
-
- ret = vec.ops->remove_first(&vec);
- cr_assert_eq(ret, -FI_ECANCELED);
-}
-
-void do_remove_last()
-{
- int ret;
-
- ret = vec.ops->remove_last(&vec);
- cr_assert(!ret, "_gnix_vec_remove_last");
-
- ret = vec.ops->remove_last(&vec);
- cr_assert_eq(ret, -FI_ECANCELED);
-}
-
-void do_unfill_remove_at()
-{
- int i, ret;
-
- for (i = 0; i < vec.attr.cur_size; i++) {
- ret = vec.ops->remove_at(&vec, i);
- cr_assert(!ret, "_gnix_vec_remove_at");
- }
-
- ret = vec.ops->remove_at(&vec, vec.attr.cur_size / 2);
- cr_assert_eq(ret, -FI_ECANCELED);
-}
-
-void do_first()
-{
- int ret;
- void *tmp;
-
- ret = vec.ops->first(&vec, &tmp);
- cr_assert(tmp, "_gnix_vec_first");
- cr_assert(!ret, "_gnix_vec_first");
-
- tmp = NULL;
-
- do_remove_first();
-
- ret = vec.ops->first(&vec, &tmp);
- cr_assert_eq(tmp, NULL);
- cr_assert_eq(ret, -FI_ECANCELED);
-}
-
-void do_last()
-{
- int ret;
- void *tmp;
-
- ret = vec.ops->last(&vec, &tmp);
- cr_assert(tmp, "_gnix_vec_last");
- cr_assert(!ret, "_gnix_vec_last");
-
- tmp = NULL;
-
- do_remove_last();
-
- ret = vec.ops->last(&vec, &tmp);
- cr_assert_eq(tmp, NULL);
- cr_assert_eq(ret, -FI_ECANCELED);
-}
-
-void do_at()
-{
- int i ,ret;
- void *tmp;
-
- for (i = 0; i < vec.attr.cur_size; i++) {
- ret = vec.ops->at(&vec, &tmp, i);
- cr_assert(!ret, "_gnix_vec_at");
-
- cr_assert(!!tmp, "_gnix_vec_at");
- tmp = NULL;
- }
-}
-
-void do_tests()
-{
- do_insert_first();
- do_first();
-
- do_insert_last();
- do_last();
-
- do_fill_insert_at();
- do_at();
- do_unfill_remove_at();
-}
-
-TestSuite(vector_lockless, .init = vector_setup_lockless,
- .fini = vector_teardown, .disabled = false);
-
-Test(vector_lockless, do_first)
-{
- do_insert_first();
- do_first();
-}
-
-Test(vector_lockless, do_last)
-{
- do_insert_last();
- do_last();
-}
-
-Test(vector_lockless, do_at)
-{
- do_fill_insert_at();
- do_at();
- do_unfill_remove_at();
-}
-
-TestSuite(vector_locked, .init = vector_setup_locked,
- .fini = vector_teardown, .disabled = false);
-
-Test(vector_locked, do_first)
-{
- do_insert_first();
- do_first();
-}
-
-Test(vector_locked, do_last)
-{
- do_insert_last();
- do_last();
-}
-
-Test(vector_locked, do_at)
-{
- do_fill_insert_at();
- do_at();
- do_unfill_remove_at();
-}
-
-TestSuite(vector_error_lockless, .init = vector_setup_error,
- .fini = vector_teardown, .disabled = false);
-
-Test(vector_error_lockless, setup_teardown_error)
-{
-
-}
diff --git a/prov/gni/test/wait.c b/prov/gni/test/wait.c
deleted file mode 100644
index 828dcc0..0000000
--- a/prov/gni/test/wait.c
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Copyright (c) 2015 Los Alamos National Security, LLC. All rights reserved.
- * Copyright (c) 2015-2016 Cray Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "gnix.h"
-#include "gnix_wait.h"
-#include <fi_list.h>
-#include <string.h>
-
-#include <criterion/criterion.h>
-
-static struct fid_fabric *fab;
-static struct fi_info *hints;
-static struct fi_info *fi;
-static struct gnix_fid_wait *wait_priv;
-static struct fi_wait_attr wait_attr;
-static struct fid_wait *wait_set;
-
-void wait_setup(void)
-{
- int ret = 0;
-
- hints = fi_allocinfo();
- cr_assert(hints, "fi_allocinfo");
-
- hints->mode = ~0;
-
- hints->fabric_attr->name = strdup("gni");
-
- ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
- cr_assert(!ret, "fi_getinfo");
-
- ret = fi_fabric(fi->fabric_attr, &fab, NULL);
- cr_assert(!ret, "fi_fabric");
-}
-
-void wait_teardown(void)
-{
- int ret = 0;
-
- ret = fi_close(&wait_set->fid);
- cr_assert(!ret, "failure in closing wait set.");
-
- ret = fi_close(&fab->fid);
- cr_assert(!ret, "failure in closing fabric.");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-}
-
-void setup_wait_type(enum fi_wait_obj wait_obj)
-{
- int ret;
-
- wait_setup();
- wait_attr.wait_obj = wait_obj;
-
- ret = fi_wait_open(fab, &wait_attr, &wait_set);
- cr_assert(!ret, "fi_wait_open");
-
- wait_priv = container_of(wait_set, struct gnix_fid_wait, wait);
-}
-
-void unspec_setup(void)
-{
- setup_wait_type(FI_WAIT_UNSPEC);
-}
-
-void fd_setup(void)
-{
- setup_wait_type(FI_WAIT_FD);
-}
-
-void mutex_cond_setup(void)
-{
- setup_wait_type(FI_WAIT_MUTEX_COND);
-}
-
-Test(wait_creation, unspec, .init = unspec_setup, .fini = wait_teardown)
-{
- cr_expect_eq(wait_priv->type, FI_WAIT_FD);
- cr_expect_eq(wait_priv->type, wait_attr.wait_obj);
- cr_expect_eq(&wait_priv->fabric->fab_fid, fab);
- cr_expect_eq(wait_priv->cond_type, FI_CQ_COND_NONE);
-}
-
-Test(wait_creation, fd, .init = fd_setup, .fini = wait_teardown)
-{
- cr_expect_eq(wait_priv->type, FI_WAIT_FD);
- cr_expect_eq(wait_priv->type, wait_attr.wait_obj);
- cr_expect_eq(&wait_priv->fabric->fab_fid, fab);
- cr_expect_eq(wait_priv->cond_type, FI_CQ_COND_NONE);
-}
-
-Test(wait_creation, mutex_cond, .init = mutex_cond_setup, .fini = wait_teardown)
-{
- cr_expect_eq(wait_priv->type, FI_WAIT_MUTEX_COND);
- cr_expect_eq(wait_priv->type, wait_attr.wait_obj);
- cr_expect_eq(&wait_priv->fabric->fab_fid, fab);
- cr_expect_eq(wait_priv->cond_type, FI_CQ_COND_NONE);
-}
-
-Test(wait_control, unspec, .init = unspec_setup, .fini = wait_teardown,
- .disabled = true)
-{
- int fd;
- int ret;
-
- ret = fi_control(&wait_priv->wait.fid, FI_GETWAIT, &fd);
- cr_expect_eq(FI_SUCCESS, ret, "fi_control failed.");
-
- cr_expect_eq(wait_priv->fd[WAIT_READ], fd);
-}
-
-Test(wait_control, fd, .init = fd_setup, .fini = wait_teardown,
- .disabled = true)
-{
- int fd;
- int ret;
-
- ret = fi_control(&wait_priv->wait.fid, FI_GETWAIT, &fd);
- cr_expect_eq(FI_SUCCESS, ret, "fi_control failed.");
-
- cr_expect_eq(wait_priv->fd[WAIT_READ], fd);
-}
-
-Test(wait_control, mutex_cond, .init = mutex_cond_setup, .fini = wait_teardown,
- .disabled = true)
-{
- int ret;
- struct fi_mutex_cond mutex_cond;
-
- ret = fi_control(&wait_priv->wait.fid, FI_GETWAIT, &mutex_cond);
- cr_expect_eq(FI_SUCCESS, ret, "fi_control failed.");
-
- ret = memcmp(&wait_priv->mutex, mutex_cond.mutex,
- sizeof(*mutex_cond.mutex));
- cr_expect_eq(0, ret, "mutex compare failed.");
-
- ret = memcmp(&wait_priv->cond, mutex_cond.cond,
- sizeof(*mutex_cond.cond));
- cr_expect_eq(0, ret, "cond compare failed.");
-}
-
-Test(wait_set, add, .init = fd_setup)
-{
- int ret;
- struct gnix_wait_entry *entry;
-
- struct fid temp_wait = {
- .fclass = FI_CLASS_CQ
- };
-
- cr_expect(slist_empty(&wait_priv->set),
- "wait set is not initially empty.");
- ret = _gnix_wait_set_add(&wait_priv->wait, &temp_wait);
-
- cr_expect_eq(FI_SUCCESS, ret, "gnix_wait_set_add failed.");
-
- cr_expect(!slist_empty(&wait_priv->set),
- "wait set is empty after add.");
-
- entry = container_of(wait_priv->set.head, struct gnix_wait_entry,
- entry);
-
- ret = memcmp(entry->wait_obj, &temp_wait, sizeof(temp_wait));
- cr_expect_eq(0, ret, "wait objects are not equal.");
-
- ret = fi_close(&wait_set->fid);
- cr_expect_eq(-FI_EBUSY, ret);
-
- ret = _gnix_wait_set_remove(&wait_priv->wait, &temp_wait);
-
- cr_expect_eq(FI_SUCCESS, ret, "gnix_wait_set_remove failed.");
-
- ret = fi_close(&wait_set->fid);
- cr_expect_eq(FI_SUCCESS, ret, "fi_close on wait set failed.");
-
- ret = fi_close(&fab->fid);
- cr_expect_eq(FI_SUCCESS, ret, "failure in closing fabric.");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-}
-
-Test(wait_set, empty_remove, .init = fd_setup)
-{
- int ret;
-
- struct fid temp_wait = {
- .fclass = FI_CLASS_CQ
- };
-
- cr_expect(slist_empty(&wait_priv->set));
- ret = _gnix_wait_set_remove(&wait_priv->wait, &temp_wait);
- cr_expect_eq(-FI_EINVAL, ret);
- cr_expect(slist_empty(&wait_priv->set));
-
- ret = fi_close(&wait_set->fid);
- cr_expect_eq(FI_SUCCESS, ret, "fi_close on wait set failed.");
-
- ret = fi_close(&fab->fid);
- cr_expect_eq(FI_SUCCESS, ret, "fi_close on fabric failed.");
-
- fi_freeinfo(fi);
- fi_freeinfo(hints);
-}
-
-Test(wait_verify, invalid_type, .init = wait_setup)
-{
- int ret;
-
- wait_attr.wait_obj = FI_WAIT_SET;
-
- ret = fi_wait_open(fab, &wait_attr, &wait_set);
- cr_expect_eq(-FI_EINVAL, ret,
- "Requesting incorrect type FI_WAIT_SET succeeded.");
-
- ret = fi_wait_open(fab, NULL, &wait_set);
- cr_expect_eq(-FI_EINVAL, ret,
- "Requesting verification with NULL attr succeeded.");
-
- wait_attr.flags = 1;
- ret = fi_wait_open(fab, &wait_attr, &wait_set);
- cr_expect_eq(-FI_EINVAL, ret,
- "Requesting verifications with flags set succeeded.");
-}
diff --git a/prov/mxm/AUTHORS b/prov/mxm/AUTHORS
deleted file mode 100644
index 07e24f3..0000000
--- a/prov/mxm/AUTHORS
+++ /dev/null
@@ -1,2 +0,0 @@
-Yohann Burette <yohann.burette at intel.com>
-Valentin Petrov <valentin.petrov at intel.com>
diff --git a/prov/mxm/src/mlxm.h b/prov/mxm/src/mlxm.h
index 435857a..90d5f21 100644
--- a/prov/mxm/src/mlxm.h
+++ b/prov/mxm/src/mlxm.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2015 Intel Corporation. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -44,18 +45,20 @@ extern "C" {
#include <string.h>
#include <unistd.h>
#include <rdma/fabric.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_domain.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_tagged.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_errno.h>
-#include <rdma/fi_log.h>
+#include <rdma/providers/fi_log.h>
+#include <rdma/providers/fi_prov.h>
#include "fi_enosys.h"
#include <mxm/api/mxm_api.h>
#include "mpool.h"
#include "uthash.h"
+#include <sys/uio.h>
+#include <fi.h>
extern int mlxm_errno_table[MXM_ERR_LAST];
static inline
diff --git a/prov/mxm/src/mlxm_av.c b/prov/mxm/src/mlxm_av.c
index e8fb204..fe5a16f 100644
--- a/prov/mxm/src/mlxm_av.c
+++ b/prov/mxm/src/mlxm_av.c
@@ -36,14 +36,12 @@ static int mlxm_av_insert(struct fid_av *av, const void *addr, size_t count,
fi_addr_t *fi_addr, uint64_t flags, void *context)
{
struct mlxm_fid_av *fid_av;
- struct mlxm_fid_ep *fid_ep;
mxm_error_t mxm_err;
void *mxm_addr;
size_t mxm_addrlen;
int i, err;
fid_av = container_of(av, struct mlxm_fid_av, av);
- fid_ep = fid_av->ep;
mxm_addrlen = fid_av->domain->mxm_addrlen;
for (i = 0; i < count; ++i) {
@@ -58,7 +56,7 @@ static int mlxm_av_insert(struct fid_av *av, const void *addr, size_t count,
(char*)mxm_addr+8,
*((mxm_conn_h*)&fi_addr[i]));
}
- return 0;
+ return count;
err_out:
return err;
}
@@ -68,8 +66,6 @@ static int mlxm_av_remove(struct fid_av *av, fi_addr_t *fi_addr, size_t count,
{
mxm_error_t mxm_err;
int i;
- struct mlxm_fid_av *fid_av;
- fid_av = container_of(av, struct mlxm_fid_av, av.fid);
if (mlxm_globals.mxm_ep) {
for (i = 0; i < count; ++i) {
diff --git a/prov/mxm/src/mlxm_domain.c b/prov/mxm/src/mlxm_domain.c
index 81729b8..edf0e1e 100644
--- a/prov/mxm/src/mlxm_domain.c
+++ b/prov/mxm/src/mlxm_domain.c
@@ -69,6 +69,7 @@ int mlxm_domain_open(struct fid_fabric *fabric, struct fi_info *info,
fid_domain->domain.fid.ops = &mlxm_fi_ops;
fid_domain->domain.ops = &mlxm_domain_ops;
+ fid_domain->domain.mr = &mlxm_mr_ops;
*domain = &fid_domain->domain;
return 0;
}
diff --git a/prov/mxm/src/mlxm_ep.c b/prov/mxm/src/mlxm_ep.c
index fd08569..c7b494c 100644
--- a/prov/mxm/src/mlxm_ep.c
+++ b/prov/mxm/src/mlxm_ep.c
@@ -81,9 +81,13 @@ static int mlxm_ep_close(fid_t fid)
static int mlxm_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
{
struct mlxm_fid_ep *fid_ep;
- int err = 0;
+ int ret;
fid_ep = container_of(fid, struct mlxm_fid_ep, ep.fid);
+ ret = ofi_ep_bind_valid(&mlxm_prov, bfid, flags);
+ if (ret)
+ return ret;
+
switch (bfid->fclass) {
case FI_CLASS_CQ:
/* TODO: check ress flags for send/recv ECs */
@@ -97,7 +101,7 @@ static int mlxm_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
default:
return -ENOSYS;
}
- return err;
+ return 0;
}
diff --git a/prov/psm/Makefile.include b/prov/psm/Makefile.include
index d8d3ab5..7130e9b 100644
--- a/prov/psm/Makefile.include
+++ b/prov/psm/Makefile.include
@@ -7,7 +7,6 @@ _psm_files = \
prov/psm/src/psmx_fabric.c \
prov/psm/src/psmx_domain.c \
prov/psm/src/psmx_cq.c \
- prov/psm/src/psmx_eq.c \
prov/psm/src/psmx_cntr.c \
prov/psm/src/psmx_av.c \
prov/psm/src/psmx_ep.c \
@@ -20,7 +19,6 @@ _psm_files = \
prov/psm/src/psmx_am.c \
prov/psm/src/psmx_mr.c \
prov/psm/src/psmx_wait.c \
- prov/psm/src/psmx_poll.c \
prov/psm/src/psmx_util.c
if HAVE_PSM_DL
diff --git a/prov/psm/src/psmx.h b/prov/psm/src/psmx.h
index 156c178..5c66b8f 100644
--- a/prov/psm/src/psmx.h
+++ b/prov/psm/src/psmx.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013-2014 Intel Corporation. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -53,7 +54,6 @@ extern "C" {
#include <netdb.h>
#include <complex.h>
#include <rdma/fabric.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_domain.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_tagged.h>
@@ -62,10 +62,11 @@ extern "C" {
#include <rdma/fi_trigger.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_errno.h>
-#include <rdma/fi_log.h>
#include "fi.h"
#include "fi_enosys.h"
#include "fi_list.h"
+#include "fi_util.h"
+#include "fi_file.h"
#include "rbtree.h"
#include "version.h"
@@ -73,6 +74,8 @@ extern struct fi_provider psmx_prov;
extern int psmx_am_compat_mode;
+#define PSMX_VERSION (FI_VERSION(1,3))
+
#define PSMX_OP_FLAGS (FI_INJECT | FI_MULTI_RECV | FI_COMPLETION | \
FI_TRIGGER | FI_INJECT_COMPLETE | \
FI_TRANSMIT_COMPLETE | FI_DELIVERY_COMPLETE)
@@ -159,7 +162,7 @@ struct psmx_am_request {
int op;
union {
struct {
- void *buf;
+ uint8_t *buf;
size_t len;
uint64_t addr;
uint64_t key;
@@ -169,7 +172,7 @@ struct psmx_am_request {
uint64_t data;
} write;
struct {
- void *buf;
+ uint8_t *buf;
size_t len;
uint64_t addr;
uint64_t key;
@@ -178,7 +181,7 @@ struct psmx_am_request {
size_t len_read;
} read;
struct {
- void *buf;
+ uint8_t *buf;
size_t len;
void *context;
void *peer_context;
@@ -187,19 +190,19 @@ struct psmx_am_request {
size_t len_sent;
} send;
struct {
- void *buf;
+ uint8_t *buf;
size_t len;
void *context;
void *src_addr;
size_t len_received;
} recv;
struct {
- void *buf;
+ uint8_t *buf;
size_t len;
uint64_t addr;
uint64_t key;
void *context;
- void *result;
+ uint8_t *result;
} atomic;
};
uint64_t cq_flags;
@@ -227,7 +230,7 @@ struct psmx_req_queue {
struct psmx_multi_recv {
uint64_t tag;
uint64_t tagsel;
- void *buf;
+ uint8_t *buf;
size_t len;
size_t offset;
int min_buf_size;
@@ -236,17 +239,15 @@ struct psmx_multi_recv {
};
struct psmx_fid_fabric {
- struct fid_fabric fabric;
- int refcnt;
+ struct util_fabric util_fabric;
struct psmx_fid_domain *active_domain;
psm_uuid_t uuid;
pthread_t name_server_thread;
};
struct psmx_fid_domain {
- struct fid_domain domain;
+ struct util_domain util_domain;
struct psmx_fid_fabric *fabric;
- int refcnt;
psm_ep_t psm_ep;
psm_epid_t psm_epid;
psm_mq_t psm_mq;
@@ -305,42 +306,6 @@ struct psmx_cq_event {
struct slist_entry list_entry;
};
-struct psmx_eq_event {
- int event;
- union {
- struct fi_eq_entry data;
- struct fi_eq_cm_entry cm;
- struct fi_eq_err_entry err;
- } eqe;
- int error;
- size_t entry_size;
- struct slist_entry list_entry;
-};
-
-struct psmx_fid_wait {
- struct fid_wait wait;
- struct psmx_fid_fabric *fabric;
- int type;
- union {
- int fd[2];
- struct {
- pthread_mutex_t mutex;
- pthread_cond_t cond;
- };
- };
-};
-
-struct psmx_poll_list {
- struct dlist_entry entry;
- struct fid *fid;
-};
-
-struct psmx_fid_poll {
- struct fid_poll poll;
- struct psmx_fid_domain *domain;
- struct dlist_entry poll_list_head;
-};
-
struct psmx_fid_cq {
struct fid_cq cq;
struct psmx_fid_domain *domain;
@@ -351,22 +316,11 @@ struct psmx_fid_cq {
struct slist free_list;
fastlock_t lock;
struct psmx_cq_event *pending_error;
- struct psmx_fid_wait *wait;
+ struct util_wait *wait;
int wait_cond;
int wait_is_local;
};
-struct psmx_fid_eq {
- struct fid_eq eq;
- struct psmx_fid_fabric *fabric;
- struct slist event_queue;
- struct slist error_queue;
- struct slist free_list;
- fastlock_t lock;
- struct psmx_fid_wait *wait;
- int wait_is_local;
-};
-
enum psmx_triggered_op {
PSMX_TRIGGERED_SEND,
PSMX_TRIGGERED_RECV,
@@ -497,15 +451,16 @@ struct psmx_trigger {
};
struct psmx_fid_cntr {
- struct fid_cntr cntr;
+ union {
+ struct fid_cntr cntr;
+ struct util_cntr util_cntr; /* for util_poll_run */
+ };
struct psmx_fid_domain *domain;
int events;
uint64_t flags;
- volatile uint64_t counter;
- volatile uint64_t error_counter;
- uint64_t counter_last_read;
- uint64_t error_counter_last_read;
- struct psmx_fid_wait *wait;
+ atomic_t counter;
+ atomic_t error_counter;
+ struct util_wait *wait;
int wait_is_local;
struct psmx_trigger *trigger;
pthread_mutex_t trigger_lock;
@@ -514,7 +469,7 @@ struct psmx_fid_cntr {
struct psmx_fid_av {
struct fid_av av;
struct psmx_fid_domain *domain;
- struct psmx_fid_eq *eq;
+ struct fid_eq *eq;
int type;
uint64_t flags;
size_t addrlen;
@@ -526,6 +481,7 @@ struct psmx_fid_av {
struct psmx_fid_ep {
struct fid_ep ep;
+ struct psmx_fid_ep *base_ep;
struct psmx_fid_domain *domain;
struct psmx_fid_av *av;
struct psmx_fid_cq *send_cq;
@@ -538,8 +494,11 @@ struct psmx_fid_ep {
struct psmx_fid_cntr *remote_read_cntr;
unsigned send_selective_completion:1;
unsigned recv_selective_completion:1;
- uint64_t flags;
+ unsigned enabled:1;
+ uint64_t tx_flags;
+ uint64_t rx_flags;
uint64_t caps;
+ atomic_t ref;
struct fi_context nocomp_send_context;
struct fi_context nocomp_recv_context;
size_t min_multi_recv;
@@ -603,34 +562,39 @@ int psmx_domain_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **domain, void *context);
int psmx_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
struct fid_wait **waitset);
+int psmx_wait_trywait(struct fid_fabric *fabric, struct fid **fids,
+ int count);
int psmx_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context);
int psmx_stx_ctx(struct fid_domain *domain, struct fi_tx_attr *attr,
struct fid_stx **stx, void *context);
int psmx_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
struct fid_cq **cq, void *context);
-int psmx_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr,
- struct fid_eq **eq, void *context);
int psmx_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
struct fid_av **av, void *context);
int psmx_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
struct fid_cntr **cntr, void *context);
-int psmx_poll_open(struct fid_domain *domain, struct fi_poll_attr *attr,
- struct fid_poll **pollset);
static inline void psmx_fabric_acquire(struct psmx_fid_fabric *fabric)
{
- ++fabric->refcnt;
+ atomic_inc(&fabric->util_fabric.ref);
}
-void psmx_fabric_release(struct psmx_fid_fabric *fabric);
+static inline void psmx_fabric_release(struct psmx_fid_fabric *fabric)
+{
+ atomic_dec(&fabric->util_fabric.ref);
+}
static inline void psmx_domain_acquire(struct psmx_fid_domain *domain)
{
- ++domain->refcnt;
+ atomic_inc(&domain->util_domain.ref);
+}
+
+static inline void psmx_domain_release(struct psmx_fid_domain *domain)
+{
+ atomic_dec(&domain->util_domain.ref);
}
-void psmx_domain_release(struct psmx_fid_domain *domain);
int psmx_domain_check_features(struct psmx_fid_domain *domain, int ep_cap);
int psmx_domain_enable_ep(struct psmx_fid_domain *domain, struct psmx_fid_ep *ep);
void psmx_domain_disable_ep(struct psmx_fid_domain *domain, struct psmx_fid_ep *ep);
@@ -644,12 +608,6 @@ int psmx_epid_to_epaddr(struct psmx_fid_domain *domain,
psm_epid_t epid, psm_epaddr_t *epaddr);
void psmx_query_mpi(void);
-void psmx_eq_enqueue_event(struct psmx_fid_eq *eq, struct psmx_eq_event *event);
-struct psmx_eq_event *psmx_eq_create_event(struct psmx_fid_eq *eq,
- uint32_t event_num,
- void *context, uint64_t data,
- int err, int prov_errno,
- void *err_data, size_t err_data_size);
void psmx_cq_enqueue_event(struct psmx_fid_cq *cq, struct psmx_cq_event *event);
struct psmx_cq_event *psmx_cq_create_event(struct psmx_fid_cq *cq,
void *op_context, void *buf,
@@ -658,9 +616,6 @@ struct psmx_cq_event *psmx_cq_create_event(struct psmx_fid_cq *cq,
size_t olen, int err);
int psmx_cq_poll_mq(struct psmx_fid_cq *cq, struct psmx_fid_domain *domain,
struct psmx_cq_event *event, int count, fi_addr_t *src_addr);
-int psmx_wait_get_obj(struct psmx_fid_wait *wait, void *arg);
-int psmx_wait_wait(struct fid_wait *wait, int timeout);
-void psmx_wait_signal(struct fid_wait *wait);
int psmx_am_init(struct psmx_fid_domain *domain);
int psmx_am_fini(struct psmx_fid_domain *domain);
@@ -689,10 +644,10 @@ void psmx_cntr_add_trigger(struct psmx_fid_cntr *cntr, struct psmx_trigger *trig
static inline void psmx_cntr_inc(struct psmx_fid_cntr *cntr)
{
- cntr->counter++;
+ atomic_inc(&cntr->counter);
psmx_cntr_check_trigger(cntr);
if (cntr->wait)
- psmx_wait_signal((struct fid_wait *)cntr->wait);
+ cntr->wait->signal(cntr->wait);
}
static inline void psmx_progress(struct psmx_fid_domain *domain)
diff --git a/prov/psm/src/psmx_atomic.c b/prov/psm/src/psmx_atomic.c
index 58810bc..f66bbf6 100644
--- a/prov/psm/src/psmx_atomic.c
+++ b/prov/psm/src/psmx_atomic.c
@@ -377,7 +377,7 @@ int psmx_am_atomic_handler(psm_am_token_t token, psm_epaddr_t epaddr,
{
psm_amarg_t rep_args[8];
int count;
- void *addr;
+ uint8_t *addr;
uint64_t key;
int datatype, op;
int err = 0;
@@ -393,7 +393,7 @@ int psmx_am_atomic_handler(psm_am_token_t token, psm_epaddr_t epaddr,
switch (args[0].u32w0 & PSMX_AM_OP_MASK) {
case PSMX_AM_REQ_ATOMIC_WRITE:
count = args[0].u32w1;
- addr = (void *)(uintptr_t)args[2].u64;
+ addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
datatype = args[4].u32w0;
op = args[4].u32w1;
@@ -429,7 +429,7 @@ int psmx_am_atomic_handler(psm_am_token_t token, psm_epaddr_t epaddr,
case PSMX_AM_REQ_ATOMIC_READWRITE:
count = args[0].u32w1;
- addr = (void *)(uintptr_t)args[2].u64;
+ addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
datatype = args[4].u32w0;
op = args[4].u32w1;
@@ -480,7 +480,7 @@ int psmx_am_atomic_handler(psm_am_token_t token, psm_epaddr_t epaddr,
case PSMX_AM_REQ_ATOMIC_COMPWRITE:
count = args[0].u32w1;
- addr = (void *)(uintptr_t)args[2].u64;
+ addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
datatype = args[4].u32w0;
op = args[4].u32w1;
@@ -496,7 +496,7 @@ int psmx_am_atomic_handler(psm_am_token_t token, psm_epaddr_t epaddr,
addr += mr->offset;
tmp_buf = malloc(len);
if (tmp_buf)
- psmx_atomic_do_compwrite(addr, src, src + len,
+ psmx_atomic_do_compwrite(addr, src, (uint8_t *)src + len,
tmp_buf, datatype, op, count);
else
op_error = -FI_ENOMEM;
@@ -812,9 +812,9 @@ ssize_t _psmx_atomic_write(struct fid_ep *ep,
req = malloc(sizeof(*req) + len);
if (!req)
return -FI_ENOMEM;
- memset((void *)req, 0, sizeof(*req));
- memcpy((void *)req+sizeof(*req), (void *)buf, len);
- buf = (void *)req + sizeof(*req);
+ memset(req, 0, sizeof(*req));
+ memcpy((uint8_t *)req+sizeof(*req), (void *)buf, len);
+ buf = (uint8_t *)req + sizeof(*req);
} else {
req = calloc(1, sizeof(*req));
if (!req)
@@ -860,7 +860,7 @@ static ssize_t psmx_atomic_write(struct fid_ep *ep,
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
return _psmx_atomic_write(ep, buf, count, desc,
dest_addr, addr, key,
- datatype, op, context, ep_priv->flags);
+ datatype, op, context, ep_priv->tx_flags);
}
static ssize_t psmx_atomic_writemsg(struct fid_ep *ep,
@@ -908,7 +908,7 @@ static ssize_t psmx_atomic_inject(struct fid_ep *ep,
return _psmx_atomic_write(ep, buf, count, NULL/*desc*/,
dest_addr, addr, key,
datatype, op, NULL,
- ep_priv->flags | FI_INJECT | PSMX_NO_COMPLETION);
+ ep_priv->tx_flags | FI_INJECT | PSMX_NO_COMPLETION);
}
ssize_t _psmx_atomic_readwrite(struct fid_ep *ep,
@@ -999,9 +999,9 @@ ssize_t _psmx_atomic_readwrite(struct fid_ep *ep,
req = malloc(sizeof(*req) + len);
if (!req)
return -FI_ENOMEM;
- memset((void *)req, 0, sizeof(*req));
- memcpy((void *)req+sizeof(*req), (void *)buf, len);
- buf = (void *)req + sizeof(*req);
+ memset(req, 0, sizeof(*req));
+ memcpy((uint8_t *)req+sizeof(*req), (void *)buf, len);
+ buf = (uint8_t *)req + sizeof(*req);
} else {
req = calloc(1, sizeof(*req));
if (!req)
@@ -1053,7 +1053,7 @@ static ssize_t psmx_atomic_readwrite(struct fid_ep *ep,
return _psmx_atomic_readwrite(ep, buf, count, desc,
result, result_desc, dest_addr,
addr, key, datatype, op,
- context, ep_priv->flags);
+ context, ep_priv->tx_flags);
}
static ssize_t psmx_atomic_readwritemsg(struct fid_ep *ep,
@@ -1205,17 +1205,17 @@ ssize_t _psmx_atomic_compwrite(struct fid_ep *ep,
req = malloc(sizeof(*req) + len + len);
if (!req)
return -FI_ENOMEM;
- memset((void *)req, 0, sizeof(*req));
- memcpy((void *)req + sizeof(*req), (void *)buf, len);
- memcpy((void *)req + sizeof(*req) + len, (void *)compare, len);
- buf = (void *)req + sizeof(*req);
- compare = buf + len;
+ memset(req, 0, sizeof(*req));
+ memcpy((uint8_t *)req + sizeof(*req), (void *)buf, len);
+ memcpy((uint8_t *)req + sizeof(*req) + len, (void *)compare, len);
+ buf = (uint8_t *)req + sizeof(*req);
+ compare = (uint8_t *)buf + len;
} else {
req = calloc(1, sizeof(*req));
if (!req)
return -FI_ENOMEM;
- if (compare != buf + len) {
+ if ((uintptr_t)compare != (uintptr_t)buf + len) {
tmp_buf = malloc(len * 2);
if (!tmp_buf) {
free(req);
@@ -1223,7 +1223,7 @@ ssize_t _psmx_atomic_compwrite(struct fid_ep *ep,
}
memcpy(tmp_buf, buf, len);
- memcpy(tmp_buf + len, compare, len);
+ memcpy((uint8_t *)tmp_buf + len, compare, len);
}
}
@@ -1273,7 +1273,7 @@ static ssize_t psmx_atomic_compwrite(struct fid_ep *ep,
compare, compare_desc,
result, result_desc,
dest_addr, addr, key,
- datatype, op, context, ep_priv->flags);
+ datatype, op, context, ep_priv->tx_flags);
}
static ssize_t psmx_atomic_compwritemsg(struct fid_ep *ep,
diff --git a/prov/psm/src/psmx_av.c b/prov/psm/src/psmx_av.c
index 9576dcc..5411dd2 100644
--- a/prov/psm/src/psmx_av.c
+++ b/prov/psm/src/psmx_av.c
@@ -115,6 +115,29 @@ static int psmx_av_check_table_size(struct psmx_fid_av *av, size_t count)
return 0;
}
+static void psmx_av_post_completion(struct psmx_fid_av *av, void *context,
+ uint64_t data, int prov_errno)
+{
+ if (prov_errno) {
+ struct fi_eq_err_entry entry;
+ entry.fid = &av->av.fid;
+ entry.context = context;
+ entry.data = data;
+ entry.err = -psmx_errno(prov_errno);
+ entry.prov_errno = prov_errno;
+ entry.err_data = NULL;
+ entry.err_data_size = 0;
+ fi_eq_write(av->eq, FI_AV_COMPLETE, &entry, sizeof(entry),
+ UTIL_FLAG_ERROR);
+ } else {
+ struct fi_eq_entry entry;
+ entry.fid = &av->av.fid;
+ entry.context = context;
+ entry.data = data;
+ fi_eq_write(av->eq, FI_AV_COMPLETE, &entry, sizeof(entry), 0);
+ }
+}
+
static int psmx_av_insert(struct fid_av *av, const void *addr, size_t count,
fi_addr_t *fi_addr, uint64_t flags, void *context)
{
@@ -125,7 +148,12 @@ static int psmx_av_insert(struct fid_av *av, const void *addr, size_t count,
int i, j;
fi_addr_t *result = NULL;
struct psmx_epaddr_context *epaddr_context;
- struct psmx_eq_event *event;
+
+ if (count && !addr) {
+ FI_INFO(&psmx_prov, FI_LOG_AV,
+ "the input address array is NULL.\n");
+ return -FI_EINVAL;
+ }
av_priv = container_of(av, struct psmx_fid_av, av);
@@ -211,23 +239,8 @@ static int psmx_av_insert(struct fid_av *av, const void *addr, size_t count,
fi_addr[i] = FI_ADDR_NOTAVAIL;
error_count++;
- if (av_priv->flags & FI_EVENT) {
- event = psmx_eq_create_event(av_priv->eq,
- FI_AV_COMPLETE, /* event */
- context, /* context */
- i, /* data: failed index */
- psmx_errno(errors[i]), /* err */
- errors[i], /* prov_errno */
- NULL, /* err_data */
- 0); /* err_data_size */
- if (!event) {
- free(mask);
- free(errors);
- return -FI_ENOMEM;
- }
-
- psmx_eq_enqueue_event(av_priv->eq, event);
- }
+ if (av_priv->flags & FI_EVENT)
+ psmx_av_post_completion(av_priv, context, i, errors[i]);
}
}
@@ -251,18 +264,7 @@ static int psmx_av_insert(struct fid_av *av, const void *addr, size_t count,
if (!(av_priv->flags & FI_EVENT))
return count - error_count;
- event = psmx_eq_create_event(av_priv->eq,
- FI_AV_COMPLETE, /* event */
- context, /* context */
- count - error_count, /* data: succ count */
- 0, /* err */
- 0, /* prov_errno */
- NULL, /* err_data */
- 0); /* err_data_size */
- if (!event)
- return -FI_ENOMEM;
-
- psmx_eq_enqueue_event(av_priv->eq, event);
+ psmx_av_post_completion(av_priv, context, count - error_count, 0);
return 0;
}
@@ -340,7 +342,6 @@ static int psmx_av_close(fid_t fid)
static int psmx_av_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
{
struct psmx_fid_av *av;
- struct psmx_fid_eq *eq;
av = container_of(fid, struct psmx_fid_av, av.fid);
@@ -349,8 +350,7 @@ static int psmx_av_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
switch (bfid->fclass) {
case FI_CLASS_EQ:
- eq = container_of(bfid, struct psmx_fid_eq, eq.fid);
- av->eq = eq;
+ av->eq = (struct fid_eq *)bfid;
break;
default:
@@ -387,10 +387,14 @@ int psmx_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
size_t count = 64;
uint64_t flags = 0;
- domain_priv = container_of(domain, struct psmx_fid_domain, domain);
+ domain_priv = container_of(domain, struct psmx_fid_domain,
+ util_domain.domain_fid);
if (attr) {
switch (attr->type) {
+ case FI_AV_UNSPEC:
+ break;
+
case FI_AV_MAP:
case FI_AV_TABLE:
type = attr->type;
@@ -409,7 +413,14 @@ int psmx_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
FI_INFO(&psmx_prov, FI_LOG_AV,
"attr->flags=%x, supported=%x\n",
attr->flags, FI_EVENT);
- return -FI_EINVAL;
+ return -FI_ENOSYS;
+ }
+
+ if (attr->name) {
+ FI_INFO(&psmx_prov, FI_LOG_AV,
+ "attr->name=%s, named AV is not supported\n",
+ attr->name);
+ return -FI_ENOSYS;
}
}
@@ -431,6 +442,9 @@ int psmx_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
av_priv->av.ops = &psmx_av_ops;
*av = &av_priv->av;
+ if (attr)
+ attr->type = type;
+
return 0;
}
diff --git a/prov/psm/src/psmx_cntr.c b/prov/psm/src/psmx_cntr.c
index 024405c..186a51d 100644
--- a/prov/psm/src/psmx_cntr.c
+++ b/prov/psm/src/psmx_cntr.c
@@ -168,7 +168,7 @@ void psmx_cntr_check_trigger(struct psmx_fid_cntr *cntr)
trigger = cntr->trigger;
while (trigger) {
- if (cntr->counter < trigger->threshold)
+ if (atomic_get(&cntr->counter) < trigger->threshold)
break;
cntr->trigger = trigger->next;
@@ -223,9 +223,7 @@ static uint64_t psmx_cntr_read(struct fid_cntr *cntr)
poll_cnt = 0;
}
- cntr_priv->counter_last_read = cntr_priv->counter;
-
- return cntr_priv->counter_last_read;
+ return atomic_get(&cntr_priv->counter);
}
static uint64_t psmx_cntr_readerr(struct fid_cntr *cntr)
@@ -234,9 +232,7 @@ static uint64_t psmx_cntr_readerr(struct fid_cntr *cntr)
cntr_priv = container_of(cntr, struct psmx_fid_cntr, cntr);
- cntr_priv->error_counter_last_read = cntr_priv->error_counter;
-
- return cntr_priv->error_counter_last_read;
+ return atomic_get(&cntr_priv->error_counter);
}
static int psmx_cntr_add(struct fid_cntr *cntr, uint64_t value)
@@ -244,12 +240,12 @@ static int psmx_cntr_add(struct fid_cntr *cntr, uint64_t value)
struct psmx_fid_cntr *cntr_priv;
cntr_priv = container_of(cntr, struct psmx_fid_cntr, cntr);
- cntr_priv->counter += value;
+ atomic_add(&cntr_priv->counter, value);
psmx_cntr_check_trigger(cntr_priv);
if (cntr_priv->wait)
- psmx_wait_signal((struct fid_wait *)cntr_priv->wait);
+ cntr_priv->wait->signal(cntr_priv->wait);
return 0;
}
@@ -259,12 +255,12 @@ static int psmx_cntr_set(struct fid_cntr *cntr, uint64_t value)
struct psmx_fid_cntr *cntr_priv;
cntr_priv = container_of(cntr, struct psmx_fid_cntr, cntr);
- cntr_priv->counter = value;
+ atomic_set(&cntr_priv->counter, value);
psmx_cntr_check_trigger(cntr_priv);
if (cntr_priv->wait)
- psmx_wait_signal((struct fid_wait *)cntr_priv->wait);
+ cntr_priv->wait->signal(cntr_priv->wait);
return 0;
}
@@ -280,17 +276,17 @@ static int psmx_cntr_wait(struct fid_cntr *cntr, uint64_t threshold, int timeout
clock_gettime(CLOCK_REALTIME, &ts0);
- while (cntr_priv->counter < threshold) {
+ while (atomic_get(&cntr_priv->counter) < threshold) {
if (cntr_priv->wait) {
- ret = psmx_wait_wait((struct fid_wait *)cntr_priv->wait,
- timeout - msec_passed);
+ ret = fi_wait((struct fid_wait *)cntr_priv->wait,
+ timeout - msec_passed);
if (ret == -FI_ETIMEDOUT)
break;
} else {
psmx_progress(cntr_priv->domain);
}
- if (cntr_priv->counter >= threshold)
+ if (atomic_get(&cntr_priv->counter) >= threshold)
break;
if (timeout < 0)
@@ -315,12 +311,14 @@ static int psmx_cntr_close(fid_t fid)
cntr = container_of(fid, struct psmx_fid_cntr, cntr.fid);
- psmx_domain_release(cntr->domain);
-
- if (cntr->wait && cntr->wait_is_local)
- fi_close((fid_t)cntr->wait);
+ if (cntr->wait) {
+ fi_poll_del(&cntr->wait->pollset->poll_fid, &cntr->cntr.fid, 0);
+ if (cntr->wait_is_local)
+ fi_close((fid_t)cntr->wait);
+ }
pthread_mutex_destroy(&cntr->trigger_lock);
+ psmx_domain_release(cntr->domain);
free(cntr);
return 0;
@@ -345,10 +343,12 @@ static int psmx_cntr_control(fid_t fid, int command, void *arg)
break;
case FI_GETWAIT:
- /*
- ret = psmx_wait_get_obj(cntr->wait, arg);
+ if (cntr->wait)
+ ret = fi_control(&cntr->wait->wait_fid.fid, FI_GETWAIT, arg);
+ else
+ return -FI_EINVAL;
break;
- */
+
default:
return -FI_ENOSYS;
}
@@ -378,7 +378,7 @@ int psmx_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
{
struct psmx_fid_domain *domain_priv;
struct psmx_fid_cntr *cntr_priv;
- struct psmx_fid_wait *wait = NULL;
+ struct fid_wait *wait = NULL;
struct fi_wait_attr wait_attr;
int wait_is_local = 0;
int events;
@@ -387,7 +387,8 @@ int psmx_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
events = FI_CNTR_EVENTS_COMP;
flags = 0;
- domain_priv = container_of(domain, struct psmx_fid_domain, domain);
+ domain_priv = container_of(domain, struct psmx_fid_domain,
+ util_domain.domain_fid);
switch (attr->events) {
case FI_CNTR_EVENTS_COMP:
@@ -403,7 +404,6 @@ int psmx_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
switch (attr->wait_obj) {
case FI_WAIT_NONE:
- case FI_WAIT_UNSPEC:
break;
case FI_WAIT_SET:
@@ -412,15 +412,16 @@ int psmx_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
"FI_WAIT_SET is specified but attr->wait_set is NULL\n");
return -FI_EINVAL;
}
- wait = (struct psmx_fid_wait *)attr->wait_set;
+ wait = attr->wait_set;
break;
+ case FI_WAIT_UNSPEC:
case FI_WAIT_FD:
case FI_WAIT_MUTEX_COND:
wait_attr.wait_obj = attr->wait_obj;
wait_attr.flags = 0;
- err = psmx_wait_open(&domain_priv->fabric->fabric,
- &wait_attr, (struct fid_wait **)&wait);
+ err = fi_wait_open(&domain_priv->fabric->util_fabric.fabric_fid,
+ &wait_attr, (struct fid_wait **)&wait);
if (err)
return err;
wait_is_local = 1;
@@ -439,25 +440,31 @@ int psmx_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
goto fail;
}
- psmx_domain_acquire(domain_priv);
-
cntr_priv->domain = domain_priv;
cntr_priv->events = events;
- cntr_priv->wait = wait;
+ if (wait)
+ cntr_priv->wait = container_of(wait, struct util_wait, wait_fid);
cntr_priv->wait_is_local = wait_is_local;
cntr_priv->flags = flags;
cntr_priv->cntr.fid.fclass = FI_CLASS_CNTR;
cntr_priv->cntr.fid.context = context;
cntr_priv->cntr.fid.ops = &psmx_fi_ops;
cntr_priv->cntr.ops = &psmx_cntr_ops;
+ atomic_initialize(&cntr_priv->counter, 0);
+ atomic_initialize(&cntr_priv->error_counter, 0);
pthread_mutex_init(&cntr_priv->trigger_lock, NULL);
+ if (wait)
+ fi_poll_add(&cntr_priv->wait->pollset->poll_fid,
+ &cntr_priv->cntr.fid, 0);
+
+ psmx_domain_acquire(domain_priv);
*cntr = &cntr_priv->cntr;
return 0;
fail:
if (wait && wait_is_local)
- fi_close(&wait->wait.fid);
+ fi_close(&wait->fid);
return err;
}
diff --git a/prov/psm/src/psmx_cq.c b/prov/psm/src/psmx_cq.c
index 6c936c8..8a11ff0 100644
--- a/prov/psm/src/psmx_cq.c
+++ b/prov/psm/src/psmx_cq.c
@@ -40,7 +40,7 @@ void psmx_cq_enqueue_event(struct psmx_fid_cq *cq, struct psmx_cq_event *event)
fastlock_release(&cq->lock);
if (cq->wait)
- psmx_wait_signal((struct fid_wait *)cq->wait);
+ cq->wait->signal(cq->wait);
}
static struct psmx_cq_event *psmx_cq_dequeue_event(struct psmx_fid_cq *cq)
@@ -374,50 +374,39 @@ int psmx_cq_poll_mq(struct psmx_fid_cq *cq, struct psmx_fid_domain *domain,
multi_recv = 0;
switch ((int)PSMX_CTXT_TYPE(fi_context)) {
- case PSMX_NOCOMP_SEND_CONTEXT:
- tmp_cntr = tmp_ep->send_cntr;
- break;
-
- case PSMX_NOCOMP_RECV_CONTEXT:
- tmp_cntr = tmp_ep->recv_cntr;
- break;
-
- case PSMX_NOCOMP_WRITE_CONTEXT:
- tmp_cntr = tmp_ep->write_cntr;
- break;
-
- case PSMX_NOCOMP_READ_CONTEXT:
- tmp_cntr = tmp_ep->read_cntr;
- break;
-
case PSMX_SEND_CONTEXT:
case PSMX_TSEND_CONTEXT:
tmp_cq = tmp_ep->send_cq;
+ /* Fall through */
+ case PSMX_NOCOMP_SEND_CONTEXT:
tmp_cntr = tmp_ep->send_cntr;
break;
+ case PSMX_MULTI_RECV_CONTEXT:
+ multi_recv = 1;
+ /* Fall through */
case PSMX_RECV_CONTEXT:
case PSMX_TRECV_CONTEXT:
tmp_cq = tmp_ep->recv_cq;
+ /* Fall through */
+ case PSMX_NOCOMP_RECV_CONTEXT:
tmp_cntr = tmp_ep->recv_cntr;
break;
- case PSMX_MULTI_RECV_CONTEXT:
- multi_recv = 1;
- tmp_cq = tmp_ep->recv_cq;
- tmp_cntr = tmp_ep->recv_cntr;
+ case PSMX_WRITE_CONTEXT:
+ tmp_cq = tmp_ep->send_cq;
+ /* Fall through */
+ case PSMX_NOCOMP_WRITE_CONTEXT:
+ tmp_cntr = tmp_ep->write_cntr;
break;
case PSMX_READ_CONTEXT:
tmp_cq = tmp_ep->send_cq;
+ /* Fall through */
+ case PSMX_NOCOMP_READ_CONTEXT:
tmp_cntr = tmp_ep->read_cntr;
break;
- case PSMX_WRITE_CONTEXT:
- tmp_cq = tmp_ep->send_cq;
- tmp_cntr = tmp_ep->write_cntr;
- break;
-
case PSMX_REMOTE_WRITE_CONTEXT:
{
struct fi_context *fi_context = psm_status.context;
@@ -444,7 +433,9 @@ int psmx_cq_poll_mq(struct psmx_fid_cq *cq, struct psmx_fid_domain *domain,
if (event == event_buffer) {
read_count++;
read_more = --count;
- event_buffer = count ? event_buffer + cq->entry_size : NULL;
+ event_buffer = count ?
+ (uint8_t *)event_buffer + cq->entry_size :
+ NULL;
if (src_addr)
src_addr = count ? src_addr + 1 : NULL;
} else {
@@ -488,7 +479,9 @@ int psmx_cq_poll_mq(struct psmx_fid_cq *cq, struct psmx_fid_domain *domain,
if (event == event_buffer) {
read_count++;
read_more = --count;
- event_buffer = count ? event_buffer + cq->entry_size : NULL;
+ event_buffer = count ?
+ (uint8_t *)event_buffer + cq->entry_size :
+ NULL;
if (src_addr)
src_addr = count ? src_addr + 1 : NULL;
} else {
@@ -504,14 +497,18 @@ int psmx_cq_poll_mq(struct psmx_fid_cq *cq, struct psmx_fid_domain *domain,
if (multi_recv) {
struct psmx_multi_recv *req;
psm_mq_req_t psm_req;
+ size_t len_remaining;
req = PSMX_CTXT_USER(fi_context);
req->offset += psm_status.nbytes;
- if (req->offset + req->min_buf_size <= req->len) {
+ len_remaining = req->len - req->offset;
+ if (len_remaining >= req->min_buf_size) {
+ if (len_remaining > PSMX_MAX_MSG_SIZE)
+ len_remaining = PSMX_MAX_MSG_SIZE;
err = psm_mq_irecv(tmp_ep->domain->psm_mq,
req->tag, req->tagsel, req->flag,
req->buf + req->offset,
- req->len - req->offset,
+ len_remaining,
(void *)fi_context, &psm_req);
if (err != PSM_OK)
return psmx_errno(err);
@@ -543,6 +540,7 @@ static ssize_t psmx_cq_readfrom(struct fid_cq *cq, void *buf, size_t count,
struct psmx_cq_event *event;
int ret;
ssize_t read_count;
+ int i;
cq_priv = container_of(cq, struct psmx_fid_cq, cq);
@@ -563,7 +561,7 @@ static ssize_t psmx_cq_readfrom(struct fid_cq *cq, void *buf, size_t count,
return -FI_EINVAL;
read_count = 0;
- while (count--) {
+ for (i = 0; i < count; i++) {
event = psmx_cq_dequeue_event(cq_priv);
if (event) {
if (!event->error) {
@@ -574,7 +572,7 @@ static ssize_t psmx_cq_readfrom(struct fid_cq *cq, void *buf, size_t count,
psmx_cq_free_event(cq_priv, event);
read_count++;
- buf += cq_priv->entry_size;
+ buf = (uint8_t *)buf + cq_priv->entry_size;
if (src_addr)
src_addr++;
continue;
@@ -589,7 +587,14 @@ static ssize_t psmx_cq_readfrom(struct fid_cq *cq, void *buf, size_t count,
}
}
- return read_count ? read_count : -FI_EAGAIN;
+ /*
+ * Return 0 if and only if the input count is 0 and the CQ is not empty.
+ * This is used by the util poll code to check the poll state.
+ */
+ if (!read_count && (count || slist_empty(&cq_priv->event_queue)))
+ read_count = -FI_EAGAIN;
+
+ return read_count;
}
static ssize_t psmx_cq_read(struct fid_cq *cq, void *buf, size_t count)
@@ -633,7 +638,7 @@ static ssize_t psmx_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count,
event_count = cq_priv->event_count;
if (event_count < threshold) {
if (cq_priv->wait) {
- psmx_wait_wait((struct fid_wait *)cq_priv->wait, timeout);
+ fi_wait((struct fid_wait *)cq_priv->wait, timeout);
} else {
clock_gettime(CLOCK_REALTIME, &ts0);
while (1) {
@@ -672,7 +677,7 @@ static int psmx_cq_signal(struct fid_cq *cq)
cq_priv = container_of(cq, struct psmx_fid_cq, cq);
if (cq_priv->wait)
- psmx_wait_signal((struct fid_wait *)cq_priv->wait);
+ cq_priv->wait->signal(cq_priv->wait);
return 0;
}
@@ -691,7 +696,6 @@ static int psmx_cq_close(fid_t fid)
cq = container_of(fid, struct psmx_fid_cq, cq.fid);
- psmx_domain_release(cq->domain);
while (!slist_empty(&cq->free_list)) {
entry = slist_remove_head(&cq->free_list);
@@ -701,9 +705,13 @@ static int psmx_cq_close(fid_t fid)
fastlock_destroy(&cq->lock);
- if (cq->wait && cq->wait_is_local)
- fi_close((fid_t)cq->wait);
+ if (cq->wait) {
+ fi_poll_del(&cq->wait->pollset->poll_fid, &cq->cq.fid, 0);
+ if (cq->wait_is_local)
+ fi_close(&cq->wait->wait_fid.fid);
+ }
+ psmx_domain_release(cq->domain);
free(cq);
return 0;
@@ -718,10 +726,12 @@ static int psmx_cq_control(struct fid *fid, int command, void *arg)
switch (command) {
case FI_GETWAIT:
- /*
- ret = psmx_wait_get_obj(cq->wait, arg);
+ if (cq->wait)
+ ret = fi_control(&cq->wait->wait_fid.fid, FI_GETWAIT, arg);
+ else
+ return -FI_EINVAL;
break;
- */
+
default:
return -FI_ENOSYS;
}
@@ -753,7 +763,7 @@ int psmx_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
{
struct psmx_fid_domain *domain_priv;
struct psmx_fid_cq *cq_priv;
- struct psmx_fid_wait *wait = NULL;
+ struct fid_wait *wait = NULL;
struct psmx_cq_event *event;
struct fi_wait_attr wait_attr;
int wait_is_local = 0;
@@ -761,7 +771,8 @@ int psmx_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
int err;
int i;
- domain_priv = container_of(domain, struct psmx_fid_domain, domain);
+ domain_priv = container_of(domain, struct psmx_fid_domain,
+ util_domain.domain_fid);
switch (attr->format) {
case FI_CQ_FORMAT_UNSPEC:
attr->format = FI_CQ_FORMAT_TAGGED;
@@ -793,7 +804,6 @@ int psmx_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
switch (attr->wait_obj) {
case FI_WAIT_NONE:
- case FI_WAIT_UNSPEC:
break;
case FI_WAIT_SET:
@@ -802,15 +812,16 @@ int psmx_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
"FI_WAIT_SET is specified but attr->wait_set is NULL\n");
return -FI_EINVAL;
}
- wait = (struct psmx_fid_wait *)attr->wait_set;
+ wait = attr->wait_set;
break;
+ case FI_WAIT_UNSPEC:
case FI_WAIT_FD:
case FI_WAIT_MUTEX_COND:
wait_attr.wait_obj = attr->wait_obj;
wait_attr.flags = 0;
- err = psmx_wait_open(&domain_priv->fabric->fabric,
- &wait_attr, (struct fid_wait **)&wait);
+ err = fi_wait_open(&domain_priv->fabric->util_fabric.fabric_fid,
+ &wait_attr, (struct fid_wait **)&wait);
if (err)
return err;
wait_is_local = 1;
@@ -849,9 +860,10 @@ int psmx_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
cq_priv->domain = domain_priv;
cq_priv->format = attr->format;
cq_priv->entry_size = entry_size;
- cq_priv->wait = wait;
- if (wait)
+ if (wait) {
+ cq_priv->wait = container_of(wait, struct util_wait, wait_fid);
cq_priv->wait_cond = attr->wait_cond;
+ }
cq_priv->wait_is_local = wait_is_local;
cq_priv->cq.fid.fclass = FI_CLASS_CQ;
@@ -873,6 +885,9 @@ int psmx_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
slist_insert_tail(&event->list_entry, &cq_priv->free_list);
}
+ if (wait)
+ fi_poll_add(&cq_priv->wait->pollset->poll_fid, &cq_priv->cq.fid, 0);
+
*cq = &cq_priv->cq;
return 0;
}
diff --git a/prov/psm/src/psmx_domain.c b/prov/psm/src/psmx_domain.c
index a6a1308..e54fbc5 100644
--- a/prov/psm/src/psmx_domain.c
+++ b/prov/psm/src/psmx_domain.c
@@ -173,14 +173,21 @@ static void psmx_domain_stop_progress(struct psmx_fid_domain *domain)
}
}
-void psmx_domain_release(struct psmx_fid_domain *domain)
+static int psmx_domain_close(fid_t fid)
{
+ struct psmx_fid_domain *domain;
int err;
- FI_INFO(&psmx_prov, FI_LOG_DOMAIN, "refcnt=%d\n", domain->refcnt);
+ domain = container_of(fid, struct psmx_fid_domain,
+ util_domain.domain_fid.fid);
- if (--domain->refcnt > 0)
- return;
+ FI_INFO(&psmx_prov, FI_LOG_DOMAIN, "refcnt=%d\n",
+ atomic_get(&domain->util_domain.ref));
+
+ psmx_domain_release(domain);
+
+ if (ofi_domain_close(&domain->util_domain))
+ return 0;
if (domain->progress_thread_enabled)
psmx_domain_stop_progress(domain);
@@ -216,21 +223,7 @@ void psmx_domain_release(struct psmx_fid_domain *domain)
domain->fabric->active_domain = NULL;
- psmx_fabric_release(domain->fabric);
-
free(domain);
-}
-
-static int psmx_domain_close(fid_t fid)
-{
- struct psmx_fid_domain *domain;
-
- FI_INFO(&psmx_prov, FI_LOG_DOMAIN, "\n");
-
- domain = container_of(fid, struct psmx_fid_domain, domain.fid);
-
- psmx_domain_release(domain);
-
return 0;
}
@@ -249,7 +242,7 @@ static struct fi_ops_domain psmx_domain_ops = {
.endpoint = psmx_ep_open,
.scalable_ep = fi_no_scalable_ep,
.cntr_open = psmx_cntr_open,
- .poll_open = psmx_poll_open,
+ .poll_open = fi_poll_create,
.stx_ctx = psmx_stx_ctx,
.srx_ctx = fi_no_srx_context,
};
@@ -259,65 +252,31 @@ static int psmx_key_compare(void *key1, void *key2)
return (key1 < key2) ? -1 : (key1 > key2);
}
-int psmx_domain_open(struct fid_fabric *fabric, struct fi_info *info,
- struct fid_domain **domain, void *context)
+static int psmx_domain_init(struct psmx_fid_domain *domain)
{
- struct psmx_fid_fabric *fabric_priv;
- struct psmx_fid_domain *domain_priv;
+ struct psmx_fid_fabric *fabric = domain->fabric;
struct psm_ep_open_opts opts;
int err;
- FI_INFO(&psmx_prov, FI_LOG_DOMAIN, "\n");
-
- fabric_priv = container_of(fabric, struct psmx_fid_fabric, fabric);
- psmx_fabric_acquire(fabric_priv);
-
- if (fabric_priv->active_domain) {
- psmx_domain_acquire(fabric_priv->active_domain);
- *domain = &fabric_priv->active_domain->domain;
- return 0;
- }
-
- if (!info->domain_attr->name || strcmp(info->domain_attr->name, PSMX_DOMAIN_NAME)) {
- err = -FI_EINVAL;
- goto err_out;
- }
-
- domain_priv = (struct psmx_fid_domain *) calloc(1, sizeof *domain_priv);
- if (!domain_priv) {
- err = -FI_ENOMEM;
- goto err_out;
- }
-
- domain_priv->domain.fid.fclass = FI_CLASS_DOMAIN;
- domain_priv->domain.fid.context = context;
- domain_priv->domain.fid.ops = &psmx_fi_ops;
- domain_priv->domain.ops = &psmx_domain_ops;
- domain_priv->domain.mr = &psmx_mr_ops;
- domain_priv->mr_mode = info->domain_attr->mr_mode;
- domain_priv->mode = info->mode;
- domain_priv->caps = info->caps;
- domain_priv->fabric = fabric_priv;
- domain_priv->progress_thread_enabled =
- (info->domain_attr->data_progress == FI_PROGRESS_AUTO && psmx_env.prog_thread);
-
psm_ep_open_opts_get_defaults(&opts);
- FI_INFO(&psmx_prov, FI_LOG_CORE, "uuid: %s\n", psmx_uuid_to_string(fabric_priv->uuid));
+ FI_INFO(&psmx_prov, FI_LOG_CORE,
+ "uuid: %s\n", psmx_uuid_to_string(fabric->uuid));
- err = psm_ep_open(fabric_priv->uuid, &opts,
- &domain_priv->psm_ep, &domain_priv->psm_epid);
+ err = psm_ep_open(fabric->uuid, &opts,
+ &domain->psm_ep, &domain->psm_epid);
if (err != PSM_OK) {
FI_WARN(&psmx_prov, FI_LOG_CORE,
"psm_ep_open returns %d, errno=%d\n", err, errno);
err = psmx_errno(err);
- goto err_out_free_domain;
+ goto err_out;
}
- FI_INFO(&psmx_prov, FI_LOG_CORE, "epid: 0x%016lx\n", domain_priv->psm_epid);
+ FI_INFO(&psmx_prov, FI_LOG_CORE,
+ "epid: 0x%016lx\n", domain->psm_epid);
- err = psm_mq_init(domain_priv->psm_ep, PSM_MQ_ORDERMASK_ALL,
- NULL, 0, &domain_priv->psm_mq);
+ err = psm_mq_init(domain->psm_ep, PSM_MQ_ORDERMASK_ALL,
+ NULL, 0, &domain->psm_mq);
if (err != PSM_OK) {
FI_WARN(&psmx_prov, FI_LOG_CORE,
"psm_mq_init returns %d, errno=%d\n", err, errno);
@@ -325,23 +284,23 @@ int psmx_domain_open(struct fid_fabric *fabric, struct fi_info *info,
goto err_out_close_ep;
}
- err = fastlock_init(&domain_priv->mr_lock);
+ err = fastlock_init(&domain->mr_lock);
if (err) {
FI_WARN(&psmx_prov, FI_LOG_CORE,
"fastlock_init(mr_lock) returns %d\n", err);
goto err_out_finalize_mq;
}
- domain_priv->mr_map = rbtNew(&psmx_key_compare);
- if (!domain_priv->mr_map) {
+ domain->mr_map = rbtNew(&psmx_key_compare);
+ if (!domain->mr_map) {
FI_WARN(&psmx_prov, FI_LOG_CORE,
"rbtNew failed\n");
goto err_out_destroy_mr_lock;
}
- domain_priv->mr_reserved_key = 1;
+ domain->mr_reserved_key = 1;
- err = fastlock_init(&domain_priv->poll_lock);
+ err = fastlock_init(&domain->poll_lock);
if (err) {
FI_WARN(&psmx_prov, FI_LOG_CORE,
"fastlock_init(poll_lock) returns %d\n", err);
@@ -354,41 +313,100 @@ int psmx_domain_open(struct fid_fabric *fabric, struct fi_info *info,
* active_domain becomes NULL again only when the domain is closed.
* At that time the AM handlers are gone with the PSM endpoint.
*/
- fabric_priv->active_domain = domain_priv;
+ fabric->active_domain = domain;
- if (psmx_domain_enable_ep(domain_priv, NULL) < 0)
+ if (psmx_domain_enable_ep(domain, NULL) < 0)
goto err_out_reset_active_domain;
- if (domain_priv->progress_thread_enabled)
- psmx_domain_start_progress(domain_priv);
+ if (domain->progress_thread_enabled)
+ psmx_domain_start_progress(domain);
- domain_priv->refcnt = 1;
- *domain = &domain_priv->domain;
return 0;
err_out_reset_active_domain:
- fabric_priv->active_domain = NULL;
- fastlock_destroy(&domain_priv->poll_lock);
+ fabric->active_domain = NULL;
+ fastlock_destroy(&domain->poll_lock);
err_out_delete_mr_map:
- rbtDelete(domain_priv->mr_map);
+ rbtDelete(domain->mr_map);
err_out_destroy_mr_lock:
- fastlock_destroy(&domain_priv->mr_lock);
+ fastlock_destroy(&domain->mr_lock);
err_out_finalize_mq:
- psm_mq_finalize(domain_priv->psm_mq);
+ psm_mq_finalize(domain->psm_mq);
err_out_close_ep:
- if (psm_ep_close(domain_priv->psm_ep, PSM_EP_CLOSE_GRACEFUL,
+ if (psm_ep_close(domain->psm_ep, PSM_EP_CLOSE_GRACEFUL,
(int64_t) psmx_env.timeout * 1000000000LL) != PSM_OK)
- psm_ep_close(domain_priv->psm_ep, PSM_EP_CLOSE_FORCE, 0);
+ psm_ep_close(domain->psm_ep, PSM_EP_CLOSE_FORCE, 0);
+
+err_out:
+ return err;
+}
+
+int psmx_domain_open(struct fid_fabric *fabric, struct fi_info *info,
+ struct fid_domain **domain, void *context)
+{
+ struct psmx_fid_fabric *fabric_priv;
+ struct psmx_fid_domain *domain_priv;
+ int err;
+
+ FI_INFO(&psmx_prov, FI_LOG_DOMAIN, "\n");
+
+ fabric_priv = container_of(fabric, struct psmx_fid_fabric,
+ util_fabric.fabric_fid);
+
+ if (fabric_priv->active_domain) {
+ psmx_domain_acquire(fabric_priv->active_domain);
+ *domain = &fabric_priv->active_domain->util_domain.domain_fid;
+ return 0;
+ }
+
+ if (!info->domain_attr->name ||
+ strcmp(info->domain_attr->name, PSMX_DOMAIN_NAME)) {
+ err = -FI_EINVAL;
+ goto err_out;
+ }
+
+ domain_priv = (struct psmx_fid_domain *) calloc(1, sizeof *domain_priv);
+ if (!domain_priv) {
+ err = -FI_ENOMEM;
+ goto err_out;
+ }
+
+ err = ofi_domain_init(fabric, info, &domain_priv->util_domain, context);
+ if (err)
+ goto err_out_free_domain;
+
+ /* fclass & context are set in ofi_domain_init */
+ domain_priv->util_domain.domain_fid.fid.ops = &psmx_fi_ops;
+ domain_priv->util_domain.domain_fid.ops = &psmx_domain_ops;
+ domain_priv->util_domain.domain_fid.mr = &psmx_mr_ops;
+ domain_priv->mr_mode = info->domain_attr->mr_mode;
+ domain_priv->mode = info->mode;
+ domain_priv->caps = info->caps;
+ domain_priv->fabric = fabric_priv;
+ domain_priv->progress_thread_enabled =
+ (info->domain_attr->data_progress == FI_PROGRESS_AUTO && psmx_env.prog_thread);
+
+ err = psmx_domain_init(domain_priv);
+ if (err)
+ goto err_out_close_domain;
+
+ /* tale the reference to count for multiple domain open calls */
+ psmx_domain_acquire(fabric_priv->active_domain);
+
+ *domain = &domain_priv->util_domain.domain_fid;
+ return 0;
+
+err_out_close_domain:
+ ofi_domain_close(&domain_priv->util_domain);
err_out_free_domain:
free(domain_priv);
err_out:
- psmx_fabric_release(fabric_priv);
return err;
}
@@ -402,19 +420,19 @@ int psmx_domain_check_features(struct psmx_fid_domain *domain, int ep_cap)
}
if ((ep_cap & FI_TAGGED) && domain->tagged_ep &&
- fi_recv_allowed(ep_cap))
+ ofi_recv_allowed(ep_cap))
return -FI_EBUSY;
if ((ep_cap & FI_MSG) && domain->msg_ep &&
- fi_recv_allowed(ep_cap))
+ ofi_recv_allowed(ep_cap))
return -FI_EBUSY;
if ((ep_cap & FI_RMA) && domain->rma_ep &&
- fi_rma_target_allowed(ep_cap))
+ ofi_rma_target_allowed(ep_cap))
return -FI_EBUSY;
if ((ep_cap & FI_ATOMICS) && domain->atomics_ep &&
- fi_rma_target_allowed(ep_cap))
+ ofi_rma_target_allowed(ep_cap))
return -FI_EBUSY;
return 0;
@@ -452,16 +470,16 @@ int psmx_domain_enable_ep(struct psmx_fid_domain *domain, struct psmx_fid_ep *ep
domain->am_initialized = 1;
}
- if ((ep_cap & FI_RMA) && fi_rma_target_allowed(ep_cap))
+ if ((ep_cap & FI_RMA) && ofi_rma_target_allowed(ep_cap))
domain->rma_ep = ep;
- if ((ep_cap & FI_ATOMICS) && fi_rma_target_allowed(ep_cap))
+ if ((ep_cap & FI_ATOMICS) && ofi_rma_target_allowed(ep_cap))
domain->atomics_ep = ep;
- if ((ep_cap & FI_TAGGED) && fi_recv_allowed(ep_cap))
+ if ((ep_cap & FI_TAGGED) && ofi_recv_allowed(ep_cap))
domain->tagged_ep = ep;
- if ((ep_cap & FI_MSG) && fi_recv_allowed(ep_cap))
+ if ((ep_cap & FI_MSG) && ofi_recv_allowed(ep_cap))
domain->msg_ep = ep;
return 0;
diff --git a/prov/psm/src/psmx_ep.c b/prov/psm/src/psmx_ep.c
index 6718aca..278b12d 100644
--- a/prov/psm/src/psmx_ep.c
+++ b/prov/psm/src/psmx_ep.c
@@ -35,7 +35,7 @@
static void psmx_ep_optimize_ops(struct psmx_fid_ep *ep)
{
if (ep->ep.tagged) {
- if (ep->flags) {
+ if (ep->tx_flags | ep->rx_flags) {
ep->ep.tagged = &psmx_tagged_ops;
FI_INFO(&psmx_prov, FI_LOG_EP_DATA,
"generic tagged ops.\n");
@@ -174,6 +174,14 @@ static int psmx_ep_close(fid_t fid)
ep = container_of(fid, struct psmx_fid_ep, ep.fid);
+ if (ep->base_ep) {
+ atomic_dec(&ep->base_ep->ref);
+ return 0;
+ }
+
+ if (atomic_get(&ep->ref))
+ return -FI_EBUSY;
+
psmx_domain_disable_ep(ep->domain, ep);
psmx_domain_release(ep->domain);
@@ -193,9 +201,10 @@ static int psmx_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
int err;
ep = container_of(fid, struct psmx_fid_ep, ep.fid);
+ err = ofi_ep_bind_valid(&psmx_prov, bfid, flags);
+ if (err)
+ return err;
- if (!bfid)
- return -FI_EINVAL;
switch (bfid->fclass) {
case FI_CLASS_EQ:
return -FI_ENOSYS;
@@ -266,10 +275,44 @@ static int psmx_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
return 0;
}
+static inline int psmx_ep_set_flags(struct psmx_fid_ep *ep, uint64_t flags)
+{
+ uint64_t real_flags = flags & ~(FI_TRANSMIT | FI_RECV);
+
+ if ((flags & FI_TRANSMIT) && (flags & FI_RECV))
+ return -EINVAL;
+ else if (flags & FI_TRANSMIT)
+ ep->tx_flags = real_flags;
+ else if (flags & FI_RECV)
+ ep->rx_flags = real_flags;
+
+ /* otherwise ok to leave the flags intact */
+
+ return 0;
+}
+
+static inline int psmx_ep_get_flags(struct psmx_fid_ep *ep, uint64_t *flags)
+{
+ uint64_t flags_in = *flags;
+
+ if ((flags_in & FI_TRANSMIT) && (flags_in & FI_RECV))
+ return -EINVAL;
+ else if (flags_in & FI_TRANSMIT)
+ *flags = ep->tx_flags;
+ else if (flags_in & FI_RECV)
+ *flags = ep->rx_flags;
+ else
+ return -EINVAL;
+
+ return 0;
+}
+
static int psmx_ep_control(fid_t fid, int command, void *arg)
{
struct fi_alias *alias;
struct psmx_fid_ep *ep, *new_ep;
+ int err;
+
ep = container_of(fid, struct psmx_fid_ep, ep.fid);
switch (command) {
@@ -279,23 +322,34 @@ static int psmx_ep_control(fid_t fid, int command, void *arg)
return -FI_ENOMEM;
alias = arg;
*new_ep = *ep;
- new_ep->flags = alias->flags;
+ err = psmx_ep_set_flags(new_ep, alias->flags);
+ if (err) {
+ free(new_ep);
+ return err;
+ }
+ new_ep->base_ep = ep;
+ atomic_inc(&ep->ref);
psmx_ep_optimize_ops(new_ep);
*alias->fid = &new_ep->ep.fid;
break;
- case FI_SETFIDFLAG:
- ep->flags = *(uint64_t *)arg;
+ case FI_SETOPSFLAG:
+ err = psmx_ep_set_flags(ep, *(uint64_t *)arg);
+ if (err)
+ return err;
psmx_ep_optimize_ops(ep);
break;
- case FI_GETFIDFLAG:
+ case FI_GETOPSFLAG:
if (!arg)
return -FI_EINVAL;
- *(uint64_t *)arg = ep->flags;
+ err = psmx_ep_get_flags(ep, arg);
+ if (err)
+ return err;
break;
case FI_ENABLE:
+ ep->enabled = 1;
return 0;
default:
@@ -307,12 +361,24 @@ static int psmx_ep_control(fid_t fid, int command, void *arg)
static ssize_t psmx_rx_size_left(struct fid_ep *ep)
{
- return 0x7fffffff; /* a random choice */
+ struct psmx_fid_ep *ep_priv;
+
+ ep_priv = container_of(ep, struct psmx_fid_ep, ep);
+ if (ep_priv->enabled)
+ return 0x7fffffff;
+ else
+ return -FI_EOPBADSTATE;
}
static ssize_t psmx_tx_size_left(struct fid_ep *ep)
{
- return 0x7fffffff; /* a random choice */
+ struct psmx_fid_ep *ep_priv;
+
+ ep_priv = container_of(ep, struct psmx_fid_ep, ep);
+ if (ep_priv->enabled)
+ return 0x7fffffff;
+ else
+ return -FI_EOPBADSTATE;
}
static struct fi_ops psmx_fi_ops = {
@@ -347,7 +413,8 @@ int psmx_ep_open(struct fid_domain *domain, struct fi_info *info,
else
ep_cap = FI_TAGGED;
- domain_priv = container_of(domain, struct psmx_fid_domain, domain.fid);
+ domain_priv = container_of(domain, struct psmx_fid_domain,
+ util_domain.domain_fid.fid);
if (!domain_priv)
return -FI_EINVAL;
@@ -365,6 +432,7 @@ int psmx_ep_open(struct fid_domain *domain, struct fi_info *info,
ep_priv->ep.ops = &psmx_ep_ops;
ep_priv->ep.cm = &psmx_cm_ops;
ep_priv->domain = domain_priv;
+ atomic_initialize(&ep_priv->ref, 0);
PSMX_CTXT_TYPE(&ep_priv->nocomp_send_context) = PSMX_NOCOMP_SEND_CONTEXT;
PSMX_CTXT_EP(&ep_priv->nocomp_send_context) = ep_priv;
@@ -394,9 +462,9 @@ int psmx_ep_open(struct fid_domain *domain, struct fi_info *info,
if (info) {
if (info->tx_attr)
- ep_priv->flags = info->tx_attr->op_flags;
+ ep_priv->tx_flags = info->tx_attr->op_flags;
if (info->rx_attr)
- ep_priv->flags |= info->rx_attr->op_flags;
+ ep_priv->rx_flags = info->rx_attr->op_flags;
}
psmx_ep_optimize_ops(ep_priv);
@@ -436,7 +504,8 @@ int psmx_stx_ctx(struct fid_domain *domain, struct fi_tx_attr *attr,
FI_INFO(&psmx_prov, FI_LOG_EP_DATA, "\n");
- domain_priv = container_of(domain, struct psmx_fid_domain, domain.fid);
+ domain_priv = container_of(domain, struct psmx_fid_domain,
+ util_domain.domain_fid.fid);
stx_priv = (struct psmx_fid_stx *) calloc(1, sizeof *stx_priv);
if (!stx_priv)
diff --git a/prov/psm/src/psmx_eq.c b/prov/psm/src/psmx_eq.c
deleted file mode 100644
index b67b112..0000000
--- a/prov/psm/src/psmx_eq.c
+++ /dev/null
@@ -1,444 +0,0 @@
-/*
- * Copyright (c) 2013-2014 Intel Corporation. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "psmx.h"
-
-void psmx_eq_enqueue_event(struct psmx_fid_eq *eq, struct psmx_eq_event *event)
-{
- fastlock_acquire(&eq->lock);
- if (event->error)
- slist_insert_tail(&event->list_entry, &eq->error_queue);
- else
- slist_insert_tail(&event->list_entry, &eq->event_queue);
- fastlock_release(&eq->lock);
-
- if (eq->wait)
- psmx_wait_signal((struct fid_wait *)eq->wait);
-}
-
-static struct psmx_eq_event *psmx_eq_dequeue_event(struct psmx_fid_eq *eq)
-{
- struct slist_entry *entry;
-
- if (slist_empty(&eq->event_queue))
- return NULL;
-
- fastlock_acquire(&eq->lock);
- entry = slist_remove_head(&eq->event_queue);
- fastlock_release(&eq->lock);
-
- return container_of(entry, struct psmx_eq_event, list_entry);
-}
-
-static struct psmx_eq_event *psmx_eq_dequeue_error(struct psmx_fid_eq *eq)
-{
- struct slist_entry *entry;
-
- if (slist_empty(&eq->error_queue))
- return NULL;
-
- fastlock_acquire(&eq->lock);
- entry = slist_remove_head(&eq->error_queue);
- fastlock_release(&eq->lock);
-
- return container_of(entry, struct psmx_eq_event, list_entry);
-}
-
-static struct psmx_eq_event *psmx_eq_peek_event(struct psmx_fid_eq *eq)
-{
- if (slist_empty(&eq->event_queue))
- return NULL;
-
- return container_of(eq->event_queue.head, struct psmx_eq_event, list_entry);
-}
-
-static struct psmx_eq_event *psmx_eq_alloc_event(struct psmx_fid_eq *eq)
-{
- struct psmx_eq_event *event;
-
- fastlock_acquire(&eq->lock);
- if (!slist_empty(&eq->free_list)) {
- event = container_of(slist_remove_head(&eq->free_list),
- struct psmx_eq_event, list_entry);
- fastlock_release(&eq->lock);
- return event;
- }
-
- fastlock_release(&eq->lock);
- event = calloc(1, sizeof(*event));
- if (!event)
- FI_WARN(&psmx_prov, FI_LOG_EQ, "out of memory.\n");
-
- return event;
-}
-
-static void psmx_eq_free_event(struct psmx_fid_eq *eq, struct psmx_eq_event *event)
-{
- memset(event, 0, sizeof(*event));
-
- fastlock_acquire(&eq->lock);
- slist_insert_tail(&event->list_entry, &eq->free_list);
- fastlock_release(&eq->lock);
-}
-
-struct psmx_eq_event *psmx_eq_create_event(struct psmx_fid_eq *eq,
- uint32_t event_num,
- void *context,
- uint64_t data,
- int err,
- int prov_errno,
- void *err_data,
- size_t err_data_size)
-{
- struct psmx_eq_event *event;
-
- event = psmx_eq_alloc_event(eq);
- if (!event)
- return NULL;
-
- event->event = event_num;
- event->eqe.data.fid = &eq->fabric->fabric.fid;
-
- if ((event->error = !!err)) {
- event->eqe.err.context = context;
- event->eqe.err.data = data;
- event->eqe.err.err = -err;
- event->eqe.err.prov_errno = prov_errno;
- event->eqe.err.err_data = err_data;
- event->eqe.err.err_data_size = err_data_size;
- event->entry_size = sizeof(event->eqe.err);
- goto out;
- }
-
- switch (event_num) {
- case FI_NOTIFY:
- event->eqe.data.context = context;
- event->eqe.data.data = data;
- event->entry_size = sizeof(event->eqe.data);
- break;
-
- case FI_CONNREQ:
- case FI_CONNECTED:
- case FI_SHUTDOWN:
- event->eqe.cm.info = context;
- /* TODO: store event->eqe.cm.data (upto sizeof(eqe)-sizeof(eqe.cm) bytes) */
- event->entry_size = sizeof(event->eqe.cm) + 0; /* TODO: extra data size */
- break;
-
- case FI_MR_COMPLETE:
- case FI_AV_COMPLETE:
- event->eqe.data.context = context;
- event->eqe.data.data = data;
- event->entry_size = sizeof(event->eqe.data);
- break;
-
- default:
- psmx_eq_free_event(eq, event);
- FI_WARN(&psmx_prov, FI_LOG_EQ, "invalid event %d.\n", event_num);
- event = NULL;
- break;
- }
-
-out:
- return event;
-}
-
-static ssize_t psmx_eq_read(struct fid_eq *eq, uint32_t *event_num, void *buf,
- size_t len, uint64_t flags)
-{
- struct psmx_fid_eq *eq_priv;
- struct psmx_eq_event *event;
- ssize_t bytes_read = 0;
-
- eq_priv = container_of(eq, struct psmx_fid_eq, eq);
-
- if (slist_empty(&eq_priv->event_queue))
- psmx_progress(eq_priv->fabric->active_domain);
-
- if (!slist_empty(&eq_priv->error_queue))
- return -FI_EAVAIL;
-
- if (flags & FI_PEEK)
- event = psmx_eq_peek_event(eq_priv);
- else
- event = psmx_eq_dequeue_event(eq_priv);
-
- if (!event)
- return -FI_EAGAIN;
-
- *event_num = event->event;
- if (buf) {
- bytes_read = MIN(len, event->entry_size);
- memcpy(buf, (void *)&event->eqe, bytes_read);
- }
-
- if (!(flags & FI_PEEK))
- psmx_eq_free_event(eq_priv, event);
-
- return bytes_read;
-}
-
-static ssize_t psmx_eq_readerr(struct fid_eq *eq, struct fi_eq_err_entry *buf,
- uint64_t flags)
-{
- struct psmx_fid_eq *eq_priv;
- struct psmx_eq_event *event;
- ssize_t bytes_read = 0;
-
- eq_priv = container_of(eq, struct psmx_fid_eq, eq);
-
- event = psmx_eq_dequeue_error(eq_priv);
- if (!event)
- return -FI_EAGAIN;
-
- if (buf) {
- bytes_read = sizeof(*buf);
- memcpy(buf, &event->eqe.err, bytes_read);
- psmx_eq_free_event(eq_priv, event);
- }
-
- return bytes_read;
-}
-
-static ssize_t psmx_eq_sread(struct fid_eq *eq, uint32_t *event, void *buf,
- size_t len, int timeout, uint64_t flags)
-{
- struct psmx_fid_eq *eq_priv;
- struct timespec ts0, ts;
- int msec_passed = 0;
- ssize_t ret = -FI_EAGAIN;
-
- eq_priv = container_of(eq, struct psmx_fid_eq, eq);
-
- if (eq_priv->wait) {
- psmx_wait_wait((struct fid_wait *)eq_priv->wait, timeout);
- ret = psmx_eq_read(eq, event, buf, len, flags);
- } else {
- clock_gettime(CLOCK_REALTIME, &ts0);
- while (1) {
- ret = psmx_eq_read(eq, event, buf, len, flags);
- if (ret != -FI_EAGAIN)
- break;
-
- if (timeout < 0)
- continue;
-
- clock_gettime(CLOCK_REALTIME, &ts);
- msec_passed = (ts.tv_sec - ts0.tv_sec) * 1000 +
- (ts.tv_nsec - ts0.tv_nsec) / 1000000;
-
- if (msec_passed >= timeout)
- break;
- }
- }
-
- return ret;
-}
-
-static ssize_t psmx_eq_write(struct fid_eq *eq, uint32_t event_num, const void *buf,
- size_t len, uint64_t flags)
-{
- struct psmx_fid_eq *eq_priv;
- struct psmx_eq_event *event;
- ssize_t bytes_written;
-
- eq_priv = container_of(eq, struct psmx_fid_eq, eq);
-
- event = psmx_eq_alloc_event(eq_priv);
- if (!event)
- return -FI_ENOMEM;
-
- bytes_written = MIN(len, sizeof(event->eqe));
-
- event->event = event_num;
- event->entry_size = bytes_written;
- memcpy(&event->eqe, buf, bytes_written);
-
- psmx_eq_enqueue_event(eq_priv, event);
-
- if (eq_priv->wait)
- psmx_wait_signal((struct fid_wait *)eq_priv->wait);
-
- return bytes_written;
-}
-
-static const char *psmx_eq_strerror(struct fid_eq *eq, int prov_errno,
- const void *err_data, char *buf, size_t len)
-{
- return psm_error_get_string(prov_errno);
-}
-
-static int psmx_eq_close(fid_t fid)
-{
- struct psmx_fid_eq *eq;
- struct slist_entry *entry;
- struct psmx_eq_event *item;
-
- eq = container_of(fid, struct psmx_fid_eq, eq.fid);
-
- psmx_fabric_release(eq->fabric);
-
- while (!slist_empty(&eq->free_list)) {
- entry = slist_remove_head(&eq->free_list);
- item = container_of(entry, struct psmx_eq_event, list_entry);
- free(item);
- }
-
- fastlock_destroy(&eq->lock);
-
- if (eq->wait && eq->wait_is_local)
- fi_close((fid_t)eq->wait);
-
- free(eq);
-
- return 0;
-}
-
-static int psmx_eq_control(struct fid *fid, int command, void *arg)
-{
- struct psmx_fid_eq *eq;
- int ret = 0;
-
- eq = container_of(fid, struct psmx_fid_eq, eq.fid);
-
- switch (command) {
- case FI_GETWAIT:
- /*
- ret = psmx_wait_get_obj(eq->wait, arg);
- break;
- */
- default:
- return -FI_ENOSYS;
- }
-
- return ret;
-}
-
-static struct fi_ops psmx_fi_ops = {
- .size = sizeof(struct fi_ops),
- .close = psmx_eq_close,
- .bind = fi_no_bind,
- .control = psmx_eq_control,
- .ops_open = fi_no_ops_open,
-};
-
-static struct fi_ops_eq psmx_eq_ops = {
- .size = sizeof(struct fi_ops_eq),
- .read = psmx_eq_read,
- .readerr = psmx_eq_readerr,
- .sread = psmx_eq_sread,
- .write = psmx_eq_write,
- .strerror = psmx_eq_strerror,
-};
-
-int psmx_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr,
- struct fid_eq **eq, void *context)
-{
- struct psmx_fid_fabric *fabric_priv;
- struct psmx_fid_eq *eq_priv;
- struct psmx_fid_wait *wait = NULL;
- struct psmx_eq_event *event;
- struct fi_wait_attr wait_attr;
- int wait_is_local = 0;
- int err;
- int i;
-
- fabric_priv = container_of(fabric, struct psmx_fid_fabric, fabric);
-
- switch (attr->wait_obj) {
- case FI_WAIT_NONE:
- case FI_WAIT_UNSPEC:
- break;
-
- case FI_WAIT_SET:
- if (!attr->wait_set) {
- FI_INFO(&psmx_prov, FI_LOG_EQ,
- "FI_WAIT_SET is specified but attr->wait_set is NULL\n");
- return -FI_EINVAL;
- }
- wait = (struct psmx_fid_wait *)attr->wait_set;
- break;
-
- case FI_WAIT_FD:
- case FI_WAIT_MUTEX_COND:
- wait_attr.wait_obj = attr->wait_obj;
- wait_attr.flags = 0;
- err = psmx_wait_open(fabric, &wait_attr, (struct fid_wait **)&wait);
- if (err)
- return err;
- wait_is_local = 1;
- break;
-
- default:
- FI_INFO(&psmx_prov, FI_LOG_EQ,
- "attr->wait_obj=%d, supported=%d...%d\n", attr->wait_obj,
- FI_WAIT_NONE, FI_WAIT_MUTEX_COND);
- return -FI_EINVAL;
- }
-
- eq_priv = (struct psmx_fid_eq *) calloc(1, sizeof *eq_priv);
- if (!eq_priv) {
- if (wait)
- free(wait);
- return -FI_ENOMEM;
- }
-
- psmx_fabric_acquire(fabric_priv);
-
- eq_priv->fabric = fabric_priv;
- eq_priv->wait = wait;
- eq_priv->wait_is_local = wait_is_local;
-
- eq_priv->eq.fid.fclass = FI_CLASS_EQ;
- eq_priv->eq.fid.context = context;
- eq_priv->eq.fid.ops = &psmx_fi_ops;
- eq_priv->eq.ops = &psmx_eq_ops;
-
- slist_init(&eq_priv->event_queue);
- slist_init(&eq_priv->error_queue);
- slist_init(&eq_priv->free_list);
- fastlock_init(&eq_priv->lock);
-
-#define PSMX_FREE_LIST_SIZE 64
- for (i=0; i<PSMX_FREE_LIST_SIZE; i++) {
- event = calloc(1, sizeof(*event));
- if (!event) {
- FI_WARN(&psmx_prov, FI_LOG_EQ, "out of memory.\n");
- exit(-1);
- }
- slist_insert_tail(&event->list_entry, &eq_priv->free_list);
- }
-
- *eq = &eq_priv->eq;
- return 0;
-}
-
diff --git a/prov/psm/src/psmx_fabric.c b/prov/psm/src/psmx_fabric.c
index ed4625b..9dba71b 100644
--- a/prov/psm/src/psmx_fabric.c
+++ b/prov/psm/src/psmx_fabric.c
@@ -34,15 +34,22 @@
struct psmx_fid_fabric *psmx_active_fabric = NULL;
-void psmx_fabric_release(struct psmx_fid_fabric *fabric)
+static int psmx_fabric_close(fid_t fid)
{
+ struct psmx_fid_fabric *fabric;
void *exit_code;
int ret;
- FI_INFO(&psmx_prov, FI_LOG_CORE, "refcnt=%d\n", fabric->refcnt);
+ fabric = container_of(fid, struct psmx_fid_fabric,
+ util_fabric.fabric_fid.fid);
+
+ FI_INFO(&psmx_prov, FI_LOG_CORE, "refcnt=%d\n",
+ atomic_get(&fabric->util_fabric.ref));
- if (--fabric->refcnt)
- return;
+ psmx_fabric_release(fabric);
+
+ if (ofi_fabric_close(&fabric->util_fabric))
+ return 0;
if (psmx_env.name_server &&
!pthread_equal(fabric->name_server_thread, pthread_self())) {
@@ -64,22 +71,11 @@ void psmx_fabric_release(struct psmx_fid_fabric *fabric)
}
if (fabric->active_domain) {
FI_WARN(&psmx_prov, FI_LOG_CORE, "forced closing of active_domain\n");
- fi_close(&fabric->active_domain->domain.fid);
+ fi_close(&fabric->active_domain->util_domain.domain_fid.fid);
}
assert(fabric == psmx_active_fabric);
psmx_active_fabric = NULL;
free(fabric);
-}
-
-static int psmx_fabric_close(fid_t fid)
-{
- struct psmx_fid_fabric *fabric;
-
- FI_INFO(&psmx_prov, FI_LOG_CORE, "\n");
-
- fabric = container_of(fid, struct psmx_fid_fabric, fabric.fid);
-
- psmx_fabric_release(fabric);
return 0;
}
@@ -93,9 +89,14 @@ static struct fi_ops_fabric psmx_fabric_ops = {
.size = sizeof(struct fi_ops_fabric),
.domain = psmx_domain_open,
.passive_ep = fi_no_passive_ep,
- .eq_open = psmx_eq_open,
+ .eq_open = ofi_eq_create,
.wait_open = psmx_wait_open,
- .trywait = fi_no_trywait
+ .trywait = psmx_wait_trywait,
+};
+
+static struct fi_fabric_attr psmx_fabric_attr = {
+ .name = PSMX_FABRIC_NAME,
+ .prov_version = PSMX_VERSION,
};
int psmx_fabric(struct fi_fabric_attr *attr,
@@ -111,7 +112,7 @@ int psmx_fabric(struct fi_fabric_attr *attr,
if (psmx_active_fabric) {
psmx_fabric_acquire(psmx_active_fabric);
- *fabric = &psmx_active_fabric->fabric;
+ *fabric = &psmx_active_fabric->util_fabric.fabric_fid;
return 0;
}
@@ -119,10 +120,18 @@ int psmx_fabric(struct fi_fabric_attr *attr,
if (!fabric_priv)
return -FI_ENOMEM;
- fabric_priv->fabric.fid.fclass = FI_CLASS_FABRIC;
- fabric_priv->fabric.fid.context = context;
- fabric_priv->fabric.fid.ops = &psmx_fabric_fi_ops;
- fabric_priv->fabric.ops = &psmx_fabric_ops;
+ ret = ofi_fabric_init(&psmx_prov, &psmx_fabric_attr, attr,
+ &fabric_priv->util_fabric, context,
+ FI_MATCH_EXACT);
+ if (ret) {
+ FI_INFO(&psmx_prov, FI_LOG_CORE, "ofi_fabric_init returns %d\n", ret);
+ free(fabric_priv);
+ return ret;
+ }
+
+ /* fclass & context initialzied in ofi_fabric_init */
+ fabric_priv->util_fabric.fabric_fid.fid.ops = &psmx_fabric_fi_ops;
+ fabric_priv->util_fabric.fabric_fid.ops = &psmx_fabric_ops;
psmx_get_uuid(fabric_priv->uuid);
@@ -138,8 +147,10 @@ int psmx_fabric(struct fi_fabric_attr *attr,
psmx_query_mpi();
- fabric_priv->refcnt = 1;
- *fabric = &fabric_priv->fabric;
+ /* take the reference to count for multiple fabric open calls */
+ psmx_fabric_acquire(fabric_priv);
+
+ *fabric = &fabric_priv->util_fabric.fabric_fid;
psmx_active_fabric = fabric_priv;
return 0;
diff --git a/prov/psm/src/psmx_init.c b/prov/psm/src/psmx_init.c
index 5088401..21ffcdc 100644
--- a/prov/psm/src/psmx_init.c
+++ b/prov/psm/src/psmx_init.c
@@ -149,8 +149,18 @@ static int psmx_getinfo(uint32_t version, const char *node, const char *service,
psmx_init_env();
- if (node && !(flags & FI_SOURCE))
+ if (node && !(flags & FI_SOURCE)) {
dest_addr = psmx_resolve_name(node, 0);
+ if (dest_addr) {
+ FI_INFO(&psmx_prov, FI_LOG_CORE,
+ "node '%s' resolved to <epid=0x%llx>\n", node,
+ *(psm_epid_t *)dest_addr);
+ } else {
+ FI_INFO(&psmx_prov, FI_LOG_CORE,
+ "failed to resolve node '%s'.\n", node);
+ return -FI_ENODATA;
+ }
+ }
if (hints) {
switch (hints->addr_format) {
@@ -467,7 +477,7 @@ static int psmx_getinfo(uint32_t version, const char *node, const char *service,
psmx_info->domain_attr->ep_cnt = 65535;
psmx_info->domain_attr->tx_ctx_cnt = 1;
psmx_info->domain_attr->rx_ctx_cnt = 1;
- psmx_info->domain_attr->max_ep_tx_ctx = 65535;
+ psmx_info->domain_attr->max_ep_tx_ctx = 1;
psmx_info->domain_attr->max_ep_rx_ctx = 1;
psmx_info->domain_attr->max_ep_stx_ctx = 65535;
psmx_info->domain_attr->max_ep_srx_ctx = 0;
@@ -482,6 +492,7 @@ static int psmx_getinfo(uint32_t version, const char *node, const char *service,
psmx_info->dest_addr = dest_addr;
psmx_info->fabric_attr->name = strdup(PSMX_FABRIC_NAME);
psmx_info->fabric_attr->prov_name = NULL;
+ psmx_info->fabric_attr->prov_version = PSMX_VERSION;
psmx_info->tx_attr->caps = psmx_info->caps;
psmx_info->tx_attr->mode = psmx_info->mode;
diff --git a/prov/psm/src/psmx_mr.c b/prov/psm/src/psmx_mr.c
index 95f5075..4cd69d6 100644
--- a/prov/psm/src/psmx_mr.c
+++ b/prov/psm/src/psmx_mr.c
@@ -35,13 +35,16 @@
struct psmx_fid_mr *psmx_mr_get(struct psmx_fid_domain *domain, uint64_t key)
{
RbtIterator it;
- struct psmx_fid_mr *mr;
+ struct psmx_fid_mr *mr = NULL;
+ fastlock_acquire(&domain->mr_lock);
it = rbtFind(domain->mr_map, (void *)key);
if (!it)
- return NULL;
+ goto exit;
rbtKeyValue(domain->mr_map, it, (void **)&key, (void **)&mr);
+exit:
+ fastlock_release(&domain->mr_lock);
return mr;
}
@@ -171,6 +174,7 @@ static void psmx_mr_normalize_iov(struct iovec *iov, size_t *count)
{
struct iovec tmp_iov;
int i, j, n, new_len;
+ uintptr_t iov_end_i, iov_end_j;
n = *count;
@@ -197,8 +201,10 @@ static void psmx_mr_normalize_iov(struct iovec *iov, size_t *count)
if (iov[j].iov_len == 0)
continue;
- if (iov[i].iov_base + iov[i].iov_len >= iov[j].iov_base) {
- new_len = iov[j].iov_base + iov[j].iov_len - iov[i].iov_base;
+ iov_end_i = (uintptr_t)iov[i].iov_base + iov[i].iov_len;
+ iov_end_j = (uintptr_t)iov[j].iov_base + iov[j].iov_len;
+ if (iov_end_i >= (uintptr_t)iov[j].iov_base) {
+ new_len = iov_end_j - (uintptr_t)iov[i].iov_base;
if (new_len > iov[i].iov_len)
iov[i].iov_len = new_len;
iov[j].iov_len = 0;
@@ -240,7 +246,8 @@ static int psmx_mr_reg(struct fid *fid, const void *buf, size_t len,
return -FI_EINVAL;
}
domain = container_of(fid, struct fid_domain, fid);
- domain_priv = container_of(domain, struct psmx_fid_domain, domain);
+ domain_priv = container_of(domain, struct psmx_fid_domain,
+ util_domain.domain_fid);
mr_priv = (struct psmx_fid_mr *) calloc(1, sizeof(*mr_priv) + sizeof(struct iovec));
if (!mr_priv)
@@ -258,10 +265,6 @@ static int psmx_mr_reg(struct fid *fid, const void *buf, size_t len,
mr_priv->mr.fid.context = context;
mr_priv->mr.fid.ops = &psmx_fi_ops;
mr_priv->mr.mem_desc = mr_priv;
- mr_priv->mr.fid.fclass = FI_CLASS_MR;
- mr_priv->mr.fid.context = context;
- mr_priv->mr.fid.ops = &psmx_fi_ops;
- mr_priv->mr.mem_desc = mr_priv;
mr_priv->mr.key = key;
mr_priv->domain = domain_priv;
mr_priv->access = access;
@@ -292,7 +295,8 @@ static int psmx_mr_regv(struct fid *fid,
return -FI_EINVAL;
}
domain = container_of(fid, struct fid_domain, fid);
- domain_priv = container_of(domain, struct psmx_fid_domain, domain);
+ domain_priv = container_of(domain, struct psmx_fid_domain,
+ util_domain.domain_fid);
if (count == 0 || iov == NULL)
return -FI_EINVAL;
@@ -344,7 +348,8 @@ static int psmx_mr_regattr(struct fid *fid, const struct fi_mr_attr *attr,
return -FI_EINVAL;
}
domain = container_of(fid, struct fid_domain, fid);
- domain_priv = container_of(domain, struct psmx_fid_domain, domain);
+ domain_priv = container_of(domain, struct psmx_fid_domain,
+ util_domain.domain_fid);
if (!attr)
return -FI_EINVAL;
diff --git a/prov/psm/src/psmx_msg.c b/prov/psm/src/psmx_msg.c
index b0f4196..5c89eed 100644
--- a/prov/psm/src/psmx_msg.c
+++ b/prov/psm/src/psmx_msg.c
@@ -114,6 +114,8 @@ ssize_t _psmx_recv(struct fid_ep *ep, void *buf, size_t len,
req->context = fi_context;
PSMX_CTXT_TYPE(fi_context) = PSMX_MULTI_RECV_CONTEXT;
PSMX_CTXT_USER(fi_context) = req;
+ if (len > PSMX_MAX_MSG_SIZE)
+ len = PSMX_MAX_MSG_SIZE;
} else {
PSMX_CTXT_TYPE(fi_context) = PSMX_RECV_CONTEXT;
PSMX_CTXT_USER(fi_context) = buf;
@@ -140,7 +142,7 @@ static ssize_t psmx_recv(struct fid_ep *ep, void *buf, size_t len, void *desc,
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
- return _psmx_recv(ep, buf, len, desc, src_addr, context, ep_priv->flags);
+ return _psmx_recv(ep, buf, len, desc, src_addr, context, ep_priv->rx_flags);
}
static ssize_t psmx_recvmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags)
@@ -152,7 +154,7 @@ static ssize_t psmx_recvmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_
return -FI_EINVAL;
if (msg->iov_count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (msg->iov_count) {
buf = msg->msg_iov[0].iov_base;
len = msg->msg_iov[0].iov_len;
@@ -176,7 +178,7 @@ static ssize_t psmx_recvv(struct fid_ep *ep, const struct iovec *iov, void **des
return -FI_EINVAL;
if (count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -308,7 +310,7 @@ static ssize_t psmx_send(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
- return _psmx_send(ep, buf, len, desc, dest_addr, context, ep_priv->flags);
+ return _psmx_send(ep, buf, len, desc, dest_addr, context, ep_priv->tx_flags);
}
static ssize_t psmx_sendmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags)
@@ -320,7 +322,7 @@ static ssize_t psmx_sendmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_
return -FI_EINVAL;
if (msg->iov_count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (msg->iov_count) {
buf = msg->msg_iov[0].iov_base;
len = msg->msg_iov[0].iov_len;
@@ -344,7 +346,7 @@ static ssize_t psmx_sendv(struct fid_ep *ep, const struct iovec *iov, void **des
return -FI_EINVAL;
if (count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -364,7 +366,7 @@ static ssize_t psmx_inject(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
return _psmx_send(ep, buf, len, NULL, dest_addr, NULL,
- ep_priv->flags | FI_INJECT | PSMX_NO_COMPLETION);
+ ep_priv->tx_flags | FI_INJECT | PSMX_NO_COMPLETION);
}
struct fi_ops_msg psmx_msg_ops = {
diff --git a/prov/psm/src/psmx_msg2.c b/prov/psm/src/psmx_msg2.c
index fe6684e..57a4d8d 100644
--- a/prov/psm/src/psmx_msg2.c
+++ b/prov/psm/src/psmx_msg2.c
@@ -330,7 +330,7 @@ int psmx_am_process_send(struct psmx_fid_domain *domain, struct psmx_am_request
req->send.len_sent = offset + len;
err = psm_am_request_short((psm_epaddr_t) req->send.dest_addr,
PSMX_AM_MSG_HANDLER, args, 4,
- (void *)req->send.buf+offset, len,
+ req->send.buf+offset, len,
am_flags, NULL, NULL);
return psmx_errno(err);
@@ -440,7 +440,7 @@ static ssize_t psmx_recv2(struct fid_ep *ep, void *buf, size_t len,
struct psmx_fid_ep *ep_priv;
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
- return _psmx_recv2(ep, buf, len, desc, src_addr, context, ep_priv->flags);
+ return _psmx_recv2(ep, buf, len, desc, src_addr, context, ep_priv->rx_flags);
}
static ssize_t psmx_recvmsg2(struct fid_ep *ep, const struct fi_msg *msg,
@@ -453,7 +453,7 @@ static ssize_t psmx_recvmsg2(struct fid_ep *ep, const struct fi_msg *msg,
return -FI_EINVAL;
if (msg->iov_count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (msg->iov_count) {
buf = msg->msg_iov[0].iov_base;
len = msg->msg_iov[0].iov_len;
@@ -477,7 +477,7 @@ static ssize_t psmx_recvv2(struct fid_ep *ep, const struct iovec *iov,
return -FI_EINVAL;
if (count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -559,7 +559,7 @@ static ssize_t psmx_send2(struct fid_ep *ep, const void *buf,
struct psmx_fid_ep *ep_priv;
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
- return _psmx_send2(ep, buf, len, desc, dest_addr, context, ep_priv->flags);
+ return _psmx_send2(ep, buf, len, desc, dest_addr, context, ep_priv->tx_flags);
}
static ssize_t psmx_sendmsg2(struct fid_ep *ep, const struct fi_msg *msg,
@@ -572,7 +572,7 @@ static ssize_t psmx_sendmsg2(struct fid_ep *ep, const struct fi_msg *msg,
return -FI_EINVAL;
if (msg->iov_count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (msg->iov_count) {
buf = msg->msg_iov[0].iov_base;
len = msg->msg_iov[0].iov_len;
@@ -596,7 +596,7 @@ static ssize_t psmx_sendv2(struct fid_ep *ep, const struct iovec *iov,
return -FI_EINVAL;
if (count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -617,7 +617,7 @@ static ssize_t psmx_inject2(struct fid_ep *ep, const void *buf, size_t len,
/* TODO: optimize it & guarantee buffered */
return _psmx_send2(ep, buf, len, NULL, dest_addr, NULL,
- ep_priv->flags | FI_INJECT | PSMX_NO_COMPLETION);
+ ep_priv->tx_flags | FI_INJECT | PSMX_NO_COMPLETION);
}
struct fi_ops_msg psmx_msg2_ops = {
diff --git a/prov/psm/src/psmx_poll.c b/prov/psm/src/psmx_poll.c
deleted file mode 100644
index d158122..0000000
--- a/prov/psm/src/psmx_poll.c
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Copyright (c) 2013-2014 Intel Corporation. All rights reserved.
- *
- * This software is waitailable to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, waitailable from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "psmx.h"
-
-int psmx_poll_add(struct fid_poll *pollset, struct fid *event_fid, uint64_t flags)
-{
-
- struct psmx_fid_poll *poll_priv;
- struct psmx_poll_list *list_item;
-
- poll_priv = container_of(pollset, struct psmx_fid_poll, poll.fid);
-
- list_item = calloc(1, sizeof(*list_item));
- if (!list_item)
- return -FI_ENOMEM;
-
- list_item->fid = event_fid;
- dlist_insert_after(&list_item->entry, &poll_priv->poll_list_head);
-
- return 0;
-}
-
-int psmx_poll_del(struct fid_poll *pollset, struct fid *event_fid, uint64_t flags)
-{
- struct psmx_fid_poll *poll_priv;
- struct psmx_poll_list *list_item;
- struct dlist_entry *p, *head;
-
- poll_priv = container_of(pollset, struct psmx_fid_poll, poll.fid);
-
- head = &poll_priv->poll_list_head;
- for (p = head->next; p != head; p = p->next) {
- list_item = container_of(p, struct psmx_poll_list, entry);
- if (list_item->fid == event_fid) {
- dlist_remove(p);
- free(list_item);
- break;
- }
- }
-
- return 0;
-}
-
-static int psmx_poll_poll(struct fid_poll *pollset, void **context, int count)
-{
- struct psmx_fid_poll *poll_priv;
- struct psmx_fid_cq *cq;
- struct psmx_fid_cntr *cntr;
- struct psmx_poll_list *list_item;
- struct dlist_entry *p, *head;
- int ret_count = 0;
-
- poll_priv = container_of(pollset, struct psmx_fid_poll, poll.fid);
-
- psmx_progress(poll_priv->domain);
-
- head = &poll_priv->poll_list_head;
- for (p = head->next; p != head && ret_count < count; p = p->next) {
- list_item = container_of(p, struct psmx_poll_list, entry);
- switch (list_item->fid->fclass) {
- case FI_CLASS_CQ:
- cq = container_of(list_item->fid, struct psmx_fid_cq, cq);
- if (cq->event_count) {
- *context++ = cq->cq.fid.context;
- ret_count++;
- }
- break;
-
- case FI_CLASS_CNTR:
- cntr = container_of(list_item->fid, struct psmx_fid_cntr, cntr);
- if (cntr->counter != cntr->counter_last_read) {
- *context++ = cntr->cntr.fid.context;
- ret_count++;
- }
- break;
-
- default:
- break;
- }
- }
-
- return ret_count;
-}
-
-static int psmx_poll_close(fid_t fid)
-{
- struct psmx_fid_poll *poll;
- struct psmx_poll_list *list_item;
- struct dlist_entry *p, *head;
-
- poll = container_of(fid, struct psmx_fid_poll, poll.fid);
-
- psmx_domain_release(poll->domain);
-
- head = &poll->poll_list_head;
- while (!dlist_empty(head)) {
- p = head->next;
- list_item = container_of(p, struct psmx_poll_list, entry);
- dlist_remove(p);
- free(list_item);
- }
-
- free(poll);
- return 0;
-}
-
-static struct fi_ops psmx_fi_ops = {
- .size = sizeof(struct fi_ops),
- .close = psmx_poll_close,
- .bind = fi_no_bind,
- .control = fi_no_control,
- .ops_open = fi_no_ops_open,
-};
-
-static struct fi_ops_poll psmx_poll_ops = {
- .size = sizeof(struct fi_ops_poll),
- .poll = psmx_poll_poll,
- .poll_add = psmx_poll_add,
- .poll_del = psmx_poll_del,
-};
-
-int psmx_poll_open(struct fid_domain *domain, struct fi_poll_attr *attr,
- struct fid_poll **pollset)
-{
- struct psmx_fid_domain *domain_priv;
- struct psmx_fid_poll *poll_priv;
-
- domain_priv = container_of(domain, struct psmx_fid_domain, domain);
-
- poll_priv = calloc(1, sizeof(*poll_priv));
- if (!poll_priv)
- return -FI_ENOMEM;
-
- psmx_domain_acquire(domain_priv);
-
- dlist_init(&poll_priv->poll_list_head);
- poll_priv->poll.fid.fclass = FI_CLASS_POLL;
- poll_priv->poll.fid.context = 0;
- poll_priv->poll.fid.ops = &psmx_fi_ops;
- poll_priv->poll.ops = &psmx_poll_ops;
- poll_priv->domain = domain_priv;
-
- *pollset = &poll_priv->poll;
- return 0;
-}
-
diff --git a/prov/psm/src/psmx_rma.c b/prov/psm/src/psmx_rma.c
index fbe43a9..424e447 100644
--- a/prov/psm/src/psmx_rma.c
+++ b/prov/psm/src/psmx_rma.c
@@ -75,7 +75,7 @@ int psmx_am_rma_handler(psm_am_token_t token, psm_epaddr_t epaddr,
psm_amarg_t *args, int nargs, void *src, uint32_t len)
{
psm_amarg_t rep_args[8];
- void *rma_addr;
+ uint8_t *rma_addr;
ssize_t rma_len;
uint64_t key;
int err = 0;
@@ -93,7 +93,7 @@ int psmx_am_rma_handler(psm_am_token_t token, psm_epaddr_t epaddr,
switch (cmd) {
case PSMX_AM_REQ_WRITE:
rma_len = args[0].u32w1;
- rma_addr = (void *)(uintptr_t)args[2].u64;
+ rma_addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
mr = psmx_mr_get(psmx_active_fabric->active_domain, key);
op_error = mr ?
@@ -141,7 +141,7 @@ int psmx_am_rma_handler(psm_am_token_t token, psm_epaddr_t epaddr,
case PSMX_AM_REQ_WRITE_LONG:
rma_len = args[0].u32w1;
- rma_addr = (void *)(uintptr_t)args[2].u64;
+ rma_addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
mr = psmx_mr_get(psmx_active_fabric->active_domain, key);
op_error = mr ?
@@ -180,7 +180,7 @@ int psmx_am_rma_handler(psm_am_token_t token, psm_epaddr_t epaddr,
case PSMX_AM_REQ_READ:
rma_len = args[0].u32w1;
- rma_addr = (void *)(uintptr_t)args[2].u64;
+ rma_addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
offset = args[4].u64;
mr = psmx_mr_get(psmx_active_fabric->active_domain, key);
@@ -210,7 +210,7 @@ int psmx_am_rma_handler(psm_am_token_t token, psm_epaddr_t epaddr,
case PSMX_AM_REQ_READ_LONG:
rma_len = args[0].u32w1;
- rma_addr = (void *)(uintptr_t)args[2].u64;
+ rma_addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
mr = psmx_mr_get(psmx_active_fabric->active_domain, key);
op_error = mr ?
@@ -604,7 +604,7 @@ static ssize_t psmx_read(struct fid_ep *ep, void *buf, size_t len,
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
return _psmx_read(ep, buf, len, desc, src_addr, addr,
- key, context, ep_priv->flags);
+ key, context, ep_priv->tx_flags);
}
static ssize_t psmx_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
@@ -710,9 +710,9 @@ ssize_t _psmx_write(struct fid_ep *ep, const void *buf, size_t len,
if (!req)
return -FI_ENOMEM;
- memset((void *)req, 0, sizeof(*req));
- memcpy((void *)req + sizeof(*req), (void *)buf, len);
- buf = (void *)req + sizeof(*req);
+ memset(req, 0, sizeof(*req));
+ memcpy((uint8_t *)req + sizeof(*req), (void *)buf, len);
+ buf = (uint8_t *)req + sizeof(*req);
} else {
req = calloc(1, sizeof(*req));
if (!req)
@@ -790,7 +790,7 @@ ssize_t _psmx_write(struct fid_ep *ep, const void *buf, size_t len,
PSMX_AM_RMA_HANDLER, args, nargs,
(void *)buf, chunk_size,
am_flags, NULL, NULL);
- buf += chunk_size;
+ buf = (const uint8_t *)buf + chunk_size;
addr += chunk_size;
len -= chunk_size;
}
@@ -821,7 +821,7 @@ static ssize_t psmx_write(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
return _psmx_write(ep, buf, len, desc, dest_addr, addr, key, context,
- ep_priv->flags, 0);
+ ep_priv->tx_flags, 0);
}
static ssize_t psmx_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
@@ -856,7 +856,7 @@ static ssize_t psmx_inject(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
return _psmx_write(ep, buf, len, NULL, dest_addr, addr, key,
- NULL, ep_priv->flags | FI_INJECT | PSMX_NO_COMPLETION, 0);
+ NULL, ep_priv->tx_flags | FI_INJECT | PSMX_NO_COMPLETION, 0);
}
static ssize_t psmx_writedata(struct fid_ep *ep, const void *buf, size_t len, void *desc,
@@ -868,7 +868,7 @@ static ssize_t psmx_writedata(struct fid_ep *ep, const void *buf, size_t len, vo
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
return _psmx_write(ep, buf, len, desc, dest_addr, addr, key, context,
- ep_priv->flags | FI_REMOTE_CQ_DATA, data);
+ ep_priv->tx_flags | FI_REMOTE_CQ_DATA, data);
}
static ssize_t psmx_injectdata(struct fid_ep *ep, const void *buf, size_t len,
@@ -880,7 +880,7 @@ static ssize_t psmx_injectdata(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
return _psmx_write(ep, buf, len, NULL, dest_addr, addr, key,
- NULL, ep_priv->flags | FI_INJECT | PSMX_NO_COMPLETION,
+ NULL, ep_priv->tx_flags | FI_INJECT | PSMX_NO_COMPLETION,
data);
}
diff --git a/prov/psm/src/psmx_tagged.c b/prov/psm/src/psmx_tagged.c
index 2c40c2f..eb0771a 100644
--- a/prov/psm/src/psmx_tagged.c
+++ b/prov/psm/src/psmx_tagged.c
@@ -315,7 +315,7 @@ static ssize_t psmx_tagged_recv(struct fid_ep *ep, void *buf, size_t len, void *
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
return _psmx_tagged_recv(ep, buf, len, desc, src_addr, tag, ignore,
- context, ep_priv->flags);
+ context, ep_priv->rx_flags);
}
static ssize_t psmx_tagged_recvmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg,
@@ -328,7 +328,7 @@ static ssize_t psmx_tagged_recvmsg(struct fid_ep *ep, const struct fi_msg_tagged
return -FI_EINVAL;
if (msg->iov_count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (msg->iov_count) {
buf = msg->msg_iov[0].iov_base;
len = msg->msg_iov[0].iov_len;
@@ -374,7 +374,7 @@ static ssize_t psmx_tagged_recvv(struct fid_ep *ep, const struct iovec *iov, voi
return -FI_EINVAL;
if (count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -398,7 +398,7 @@ static ssize_t psmx_tagged_recvv_no_flag(struct fid_ep *ep, const struct iovec *
return -FI_EINVAL;
if (count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -423,7 +423,7 @@ static ssize_t psmx_tagged_recvv_no_event(struct fid_ep *ep, const struct iovec
return -FI_EINVAL;
if (count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -761,7 +761,7 @@ static ssize_t psmx_tagged_send(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
return _psmx_tagged_send(ep, buf, len, desc, dest_addr, tag, context,
- ep_priv->flags);
+ ep_priv->tx_flags);
}
static ssize_t psmx_tagged_sendmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg,
@@ -774,7 +774,7 @@ static ssize_t psmx_tagged_sendmsg(struct fid_ep *ep, const struct fi_msg_tagged
return -FI_EINVAL;
if (msg->iov_count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (msg->iov_count) {
buf = msg->msg_iov[0].iov_base;
len = msg->msg_iov[0].iov_len;
@@ -798,7 +798,7 @@ static ssize_t psmx_tagged_sendv(struct fid_ep *ep, const struct iovec *iov, voi
return -FI_EINVAL;
if (count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -823,7 +823,7 @@ static ssize_t psmx_tagged_sendv_no_flag_av_map(struct fid_ep *ep, const struct
return -FI_EINVAL;
if (count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -849,7 +849,7 @@ static ssize_t psmx_tagged_sendv_no_flag_av_table(struct fid_ep *ep, const struc
return -FI_EINVAL;
if (count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -875,7 +875,7 @@ static ssize_t psmx_tagged_sendv_no_event_av_map(struct fid_ep *ep, const struct
return -FI_EINVAL;
if (count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -901,7 +901,7 @@ static ssize_t psmx_tagged_sendv_no_event_av_table(struct fid_ep *ep, const stru
return -FI_EINVAL;
if (count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -923,7 +923,7 @@ static ssize_t psmx_tagged_inject(struct fid_ep *ep, const void *buf, size_t len
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
return _psmx_tagged_send(ep, buf, len, NULL, dest_addr, tag, NULL,
- ep_priv->flags | FI_INJECT | PSMX_NO_COMPLETION);
+ ep_priv->tx_flags | FI_INJECT | PSMX_NO_COMPLETION);
}
/* general case */
diff --git a/prov/psm/src/psmx_wait.c b/prov/psm/src/psmx_wait.c
index c4f75b6..50c0c37 100644
--- a/prov/psm/src/psmx_wait.c
+++ b/prov/psm/src/psmx_wait.c
@@ -109,220 +109,77 @@ static void psmx_wait_stop_progress(void)
;
}
-int psmx_wait_get_obj(struct psmx_fid_wait *wait, void *arg)
-{
- void *obj_ptr;
- int obj_size = 0;
- struct fi_mutex_cond mutex_cond;
-
- if (!arg)
- return -FI_EINVAL;
+static struct fi_ops_wait *psmx_wait_ops_save;
+static struct fi_ops_wait psmx_wait_ops;
- if (wait) {
- switch (wait->type) {
- case FI_WAIT_FD:
- obj_size = sizeof(wait->fd[0]);
- obj_ptr = &wait->fd[0];
- break;
-
- case FI_WAIT_MUTEX_COND:
- mutex_cond.mutex = &wait->mutex;
- mutex_cond.cond = &wait->cond;
- obj_size = sizeof(mutex_cond);
- obj_ptr = &mutex_cond;
- break;
-
- default:
- break;
- }
- }
-
- if (obj_size) {
- memcpy(arg, obj_ptr, obj_size);
- }
-
- return 0;
-}
-
-int psmx_wait_wait(struct fid_wait *wait, int timeout)
+static int psmx_wait_wait(struct fid_wait *wait, int timeout)
{
- struct psmx_fid_wait *wait_priv;
- int err = 0;
+ struct util_wait *wait_priv;
+ struct psmx_fid_fabric *fabric;
+ int err;
- wait_priv = container_of(wait, struct psmx_fid_wait, wait.fid);
+ wait_priv = container_of(wait, struct util_wait, wait_fid);
+ fabric = container_of(wait_priv->fabric, struct psmx_fid_fabric,
+ util_fabric);
- psmx_wait_start_progress(wait_priv->fabric->active_domain);
+ psmx_wait_start_progress(fabric->active_domain);
- switch (wait_priv->type) {
- case FI_WAIT_UNSPEC:
- /* TODO: optimized custom wait */
- break;
-
- case FI_WAIT_FD:
- err = fi_poll_fd(wait_priv->fd[0], timeout);
- if (err > 0)
- err = 0;
- else if (err == 0)
- err = -FI_ETIMEDOUT;
- break;
-
- case FI_WAIT_MUTEX_COND:
- err = fi_wait_cond(&wait_priv->cond,
- &wait_priv->mutex, timeout);
- break;
-
- default:
- break;
- }
+ err = psmx_wait_ops_save->wait(wait, timeout);
psmx_wait_stop_progress();
return err;
}
-void psmx_wait_signal(struct fid_wait *wait)
-{
- struct psmx_fid_wait *wait_priv;
- static char c = 'x';
-
- wait_priv = container_of(wait, struct psmx_fid_wait, wait.fid);
-
- switch (wait_priv->type) {
- case FI_WAIT_UNSPEC:
- /* TODO: optimized custom wait */
- break;
-
- case FI_WAIT_FD:
- if (write(wait_priv->fd[1], &c, 1) != 1)
- FI_WARN(&psmx_prov, FI_LOG_EQ,
- "error signaling wait object\n");
- break;
-
- case FI_WAIT_MUTEX_COND:
- pthread_cond_signal(&wait_priv->cond);
- break;
- }
-}
-
-static int psmx_wait_close(fid_t fid)
-{
- struct psmx_fid_wait *wait;
-
- wait = container_of(fid, struct psmx_fid_wait, wait.fid);
- psmx_fabric_release(wait->fabric);
-
- if (wait->type == FI_WAIT_FD) {
- close(wait->fd[0]);
- close(wait->fd[1]);
- } else if (wait->type == FI_WAIT_MUTEX_COND) {
- pthread_mutex_destroy(&wait->mutex);
- pthread_cond_destroy(&wait->cond);
- }
-
- free(wait);
- return 0;
-}
-
-static struct fi_ops psmx_fi_ops = {
- .size = sizeof(struct fi_ops),
- .close = psmx_wait_close,
- .bind = fi_no_bind,
- .control = fi_no_control,
- .ops_open = fi_no_ops_open,
-};
-
-static struct fi_ops_wait psmx_wait_ops = {
- .size = sizeof(struct fi_ops_wait),
- .wait = psmx_wait_wait,
-};
-
-static int psmx_wait_init(struct psmx_fid_wait *wait, int type)
+int psmx_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
+ struct fid_wait **waitset)
{
- long flags = 0;
-
- wait->type = type;
-
- switch (type) {
- case FI_WAIT_UNSPEC:
- /* TODO: optimized custom wait */
- break;
-
- case FI_WAIT_FD:
- if (socketpair(AF_UNIX, SOCK_STREAM, 0, wait->fd))
- return -errno;
+ struct fid_wait *wait;
+ int err;
- if (fcntl(wait->fd[0], F_GETFL, &flags) == -1) {
- close(wait->fd[0]);
- close(wait->fd[1]);
- return -errno;
- }
+ err = ofi_wait_fd_open(fabric, attr, &wait);
+ if (err)
+ return err;
- if (fcntl(wait->fd[0], F_SETFL, flags | O_NONBLOCK)) {
- close(wait->fd[0]);
- close(wait->fd[1]);
- return -errno;
- }
- break;
-
- case FI_WAIT_MUTEX_COND:
- pthread_mutex_init(&wait->mutex, NULL);
- pthread_cond_init(&wait->cond, NULL);
- break;
-
- default:
- break;
- }
+ psmx_wait_ops_save = wait->ops;
+ psmx_wait_ops = *psmx_wait_ops_save;
+ psmx_wait_ops.wait = psmx_wait_wait;
+ wait->ops = &psmx_wait_ops;
+ *waitset = wait;
return 0;
}
-int psmx_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
- struct fid_wait **waitset)
+int psmx_wait_trywait(struct fid_fabric *fabric, struct fid **fids, int count)
{
- struct psmx_fid_fabric *fabric_priv;
- struct psmx_fid_wait *wait_priv;
- int type = FI_WAIT_FD;
- int err;
-
- if (attr) {
- switch (attr->wait_obj) {
- case FI_WAIT_UNSPEC:
- break;
-
- case FI_WAIT_FD:
- case FI_WAIT_MUTEX_COND:
- type = attr->wait_obj;
- break;
-
- default:
- FI_INFO(&psmx_prov, FI_LOG_EQ,
- "attr->wait_obj=%d, supported=%d,%d,%d\n",
- attr->wait_obj, FI_WAIT_UNSPEC,
- FI_WAIT_FD, FI_WAIT_MUTEX_COND);
- return -FI_EINVAL;
+ struct psmx_fid_cq *cq_priv;
+ struct util_eq *eq;
+ struct util_wait *wait;
+ int i, ret;
+
+ for (i = 0; i < count; i++) {
+ switch (fids[i]->fclass) {
+ case FI_CLASS_CQ:
+ cq_priv = container_of(fids[i], struct psmx_fid_cq, cq);
+ wait = cq_priv->wait;
+ break;
+ case FI_CLASS_EQ:
+ eq = container_of(fids[i], struct util_eq, eq_fid.fid);
+ wait = eq->wait;
+ break;
+ case FI_CLASS_CNTR:
+ return -FI_ENOSYS;
+ case FI_CLASS_WAIT:
+ wait = container_of(fids[i], struct util_wait, wait_fid.fid);
+ break;
+ default:
+ return -FI_EINVAL;
}
- }
- wait_priv = calloc(1, sizeof(*wait_priv));
- if (!wait_priv)
- return -FI_ENOMEM;
-
- err = psmx_wait_init(wait_priv, type);
- if (err) {
- free(wait_priv);
- return err;
+ ret = wait->try(wait);
+ if (ret)
+ return ret;
}
-
- fabric_priv = container_of(fabric, struct psmx_fid_fabric, fabric);
- psmx_fabric_acquire(fabric_priv);
-
- wait_priv->fabric = fabric_priv;
- wait_priv->wait.fid.fclass = FI_CLASS_WAIT;
- wait_priv->wait.fid.context = 0;
- wait_priv->wait.fid.ops = &psmx_fi_ops;
- wait_priv->wait.ops = &psmx_wait_ops;
-
- *waitset = &wait_priv->wait;
return 0;
}
diff --git a/prov/psm2/src/psmx2.h b/prov/psm2/src/psmx2.h
index a5b1737..264ca8d 100644
--- a/prov/psm2/src/psmx2.h
+++ b/prov/psm2/src/psmx2.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013-2014 Intel Corporation. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -49,11 +50,11 @@ extern "C" {
#include <fcntl.h>
#include <pthread.h>
#include <sys/types.h>
+#include <sys/stat.h>
#include <sys/socket.h>
#include <netdb.h>
#include <complex.h>
#include <rdma/fabric.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_domain.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_tagged.h>
@@ -62,7 +63,6 @@ extern "C" {
#include <rdma/fi_trigger.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_errno.h>
-#include <rdma/fi_log.h>
#include "fi.h"
#include "fi_enosys.h"
#include "fi_list.h"
@@ -124,7 +124,7 @@ extern struct fi_provider psmx2_prov;
#define PSMX2_SIGN_MASK (0x0080000000000000UL)
#define PSMX2_SIGN_EXT (0xFF00000000000000UL)
#define PSMX2_VL_MASK (0xFF00000000000000UL)
-
+
#define PSMX2_EP_TO_ADDR(ep,vl) ((((uint64_t)vl) << 56) | \
((uint64_t)ep & PSMX2_EP_MASK))
#define PSMX2_ADDR_TO_VL(addr) ((uint8_t)((addr & PSMX2_VL_MASK) >> 56))
@@ -207,13 +207,14 @@ enum {
PSMX2_AM_REQ_ATOMIC_COMPWRITE,
PSMX2_AM_REP_ATOMIC_COMPWRITE,
PSMX2_AM_REQ_WRITEV,
+ PSMX2_AM_REQ_READV,
};
struct psmx2_am_request {
int op;
union {
struct {
- void *buf;
+ uint8_t *buf;
size_t len;
uint64_t addr;
uint64_t key;
@@ -224,7 +225,10 @@ struct psmx2_am_request {
uint64_t data;
} write;
struct {
- void *buf;
+ union {
+ uint8_t *buf; /* for read */
+ size_t iov_count; /* for readv */
+ };
size_t len;
uint64_t addr;
uint64_t key;
@@ -235,12 +239,16 @@ struct psmx2_am_request {
size_t len_read;
} read;
struct {
- void *buf;
+ union {
+ uint8_t *buf; /* for result_count == 1 */
+ size_t iov_count; /* for result_count > 1 */
+ };
size_t len;
uint64_t addr;
uint64_t key;
void *context;
- void *result;
+ uint8_t *result;
+ int datatype;
} atomic;
};
uint64_t cq_flags;
@@ -249,6 +257,10 @@ struct psmx2_am_request {
int no_event;
int error;
struct slist_entry list_entry;
+ union {
+ struct iovec iov[0]; /* for readv, must be the last field */
+ struct fi_ioc ioc[0]; /* for atomic read, must be the last field */
+ };
};
#define PSMX2_IOV_PROTO_PACK 0
@@ -282,7 +294,7 @@ struct psmx2_sendv_reply {
struct fi_context fi_context;
int no_completion;
int multi_recv;
- void *buf;
+ uint8_t *buf;
void *user_context;
size_t iov_done;
size_t bytes_received;
@@ -301,7 +313,7 @@ struct psmx2_multi_recv {
psm2_epaddr_t src_addr;
psm2_mq_tag_t tag;
psm2_mq_tag_t tagsel;
- void *buf;
+ uint8_t *buf;
size_t len;
size_t offset;
int min_buf_size;
@@ -334,7 +346,7 @@ struct psmx2_fid_domain {
uint64_t vl_map[(PSMX2_MAX_VL+1)/sizeof(uint64_t)];
int vl_alloc;
struct psmx2_fid_ep *eps[PSMX2_MAX_VL+1];
-
+
int am_initialized;
/* incoming req queue for AM based RMA request. */
@@ -395,9 +407,13 @@ enum psmx2_triggered_op {
PSMX2_TRIGGERED_WRITE,
PSMX2_TRIGGERED_WRITEV,
PSMX2_TRIGGERED_READ,
+ PSMX2_TRIGGERED_READV,
PSMX2_TRIGGERED_ATOMIC_WRITE,
+ PSMX2_TRIGGERED_ATOMIC_WRITEV,
PSMX2_TRIGGERED_ATOMIC_READWRITE,
+ PSMX2_TRIGGERED_ATOMIC_READWRITEV,
PSMX2_TRIGGERED_ATOMIC_COMPWRITE,
+ PSMX2_TRIGGERED_ATOMIC_COMPWRITEV,
};
struct psmx2_trigger {
@@ -504,6 +520,17 @@ struct psmx2_trigger {
} read;
struct {
struct fid_ep *ep;
+ const struct iovec *iov;
+ size_t count;
+ void *desc;
+ fi_addr_t src_addr;
+ uint64_t addr;
+ uint64_t key;
+ void *context;
+ uint64_t flags;
+ } readv;
+ struct {
+ struct fid_ep *ep;
const void *buf;
size_t count;
void *desc;
@@ -517,6 +544,19 @@ struct psmx2_trigger {
} atomic_write;
struct {
struct fid_ep *ep;
+ const struct fi_ioc *iov;
+ size_t count;
+ void *desc;
+ fi_addr_t dest_addr;
+ uint64_t addr;
+ uint64_t key;
+ enum fi_datatype datatype;
+ enum fi_op atomic_op;
+ void *context;
+ uint64_t flags;
+ } atomic_writev;
+ struct {
+ struct fid_ep *ep;
const void *buf;
size_t count;
void *desc;
@@ -532,6 +572,22 @@ struct psmx2_trigger {
} atomic_readwrite;
struct {
struct fid_ep *ep;
+ const struct fi_ioc *iov;
+ size_t count;
+ void **desc;
+ struct fi_ioc *resultv;
+ void **result_desc;
+ size_t result_count;
+ fi_addr_t dest_addr;
+ uint64_t addr;
+ uint64_t key;
+ enum fi_datatype datatype;
+ enum fi_op atomic_op;
+ void *context;
+ uint64_t flags;
+ } atomic_readwritev;
+ struct {
+ struct fid_ep *ep;
const void *buf;
size_t count;
void *desc;
@@ -547,20 +603,40 @@ struct psmx2_trigger {
void *context;
uint64_t flags;
} atomic_compwrite;
+ struct {
+ struct fid_ep *ep;
+ const struct fi_ioc *iov;
+ size_t count;
+ void **desc;
+ const struct fi_ioc *comparev;
+ void **compare_desc;
+ size_t compare_count;
+ struct fi_ioc *resultv;
+ void **result_desc;
+ size_t result_count;
+ fi_addr_t dest_addr;
+ uint64_t addr;
+ uint64_t key;
+ enum fi_datatype datatype;
+ enum fi_op atomic_op;
+ void *context;
+ uint64_t flags;
+ } atomic_compwritev;
};
struct psmx2_trigger *next; /* used for randomly accessed trigger list */
struct slist_entry list_entry; /* used for ready-to-fire trigger queue */
};
struct psmx2_fid_cntr {
- struct fid_cntr cntr;
+ union {
+ struct fid_cntr cntr;
+ struct util_cntr util_cntr; /* for util_poll_run */
+ };
struct psmx2_fid_domain *domain;
int events;
uint64_t flags;
- volatile uint64_t counter;
- volatile uint64_t error_counter;
- uint64_t counter_last_read;
- uint64_t error_counter_last_read;
+ atomic_t counter;
+ atomic_t error_counter;
struct util_wait *wait;
int wait_is_local;
struct psmx2_trigger *trigger;
@@ -583,6 +659,7 @@ struct psmx2_fid_av {
struct psmx2_fid_ep {
struct fid_ep ep;
+ struct psmx2_fid_ep *base_ep;
struct psmx2_fid_domain *domain;
struct psmx2_fid_av *av;
struct psmx2_fid_cq *send_cq;
@@ -596,8 +673,11 @@ struct psmx2_fid_ep {
uint8_t vlane;
unsigned send_selective_completion:1;
unsigned recv_selective_completion:1;
- uint64_t flags;
+ unsigned enabled:1;
+ uint64_t tx_flags;
+ uint64_t rx_flags;
uint64_t caps;
+ atomic_t ref;
struct fi_context nocomp_send_context;
struct fi_context nocomp_recv_context;
struct slist free_context_list;
@@ -672,6 +752,8 @@ int psmx2_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
struct fid_cntr **cntr, void *context);
int psmx2_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
struct fid_wait **waitset);
+int psmx2_wait_trywait(struct fid_fabric *fabric, struct fid **fids,
+ int count);
static inline void psmx2_fabric_acquire(struct psmx2_fid_fabric *fabric)
{
@@ -744,7 +826,7 @@ int psmx2_handle_sendv_req(struct psmx2_fid_ep *ep, psm2_mq_status2_t *psm2_stat
static inline void psmx2_cntr_inc(struct psmx2_fid_cntr *cntr)
{
- cntr->counter++;
+ atomic_inc(&cntr->counter);
psmx2_cntr_check_trigger(cntr);
if (cntr->wait)
cntr->wait->signal(cntr->wait);
@@ -825,6 +907,13 @@ ssize_t psmx2_read_generic(
uint64_t addr, uint64_t key,
void *context, uint64_t flags);
+ssize_t psmx2_readv_generic(
+ struct fid_ep *ep,
+ const struct iovec *iov, void *desc,
+ size_t count, fi_addr_t src_addr,
+ uint64_t addr, uint64_t key,
+ void *context, uint64_t flags);
+
ssize_t psmx2_atomic_write_generic(
struct fid_ep *ep,
const void *buf,
diff --git a/prov/psm2/src/psmx2_atomic.c b/prov/psm2/src/psmx2_atomic.c
index 142c4a3..b72571c 100644
--- a/prov/psm2/src/psmx2_atomic.c
+++ b/prov/psm2/src/psmx2_atomic.c
@@ -61,6 +61,50 @@ void psmx2_atomic_fini(void)
fastlock_destroy(&psmx2_atomic_lock);
}
+static inline void psmx2_ioc_read(const struct fi_ioc *ioc, size_t count,
+ int datatype, uint8_t *buf, size_t len)
+{
+ int i;
+ size_t copy_len;
+
+ for (i=0; i<count && len; i++) {
+ copy_len = fi_datatype_size(datatype) * ioc[i].count;
+ if (copy_len > len)
+ copy_len = len;
+ memcpy(buf, ioc[i].addr, copy_len);
+ buf += copy_len;
+ len -= copy_len;
+ }
+}
+
+static inline void psmx2_ioc_write(struct fi_ioc *ioc, size_t count,
+ int datatype, const uint8_t *buf, size_t len)
+{
+ int i;
+ size_t copy_len;
+
+ for (i=0; i<count && len; i++) {
+ copy_len = fi_datatype_size(datatype) * ioc[i].count;
+ if (copy_len > len)
+ copy_len = len;
+ memcpy(ioc[i].addr, buf, copy_len);
+ buf += copy_len;
+ len -= copy_len;
+ }
+}
+
+static inline size_t psmx2_ioc_size(const struct fi_ioc *ioc, size_t count,
+ int datatype)
+{
+ int i;
+ size_t len = 0;
+
+ for (i=0; i<count; i++)
+ len += fi_datatype_size(datatype) * ioc[i].count;
+
+ return len;
+}
+
#define CASE_INT_TYPE(FUNC,...) \
case FI_INT8: FUNC(__VA_ARGS__,int8_t); break; \
case FI_UINT8: FUNC(__VA_ARGS__,uint8_t); break; \
@@ -377,7 +421,7 @@ int psmx2_am_atomic_handler(psm2_am_token_t token,
{
psm2_amarg_t rep_args[8];
int count;
- void *addr;
+ uint8_t *addr;
uint64_t key;
int datatype, op;
int err = 0;
@@ -405,7 +449,7 @@ int psmx2_am_atomic_handler(psm2_am_token_t token,
switch (cmd) {
case PSMX2_AM_REQ_ATOMIC_WRITE:
count = args[0].u32w1;
- addr = (void *)(uintptr_t)args[2].u64;
+ addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
datatype = args[4].u32w0;
op = args[4].u32w1;
@@ -440,7 +484,7 @@ int psmx2_am_atomic_handler(psm2_am_token_t token,
case PSMX2_AM_REQ_ATOMIC_READWRITE:
count = args[0].u32w1;
- addr = (void *)(uintptr_t)args[2].u64;
+ addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
datatype = args[4].u32w0;
op = args[4].u32w1;
@@ -491,7 +535,7 @@ int psmx2_am_atomic_handler(psm2_am_token_t token,
case PSMX2_AM_REQ_ATOMIC_COMPWRITE:
count = args[0].u32w1;
- addr = (void *)(uintptr_t)args[2].u64;
+ addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
datatype = args[4].u32w0;
op = args[4].u32w1;
@@ -508,7 +552,7 @@ int psmx2_am_atomic_handler(psm2_am_token_t token,
addr += mr->offset;
tmp_buf = malloc(len);
if (tmp_buf)
- psmx2_atomic_do_compwrite(addr, src, src + len,
+ psmx2_atomic_do_compwrite(addr, src, (uint8_t *)src + len,
tmp_buf, datatype,
op, count);
else
@@ -569,8 +613,13 @@ int psmx2_am_atomic_handler(psm2_am_token_t token,
op_error = (int)args[0].u32w1;
assert(op_error || req->atomic.len == len);
- if (!op_error)
- memcpy(req->atomic.result, src, len);
+ if (!op_error) {
+ if (req->atomic.result)
+ memcpy(req->atomic.result, src, len);
+ else
+ psmx2_ioc_write(req->ioc, req->atomic.iov_count,
+ req->atomic.datatype, src, len);
+ }
if (req->ep->send_cq && !req->no_event) {
event = psmx2_cq_create_event(
@@ -833,9 +882,9 @@ ssize_t psmx2_atomic_write_generic(struct fid_ep *ep,
req = malloc(sizeof(*req) + len);
if (!req)
return -FI_ENOMEM;
- memset((void *)req, 0, sizeof(*req));
- memcpy((void *)req+sizeof(*req), (void *)buf, len);
- buf = (void *)req + sizeof(*req);
+ memset(req, 0, sizeof(*req));
+ memcpy((uint8_t *)req+sizeof(*req), (void *)buf, len);
+ buf = (uint8_t *)req + sizeof(*req);
} else {
req = calloc(1, sizeof(*req));
if (!req)
@@ -851,6 +900,7 @@ ssize_t psmx2_atomic_write_generic(struct fid_ep *ep,
req->atomic.addr = addr;
req->atomic.key = key;
req->atomic.context = context;
+ req->atomic.datatype = datatype;
req->ep = ep_priv;
req->cq_flags = FI_WRITE | FI_ATOMIC;
@@ -869,6 +919,154 @@ ssize_t psmx2_atomic_write_generic(struct fid_ep *ep,
return 0;
}
+ssize_t psmx2_atomic_writev_generic(struct fid_ep *ep,
+ const struct fi_ioc *iov,
+ void **desc, size_t count,
+ fi_addr_t dest_addr,
+ uint64_t addr, uint64_t key,
+ enum fi_datatype datatype,
+ enum fi_op op, void *context,
+ uint64_t flags)
+{
+ struct psmx2_fid_ep *ep_priv;
+ struct psmx2_fid_av *av;
+ struct psmx2_epaddr_context *epaddr_context;
+ struct psmx2_am_request *req;
+ psm2_amarg_t args[8];
+ psm2_epaddr_t psm2_epaddr;
+ uint8_t vlane;
+ int am_flags = PSM2_AM_FLAG_ASYNC;
+ int chunk_size;
+ size_t idx;
+ size_t len;
+ uint8_t *buf;
+ int err;
+
+ ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
+
+ if (flags & FI_TRIGGER) {
+ struct psmx2_trigger *trigger;
+ struct fi_triggered_context *ctxt = context;
+
+ trigger = calloc(1, sizeof(*trigger));
+ if (!trigger)
+ return -FI_ENOMEM;
+
+ trigger->op = PSMX2_TRIGGERED_ATOMIC_WRITEV;
+ trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
+ struct psmx2_fid_cntr, cntr);
+ trigger->threshold = ctxt->trigger.threshold.threshold;
+ trigger->atomic_writev.ep = ep;
+ trigger->atomic_writev.iov = iov;
+ trigger->atomic_writev.count = count;
+ trigger->atomic_writev.desc = desc;
+ trigger->atomic_writev.dest_addr = dest_addr;
+ trigger->atomic_writev.addr = addr;
+ trigger->atomic_writev.key = key;
+ trigger->atomic_writev.datatype = datatype;
+ trigger->atomic_writev.atomic_op = op;
+ trigger->atomic_writev.context = context;
+ trigger->atomic_writev.flags = flags & ~FI_TRIGGER;
+
+ psmx2_cntr_add_trigger(trigger->cntr, trigger);
+ return 0;
+ }
+
+ if (!iov || !count)
+ return -FI_EINVAL;
+
+ while (count && !iov[count-1].count)
+ count--;
+
+ if (datatype < 0 || datatype >= FI_DATATYPE_LAST)
+ return -FI_EINVAL;
+
+ if (op < 0 || op >= FI_ATOMIC_OP_LAST)
+ return -FI_EINVAL;
+
+ av = ep_priv->av;
+ if (av && av->type == FI_AV_TABLE) {
+ idx = dest_addr;
+ if (idx >= av->last)
+ return -FI_EINVAL;
+
+ psm2_epaddr = av->epaddrs[idx];
+ vlane = av->vlanes[idx];
+ } else {
+ if (!dest_addr)
+ return -FI_EINVAL;
+
+ psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr);
+ vlane = PSMX2_ADDR_TO_VL(dest_addr);
+ }
+
+ len = psmx2_ioc_size(iov, count, datatype);
+
+ epaddr_context = psm2_epaddr_getctxt((void *)psm2_epaddr);
+ if (epaddr_context->epid == ep_priv->domain->psm2_epid) {
+ buf = malloc(len);
+ if (!buf)
+ return -FI_ENOMEM;
+
+ psmx2_ioc_read(iov, count, datatype, buf, len);
+
+ err = psmx2_atomic_self(PSMX2_AM_REQ_ATOMIC_WRITE, ep_priv,
+ ep_priv->domain->eps[vlane],
+ buf, len / fi_datatype_size(datatype),
+ NULL, NULL, NULL, NULL, NULL, addr,
+ key, datatype, op, context, flags);
+
+ free(buf);
+ return err;
+ }
+
+ chunk_size = psmx2_am_param.max_request_short;
+ if (len > chunk_size)
+ return -FI_EMSGSIZE;
+
+ if (count > 1) {
+ req = malloc(sizeof(*req) + len);
+ if (!req)
+ return -FI_ENOMEM;
+
+ buf = (uint8_t *)req + sizeof(*req);
+ memset(req, 0, sizeof(*req));
+ psmx2_ioc_read(iov, count, datatype, buf, len);
+ } else {
+ req = calloc(1, sizeof(*req));
+ if (!req)
+ return -FI_ENOMEM;
+ buf = iov[0].addr;
+ }
+
+ req->no_event = (flags & PSMX2_NO_COMPLETION) ||
+ (ep_priv->send_selective_completion && !(flags & FI_COMPLETION));
+
+ req->op = PSMX2_AM_REQ_ATOMIC_WRITE;
+ req->atomic.buf = (void *)buf;
+ req->atomic.len = len;
+ req->atomic.addr = addr;
+ req->atomic.key = key;
+ req->atomic.context = context;
+ req->atomic.datatype = datatype;
+ req->ep = ep_priv;
+ req->cq_flags = FI_WRITE | FI_ATOMIC;
+
+ args[0].u32w0 = PSMX2_AM_REQ_ATOMIC_WRITE;
+ PSMX2_AM_SET_DST(args[0].u32w0, vlane);
+ args[0].u32w1 = len / fi_datatype_size(datatype);
+ args[1].u64 = (uint64_t)(uintptr_t)req;
+ args[2].u64 = addr;
+ args[3].u64 = key;
+ args[4].u32w0 = datatype;
+ args[4].u32w1 = op;
+ psm2_am_request_short(psm2_epaddr,
+ PSMX2_AM_ATOMIC_HANDLER, args, 5,
+ (void *)buf, len, am_flags, NULL, NULL);
+
+ return 0;
+}
+
static ssize_t psmx2_atomic_write(struct fid_ep *ep,
const void *buf,
size_t count, void *desc,
@@ -882,16 +1080,25 @@ static ssize_t psmx2_atomic_write(struct fid_ep *ep,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_atomic_write_generic(ep, buf, count, desc, dest_addr,
addr, key, datatype, op, context,
- ep_priv->flags);
+ ep_priv->tx_flags);
}
static ssize_t psmx2_atomic_writemsg(struct fid_ep *ep,
const struct fi_msg_atomic *msg,
uint64_t flags)
{
- if (!msg || msg->iov_count != 1 || !msg->msg_iov || !msg->rma_iov)
+ if (!msg || !msg->iov_count || !msg->msg_iov || !msg->rma_iov ||
+ msg->rma_iov_count != 1)
return -FI_EINVAL;
+ if (msg->iov_count > 1)
+ return psmx2_atomic_writev_generic(ep, msg->msg_iov, msg->desc,
+ msg->iov_count, msg->addr,
+ msg->rma_iov[0].addr,
+ msg->rma_iov[0].key,
+ msg->datatype, msg->op,
+ msg->context, flags);
+
return psmx2_atomic_write_generic(ep, msg->msg_iov[0].addr,
msg->msg_iov[0].count,
msg->desc ? msg->desc[0] : NULL,
@@ -908,12 +1115,23 @@ static ssize_t psmx2_atomic_writev(struct fid_ep *ep,
enum fi_datatype datatype,
enum fi_op op, void *context)
{
- if (!iov || count != 1)
+ struct psmx2_fid_ep *ep_priv;
+
+ ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
+
+ if (!iov || !count)
return -FI_EINVAL;
- return psmx2_atomic_write(ep, iov->addr, iov->count,
- desc ? desc[0] : NULL, dest_addr, addr,
- key, datatype, op, context);
+ if (count > 1)
+ return psmx2_atomic_writev_generic(ep, iov, desc, count,
+ dest_addr, addr, key,
+ datatype, op, context,
+ ep_priv->tx_flags);
+
+ return psmx2_atomic_write_generic(ep, iov->addr, iov->count,
+ desc ? desc[0] : NULL, dest_addr,
+ addr, key, datatype, op, context,
+ ep_priv->tx_flags);
}
static ssize_t psmx2_atomic_inject(struct fid_ep *ep,
@@ -930,7 +1148,7 @@ static ssize_t psmx2_atomic_inject(struct fid_ep *ep,
return psmx2_atomic_write_generic(ep, buf, count, NULL/*desc*/,
dest_addr, addr, key,
datatype, op, NULL,
- ep_priv->flags | FI_INJECT | PSMX2_NO_COMPLETION);
+ ep_priv->tx_flags | FI_INJECT | PSMX2_NO_COMPLETION);
}
ssize_t psmx2_atomic_readwrite_generic(struct fid_ep *ep,
@@ -1028,9 +1246,9 @@ ssize_t psmx2_atomic_readwrite_generic(struct fid_ep *ep,
req = malloc(sizeof(*req) + len);
if (!req)
return -FI_ENOMEM;
- memset((void *)req, 0, sizeof(*req));
- memcpy((void *)req+sizeof(*req), (void *)buf, len);
- buf = (void *)req + sizeof(*req);
+ memset(req, 0, sizeof(*req));
+ memcpy((uint8_t *)req+sizeof(*req), (void *)buf, len);
+ buf = (uint8_t *)req + sizeof(*req);
} else {
req = calloc(1, sizeof(*req));
if (!req)
@@ -1047,6 +1265,7 @@ ssize_t psmx2_atomic_readwrite_generic(struct fid_ep *ep,
req->atomic.key = key;
req->atomic.context = context;
req->atomic.result = result;
+ req->atomic.datatype = datatype;
req->ep = ep_priv;
if (op == FI_ATOMIC_READ)
req->cq_flags = FI_READ | FI_ATOMIC;
@@ -1068,6 +1287,210 @@ ssize_t psmx2_atomic_readwrite_generic(struct fid_ep *ep,
return 0;
}
+ssize_t psmx2_atomic_readwritev_generic(struct fid_ep *ep,
+ const struct fi_ioc *iov,
+ void **desc, size_t count,
+ struct fi_ioc *resultv,
+ void **result_desc,
+ size_t result_count,
+ fi_addr_t dest_addr,
+ uint64_t addr, uint64_t key,
+ enum fi_datatype datatype,
+ enum fi_op op, void *context,
+ uint64_t flags)
+{
+ struct psmx2_fid_ep *ep_priv;
+ struct psmx2_fid_av *av;
+ struct psmx2_epaddr_context *epaddr_context;
+ struct psmx2_am_request *req;
+ psm2_amarg_t args[8];
+ psm2_epaddr_t psm2_epaddr;
+ uint8_t vlane;
+ int am_flags = PSM2_AM_FLAG_ASYNC;
+ int chunk_size;
+ size_t idx;
+ size_t len, result_len, iov_size;
+ uint8_t *buf, *result;
+ void *desc0, *result_desc0;
+ int err;
+
+ ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
+
+ if (flags & FI_TRIGGER) {
+ struct psmx2_trigger *trigger;
+ struct fi_triggered_context *ctxt = context;
+
+ trigger = calloc(1, sizeof(*trigger));
+ if (!trigger)
+ return -FI_ENOMEM;
+
+ trigger->op = PSMX2_TRIGGERED_ATOMIC_READWRITEV;
+ trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
+ struct psmx2_fid_cntr, cntr);
+ trigger->threshold = ctxt->trigger.threshold.threshold;
+ trigger->atomic_readwritev.ep = ep;
+ trigger->atomic_readwritev.iov = iov;
+ trigger->atomic_readwritev.count = count;
+ trigger->atomic_readwritev.desc = desc;
+ trigger->atomic_readwritev.resultv = resultv;
+ trigger->atomic_readwritev.result_desc = result_desc;
+ trigger->atomic_readwritev.result_count = result_count;
+ trigger->atomic_readwritev.dest_addr = dest_addr;
+ trigger->atomic_readwritev.addr = addr;
+ trigger->atomic_readwritev.key = key;
+ trigger->atomic_readwritev.datatype = datatype;
+ trigger->atomic_readwritev.atomic_op = op;
+ trigger->atomic_readwritev.context = context;
+ trigger->atomic_readwritev.flags = flags & ~FI_TRIGGER;
+
+ psmx2_cntr_add_trigger(trigger->cntr, trigger);
+ return 0;
+ }
+
+ if (((!iov || !count) && op != FI_ATOMIC_READ) || !resultv ||
+ !result_count)
+ return -FI_EINVAL;
+
+ while (count && !iov[count-1].count)
+ count--;
+
+ while (result_count && !resultv[result_count-1].count)
+ result_count--;
+
+ if (datatype < 0 || datatype >= FI_DATATYPE_LAST)
+ return -FI_EINVAL;
+
+ if (op < 0 || op >= FI_ATOMIC_OP_LAST)
+ return -FI_EINVAL;
+
+ result_len = psmx2_ioc_size(resultv, result_count, datatype);
+
+ if (op != FI_ATOMIC_READ) {
+ buf = iov[0].addr; /* as default for count == 1 */
+ len = psmx2_ioc_size(iov, count, datatype);
+ desc0 = desc ? desc[0] : NULL;
+ } else {
+ buf = NULL;
+ len = result_len;
+ desc0 = NULL;
+ }
+
+ if (result_len < len)
+ return -FI_EINVAL;
+
+ av = ep_priv->av;
+ if (av && av->type == FI_AV_TABLE) {
+ idx = dest_addr;
+ if (idx >= av->last)
+ return -FI_EINVAL;
+
+ psm2_epaddr = av->epaddrs[idx];
+ vlane = av->vlanes[idx];
+ } else {
+ if (!dest_addr)
+ return -FI_EINVAL;
+
+ psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr);
+ vlane = PSMX2_ADDR_TO_VL(dest_addr);
+ }
+
+ epaddr_context = psm2_epaddr_getctxt((void *)psm2_epaddr);
+ if (epaddr_context->epid == ep_priv->domain->psm2_epid) {
+ if (buf && count > 1) {
+ buf = malloc(len);
+ psmx2_ioc_read(iov, count, datatype, buf, len);
+ desc0 = NULL;
+ }
+
+ if (result_count > 1) {
+ result = malloc(len);
+ if (!result) {
+ if (buf && count > 1)
+ free(buf);
+ return -FI_ENOMEM;
+ }
+ result_desc0 = result_desc ? result_desc[0] : NULL;
+ } else {
+ result = resultv[0].addr;
+ result_desc0 = NULL;
+ }
+
+ err = psmx2_atomic_self(PSMX2_AM_REQ_ATOMIC_READWRITE,
+ ep_priv, ep_priv->domain->eps[vlane],
+ buf, len / fi_datatype_size(datatype),
+ desc0, NULL, NULL, result, result_desc0,
+ addr, key, datatype, op, context, flags);
+
+ if (result_count > 1) {
+ psmx2_ioc_write(resultv, result_count, datatype, result, len);
+ free(result);
+ }
+
+ if (buf && count > 1)
+ free(buf);
+
+ return err;
+ }
+
+ chunk_size = psmx2_am_param.max_request_short;
+ if (len > chunk_size)
+ return -FI_EMSGSIZE;
+
+ iov_size = result_count > 1 ? result_count * sizeof(struct fi_ioc) : 0;
+
+ if (((flags & FI_INJECT) || count > 1) && op != FI_ATOMIC_READ) {
+ req = malloc(sizeof(*req) + iov_size + len);
+ if (!req)
+ return -FI_ENOMEM;
+ buf = (uint8_t *)req + sizeof(*req) + iov_size;
+ memset(req, 0, sizeof(*req));
+ psmx2_ioc_read(iov, count, datatype, buf, len);
+ } else {
+ req = calloc(1, sizeof(*req) + iov_size);
+ if (!req)
+ return -FI_ENOMEM;
+ }
+
+ if (iov_size) {
+ memcpy(req->ioc, resultv, iov_size);
+ req->atomic.iov_count = result_count;
+ req->atomic.result = NULL;
+ } else {
+ req->atomic.buf = buf;
+ req->atomic.result = resultv[0].addr;
+ }
+
+ req->no_event = (flags & PSMX2_NO_COMPLETION) ||
+ (ep_priv->send_selective_completion && !(flags & FI_COMPLETION));
+
+ req->op = PSMX2_AM_REQ_ATOMIC_READWRITE;
+ req->atomic.buf = (void *)buf;
+ req->atomic.len = len;
+ req->atomic.addr = addr;
+ req->atomic.key = key;
+ req->atomic.context = context;
+ req->atomic.datatype = datatype;
+ req->ep = ep_priv;
+ if (op == FI_ATOMIC_READ)
+ req->cq_flags = FI_READ | FI_ATOMIC;
+ else
+ req->cq_flags = FI_WRITE | FI_ATOMIC;
+
+ args[0].u32w0 = PSMX2_AM_REQ_ATOMIC_READWRITE;
+ PSMX2_AM_SET_DST(args[0].u32w0, vlane);
+ args[0].u32w1 = len / fi_datatype_size(datatype);
+ args[1].u64 = (uint64_t)(uintptr_t)req;
+ args[2].u64 = addr;
+ args[3].u64 = key;
+ args[4].u32w0 = datatype;
+ args[4].u32w1 = op;
+ psm2_am_request_short(psm2_epaddr,
+ PSMX2_AM_ATOMIC_HANDLER, args, 5,
+ (void *)buf, (buf?len:0), am_flags, NULL, NULL);
+
+ return 0;
+}
+
static ssize_t psmx2_atomic_readwrite(struct fid_ep *ep,
const void *buf,
size_t count, void *desc,
@@ -1083,7 +1506,7 @@ static ssize_t psmx2_atomic_readwrite(struct fid_ep *ep,
return psmx2_atomic_readwrite_generic(ep, buf, count, desc,
result, result_desc, dest_addr,
addr, key, datatype, op,
- context, ep_priv->flags);
+ context, ep_priv->tx_flags);
}
static ssize_t psmx2_atomic_readwritemsg(struct fid_ep *ep,
@@ -1095,27 +1518,37 @@ static ssize_t psmx2_atomic_readwritemsg(struct fid_ep *ep,
{
void *buf;
size_t count;
+ void *desc;
- if (!msg || !msg->rma_iov)
+ if (!msg || !msg->rma_iov || msg->rma_iov_count !=1 || !resultv ||
+ !result_count)
+ return -FI_EINVAL;
+
+ if ((msg->op != FI_ATOMIC_READ) && (!msg->msg_iov || !msg->iov_count))
return -FI_EINVAL;
- if (msg->op == FI_ATOMIC_READ) {
- if (result_count != 1 || !resultv)
- return -FI_EINVAL;
+ if ((msg->op != FI_ATOMIC_READ && msg->iov_count > 1) ||
+ result_count > 1)
+ return psmx2_atomic_readwritev_generic(ep, msg->msg_iov, msg->desc,
+ msg->iov_count, resultv,
+ result_desc, result_count,
+ msg->addr,
+ msg->rma_iov[0].addr,
+ msg->rma_iov[0].key,
+ msg->datatype, msg->op,
+ msg->context, flags);
+ if (msg->op == FI_ATOMIC_READ) {
buf = NULL;
count = resultv[0].count;
+ desc = result_desc ? result_desc[0] : NULL;
} else {
- if (msg->iov_count != 1 || !msg->msg_iov)
- return -FI_EINVAL;
-
buf = msg->msg_iov[0].addr;
count = msg->msg_iov[0].count;
+ desc = msg->desc ? msg->desc[0] : NULL;
}
- return psmx2_atomic_readwrite_generic(ep, buf, count,
- msg->desc ? msg->desc[0] : NULL,
- resultv[0].addr,
+ return psmx2_atomic_readwrite_generic(ep, buf, count, desc, resultv[0].addr,
result_desc ? result_desc[0] : NULL,
msg->addr, msg->rma_iov[0].addr,
msg->rma_iov[0].key, msg->datatype,
@@ -1132,14 +1565,38 @@ static ssize_t psmx2_atomic_readwritev(struct fid_ep *ep,
enum fi_datatype datatype,
enum fi_op op, void *context)
{
- if (!iov || count != 1 || !resultv)
+ struct psmx2_fid_ep *ep_priv;
+ void *buf;
+ void *src_desc;
+
+ ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
+
+ if (!resultv || !result_count)
+ return -FI_EINVAL;
+
+ if ((op != FI_ATOMIC_READ) && (!iov || !count))
return -FI_EINVAL;
+
+ if ((op != FI_ATOMIC_READ && count > 1) || result_count > 1)
+ return psmx2_atomic_readwritev_generic(ep, iov, desc, count,
+ resultv, result_desc, result_count,
+ dest_addr, addr, key, datatype, op,
+ context, ep_priv->tx_flags);
+
+ if (FI_ATOMIC_READ) {
+ buf = NULL;
+ count = resultv[0].count;
+ src_desc = result_desc ? result_desc[0] : NULL;
+ } else {
+ buf = iov[0].addr;
+ count = iov[0].count;
+ src_desc = desc ? desc[0] : NULL;
+ }
- return psmx2_atomic_readwrite(ep, iov->addr, iov->count,
- desc ? desc[0] : NULL,
- resultv[0].addr,
- result_desc ? result_desc[0] : NULL,
- dest_addr, addr, key, datatype, op, context);
+ return psmx2_atomic_readwrite_generic(ep, buf, count, src_desc, resultv[0].addr,
+ result_desc ? result_desc[0] : NULL,
+ dest_addr, addr, key, datatype, op,
+ context, ep_priv->tx_flags);
}
ssize_t psmx2_atomic_compwrite_generic(struct fid_ep *ep,
@@ -1242,17 +1699,17 @@ ssize_t psmx2_atomic_compwrite_generic(struct fid_ep *ep,
req = malloc(sizeof(*req) + len + len);
if (!req)
return -FI_ENOMEM;
- memset((void *)req, 0, sizeof(*req));
- memcpy((void *)req + sizeof(*req), (void *)buf, len);
- memcpy((void *)req + sizeof(*req) + len, (void *)compare, len);
- buf = (void *)req + sizeof(*req);
- compare = buf + len;
+ memset(req, 0, sizeof(*req));
+ memcpy((uint8_t *)req + sizeof(*req), (void *)buf, len);
+ memcpy((uint8_t *)req + sizeof(*req) + len, (void *)compare, len);
+ buf = (uint8_t *)req + sizeof(*req);
+ compare = (uint8_t *)buf + len;
} else {
req = calloc(1, sizeof(*req));
if (!req)
return -FI_ENOMEM;
- if (compare != buf + len) {
+ if ((uintptr_t)compare != (uintptr_t)buf + len) {
tmp_buf = malloc(len * 2);
if (!tmp_buf) {
free(req);
@@ -1260,7 +1717,7 @@ ssize_t psmx2_atomic_compwrite_generic(struct fid_ep *ep,
}
memcpy(tmp_buf, buf, len);
- memcpy(tmp_buf + len, compare, len);
+ memcpy((uint8_t *)tmp_buf + len, compare, len);
}
}
@@ -1274,6 +1731,7 @@ ssize_t psmx2_atomic_compwrite_generic(struct fid_ep *ep,
req->atomic.key = key;
req->atomic.context = context;
req->atomic.result = result;
+ req->atomic.datatype = datatype;
req->ep = ep_priv;
req->cq_flags = FI_WRITE | FI_ATOMIC;
@@ -1294,6 +1752,234 @@ ssize_t psmx2_atomic_compwrite_generic(struct fid_ep *ep,
return 0;
}
+ssize_t psmx2_atomic_compwritev_generic(struct fid_ep *ep,
+ const struct fi_ioc *iov,
+ void **desc, size_t count,
+ const struct fi_ioc *comparev,
+ void **compare_desc,
+ size_t compare_count,
+ struct fi_ioc *resultv,
+ void **result_desc,
+ size_t result_count,
+ fi_addr_t dest_addr,
+ uint64_t addr, uint64_t key,
+ enum fi_datatype datatype,
+ enum fi_op op, void *context,
+ uint64_t flags)
+{
+ struct psmx2_fid_ep *ep_priv;
+ struct psmx2_fid_av *av;
+ struct psmx2_epaddr_context *epaddr_context;
+ struct psmx2_am_request *req;
+ psm2_amarg_t args[8];
+ psm2_epaddr_t psm2_epaddr;
+ uint8_t vlane;
+ int am_flags = PSM2_AM_FLAG_ASYNC;
+ int chunk_size;
+ size_t idx;
+ size_t len, compare_len, result_len, iov_size;
+ uint8_t *buf, *compare, *result;
+ void *desc0, *compare_desc0, *result_desc0;
+ int err;
+
+ ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
+
+ if (flags & FI_TRIGGER) {
+ struct psmx2_trigger *trigger;
+ struct fi_triggered_context *ctxt = context;
+
+ trigger = calloc(1, sizeof(*trigger));
+ if (!trigger)
+ return -FI_ENOMEM;
+
+ trigger->op = PSMX2_TRIGGERED_ATOMIC_COMPWRITEV;
+ trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
+ struct psmx2_fid_cntr, cntr);
+ trigger->threshold = ctxt->trigger.threshold.threshold;
+ trigger->atomic_compwritev.ep = ep;
+ trigger->atomic_compwritev.iov = iov;
+ trigger->atomic_compwritev.desc = desc;
+ trigger->atomic_compwritev.count = count;
+ trigger->atomic_compwritev.comparev = comparev;
+ trigger->atomic_compwritev.compare_desc = compare_desc;
+ trigger->atomic_compwritev.compare_count = compare_count;
+ trigger->atomic_compwritev.resultv = resultv;
+ trigger->atomic_compwritev.result_desc = result_desc;
+ trigger->atomic_compwritev.result_count = result_count;
+ trigger->atomic_compwritev.dest_addr = dest_addr;
+ trigger->atomic_compwritev.addr = addr;
+ trigger->atomic_compwritev.key = key;
+ trigger->atomic_compwritev.datatype = datatype;
+ trigger->atomic_compwritev.atomic_op = op;
+ trigger->atomic_compwritev.context = context;
+ trigger->atomic_compwritev.flags = flags & ~FI_TRIGGER;
+
+ psmx2_cntr_add_trigger(trigger->cntr, trigger);
+ return 0;
+ }
+
+ if (!iov || !count || !comparev || !compare_count || !resultv ||
+ !result_count)
+ return -FI_EINVAL;
+
+ while (count && !iov[count-1].count)
+ count--;
+
+ while (compare_count && !comparev[compare_count-1].count)
+ compare_count--;
+
+ while (result_count && !resultv[result_count-1].count)
+ result_count--;
+
+ if (datatype < 0 || datatype >= FI_DATATYPE_LAST)
+ return -FI_EINVAL;
+
+ if (op < 0 || op >= FI_ATOMIC_OP_LAST)
+ return -FI_EINVAL;
+
+ len = psmx2_ioc_size(iov, count, datatype);
+ compare_len = psmx2_ioc_size(comparev, compare_count, datatype);
+ result_len = psmx2_ioc_size(resultv, result_count, datatype);
+
+ if (compare_len < len || result_len < len)
+ return -FI_EINVAL;
+
+ av = ep_priv->av;
+ if (av && av->type == FI_AV_TABLE) {
+ idx = dest_addr;
+ if (idx >= av->last)
+ return -FI_EINVAL;
+
+ psm2_epaddr = av->epaddrs[idx];
+ vlane = av->vlanes[idx];
+ } else {
+ if (!dest_addr)
+ return -FI_EINVAL;
+
+ psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr);
+ vlane = PSMX2_ADDR_TO_VL(dest_addr);
+ }
+
+ epaddr_context = psm2_epaddr_getctxt((void *)psm2_epaddr);
+ if (epaddr_context->epid == ep_priv->domain->psm2_epid) {
+ if (count > 1) {
+ buf = malloc(len);
+ if (!buf)
+ return -FI_ENOMEM;
+ psmx2_ioc_read(iov, count, datatype, buf, len);
+ desc0 = NULL;
+ } else {
+ buf = iov[0].addr;
+ desc0 = desc ? desc[0] : NULL;
+ }
+
+ if (compare_count > 1) {
+ compare = malloc(len);
+ if (!compare) {
+ if (count > 1)
+ free(buf);
+ return -FI_ENOMEM;
+ }
+ psmx2_ioc_read(comparev, compare_count, datatype, compare, len);
+ compare_desc0 = NULL;
+ } else {
+ compare = comparev[0].addr;
+ compare_desc0 = compare_desc ? compare_desc[0] : NULL;
+ }
+
+ if (result_count > 1) {
+ result = malloc(len);
+ if (!result) {
+ if (compare_count > 1)
+ free(compare);
+ if (count > 1)
+ free(buf);
+ return -FI_ENOMEM;
+ }
+ result_desc0 = NULL;
+ } else {
+ result = resultv[0].addr;
+ result_desc0 = result_desc ? result_desc[0] : NULL;
+ }
+
+ err = psmx2_atomic_self(PSMX2_AM_REQ_ATOMIC_COMPWRITE,
+ ep_priv, ep_priv->domain->eps[vlane],
+ buf, len / fi_datatype_size(datatype), desc0,
+ compare, compare_desc0, result, result_desc0,
+ addr, key, datatype, op, context, flags);
+
+ if (result_count > 1) {
+ psmx2_ioc_write(resultv, result_count, datatype, result, len);
+ free(result);
+ }
+
+ if (compare_count > 1)
+ free(compare);
+
+ if (count > 1)
+ free(buf);
+
+ return err;
+ }
+
+ chunk_size = psmx2_am_param.max_request_short;
+ if (len * 2 > chunk_size)
+ return -FI_EMSGSIZE;
+
+ iov_size = result_count > 1 ? result_count * sizeof(struct fi_ioc) : 0;
+
+ if ((flags & FI_INJECT) || count > 1 || compare_count > 1 ||
+ (uintptr_t)comparev[0].addr != (uintptr_t)iov[0].addr + len) {
+ req = malloc(sizeof(*req) + iov_size + len + len);
+ if (!req)
+ return -FI_ENOMEM;
+ buf = (uint8_t *)req + sizeof(*req) + iov_size;
+ memset(req, 0, sizeof(*req));
+ psmx2_ioc_read(iov, count, datatype, buf, len);
+ psmx2_ioc_read(comparev, compare_count, datatype, buf + len, len);
+ } else {
+ req = calloc(1, sizeof(*req) + iov_size);
+ if (!req)
+ return -FI_ENOMEM;
+ buf = iov[0].addr;
+ }
+
+ if (iov_size) {
+ memcpy(req->ioc, resultv, iov_size);
+ req->atomic.iov_count = result_count;
+ req->atomic.result = NULL;
+ } else {
+ req->atomic.buf = buf;
+ req->atomic.result = resultv[0].addr;
+ }
+
+ req->no_event = (flags & PSMX2_NO_COMPLETION) ||
+ (ep_priv->send_selective_completion && !(flags & FI_COMPLETION));
+
+ req->op = PSMX2_AM_REQ_ATOMIC_COMPWRITE;
+ req->atomic.len = len;
+ req->atomic.addr = addr;
+ req->atomic.key = key;
+ req->atomic.context = context;
+ req->atomic.datatype = datatype;
+ req->ep = ep_priv;
+ req->cq_flags = FI_WRITE | FI_ATOMIC;
+
+ args[0].u32w0 = PSMX2_AM_REQ_ATOMIC_COMPWRITE;
+ PSMX2_AM_SET_DST(args[0].u32w0, vlane);
+ args[0].u32w1 = len / fi_datatype_size(datatype);
+ args[1].u64 = (uint64_t)(uintptr_t)req;
+ args[2].u64 = addr;
+ args[3].u64 = key;
+ args[4].u32w0 = datatype;
+ args[4].u32w1 = op;
+ psm2_am_request_short(psm2_epaddr,
+ PSMX2_AM_ATOMIC_HANDLER, args, 5,
+ buf, len * 2, am_flags, NULL, NULL);
+
+ return 0;
+}
+
static ssize_t psmx2_atomic_compwrite(struct fid_ep *ep,
const void *buf,
size_t count, void *desc,
@@ -1311,7 +1997,7 @@ static ssize_t psmx2_atomic_compwrite(struct fid_ep *ep,
compare, compare_desc,
result, result_desc,
dest_addr, addr, key,
- datatype, op, context, ep_priv->flags);
+ datatype, op, context, ep_priv->tx_flags);
}
static ssize_t psmx2_atomic_compwritemsg(struct fid_ep *ep,
@@ -1324,9 +2010,20 @@ static ssize_t psmx2_atomic_compwritemsg(struct fid_ep *ep,
size_t result_count,
uint64_t flags)
{
- if (!msg || msg->iov_count != 1 || !msg->msg_iov || !msg->rma_iov || !resultv)
+ if (!msg || !msg->msg_iov || msg->iov_count != 1 ||
+ !msg->rma_iov || msg->rma_iov_count != 1 ||
+ !comparev || !compare_count || !resultv || !result_count)
return -FI_EINVAL;
+ if (msg->iov_count > 1 || compare_count > 1 || result_count > 1)
+ return psmx2_atomic_compwritev_generic(ep, msg->msg_iov, msg->desc,
+ msg->iov_count, comparev,
+ compare_desc, compare_count,
+ resultv, result_desc, result_count,
+ msg->addr, msg->rma_iov[0].addr,
+ msg->rma_iov[0].key, msg->datatype,
+ msg->op, msg->context, flags);
+
return psmx2_atomic_compwrite_generic(ep, msg->msg_iov[0].addr,
msg->msg_iov[0].count,
msg->desc ? msg->desc[0] : NULL,
@@ -1353,16 +2050,30 @@ static ssize_t psmx2_atomic_compwritev(struct fid_ep *ep,
enum fi_datatype datatype,
enum fi_op op, void *context)
{
- if (!iov || count != 1 || !comparev || !resultv)
+ struct psmx2_fid_ep *ep_priv;
+
+ ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
+
+ if (!iov || !count || !comparev || !compare_count || !resultv || !result_count)
return -FI_EINVAL;
- return psmx2_atomic_compwrite(ep, iov->addr, iov->count,
- desc ? desc[0] : NULL,
- comparev[0].addr,
- compare_desc ? compare_desc[0] : NULL,
- resultv[0].addr,
- result_desc ? result_desc[0] : NULL,
- dest_addr, addr, key, datatype, op, context);
+ if (count > 1 || compare_count > 1 || result_count > 1)
+ return psmx2_atomic_compwritev_generic(ep, iov, desc, count,
+ comparev, compare_desc,
+ compare_count, resultv,
+ result_desc, result_count,
+ dest_addr, addr, key,
+ datatype, op, context,
+ ep_priv->tx_flags);
+
+ return psmx2_atomic_compwrite_generic(ep, iov->addr, iov->count,
+ desc ? desc[0] : NULL,
+ comparev[0].addr,
+ compare_desc ? compare_desc[0] : NULL,
+ resultv[0].addr,
+ result_desc ? result_desc[0] : NULL,
+ dest_addr, addr, key, datatype, op,
+ context, ep_priv->tx_flags);
}
static int psmx2_atomic_writevalid(struct fid_ep *ep,
diff --git a/prov/psm2/src/psmx2_av.c b/prov/psm2/src/psmx2_av.c
index 49f16d0..9deb7bd 100644
--- a/prov/psm2/src/psmx2_av.c
+++ b/prov/psm2/src/psmx2_av.c
@@ -241,6 +241,12 @@ static int psmx2_av_insert(struct fid_av *av, const void *addr,
int error_count;
int i;
+ if (count && !addr) {
+ FI_INFO(&psmx2_prov, FI_LOG_AV,
+ "the input address array is NULL.\n");
+ return -FI_EINVAL;
+ }
+
av_priv = container_of(av, struct psmx2_fid_av, av);
if ((av_priv->flags & FI_EVENT) && !av_priv->eq)
@@ -313,6 +319,7 @@ static int psmx2_av_lookup(struct fid_av *av, fi_addr_t fi_addr, void *addr,
av_priv = container_of(av, struct psmx2_fid_av, av);
+ memset(&name, 0, sizeof(name));
if (av_priv->type == FI_AV_TABLE) {
idx = (int)(int64_t)fi_addr;
if (idx >= av_priv->last)
@@ -417,6 +424,9 @@ int psmx2_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
if (attr) {
switch (attr->type) {
+ case FI_AV_UNSPEC:
+ break;
+
case FI_AV_MAP:
case FI_AV_TABLE:
type = attr->type;
@@ -435,7 +445,14 @@ int psmx2_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
FI_INFO(&psmx2_prov, FI_LOG_AV,
"attr->flags=%x, supported=%x\n",
attr->flags, FI_EVENT);
- return -FI_EINVAL;
+ return -FI_ENOSYS;
+ }
+
+ if (attr->name) {
+ FI_INFO(&psmx2_prov, FI_LOG_AV,
+ "attr->name=%s, named AV is not supported\n",
+ attr->name);
+ return -FI_ENOSYS;
}
}
@@ -457,6 +474,9 @@ int psmx2_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
av_priv->av.ops = &psmx2_av_ops;
*av = &av_priv->av;
+ if (attr)
+ attr->type = type;
+
return 0;
}
diff --git a/prov/psm2/src/psmx2_cm.c b/prov/psm2/src/psmx2_cm.c
index a8fd6f8..69acc0e 100644
--- a/prov/psm2/src/psmx2_cm.c
+++ b/prov/psm2/src/psmx2_cm.c
@@ -46,6 +46,7 @@ static int psmx2_cm_getname(fid_t fid, void *addr, size_t *addrlen)
return -FI_ETOOSMALL;
}
+ memset(epname, 0, sizeof(*epname));
epname->epid = ep->domain->psm2_epid;
epname->vlane = ep->vlane;
*addrlen = sizeof(struct psmx2_ep_name);
diff --git a/prov/psm2/src/psmx2_cntr.c b/prov/psm2/src/psmx2_cntr.c
index a89d137..49352ae 100644
--- a/prov/psm2/src/psmx2_cntr.c
+++ b/prov/psm2/src/psmx2_cntr.c
@@ -136,6 +136,18 @@ int psmx2_process_trigger(struct psmx2_fid_domain *domain,
trigger->read.flags);
break;
+ case PSMX2_TRIGGERED_READV:
+ psmx2_readv_generic(trigger->readv.ep,
+ trigger->readv.iov,
+ trigger->readv.desc,
+ trigger->readv.count,
+ trigger->readv.src_addr,
+ trigger->readv.addr,
+ trigger->readv.key,
+ trigger->readv.context,
+ trigger->readv.flags);
+ break;
+
case PSMX2_TRIGGERED_ATOMIC_WRITE:
psmx2_atomic_write_generic(
trigger->atomic_write.ep,
@@ -208,7 +220,7 @@ void psmx2_cntr_check_trigger(struct psmx2_fid_cntr *cntr)
trigger = cntr->trigger;
while (trigger) {
- if (cntr->counter < trigger->threshold)
+ if (atomic_get(&cntr->counter) < trigger->threshold)
break;
cntr->trigger = trigger->next;
@@ -265,9 +277,7 @@ static uint64_t psmx2_cntr_read(struct fid_cntr *cntr)
poll_cnt = 0;
}
- cntr_priv->counter_last_read = cntr_priv->counter;
-
- return cntr_priv->counter_last_read;
+ return atomic_get(&cntr_priv->counter);
}
static uint64_t psmx2_cntr_readerr(struct fid_cntr *cntr)
@@ -276,9 +286,7 @@ static uint64_t psmx2_cntr_readerr(struct fid_cntr *cntr)
cntr_priv = container_of(cntr, struct psmx2_fid_cntr, cntr);
- cntr_priv->error_counter_last_read = cntr_priv->error_counter;
-
- return cntr_priv->error_counter_last_read;
+ return atomic_get(&cntr_priv->error_counter);
}
static int psmx2_cntr_add(struct fid_cntr *cntr, uint64_t value)
@@ -286,7 +294,7 @@ static int psmx2_cntr_add(struct fid_cntr *cntr, uint64_t value)
struct psmx2_fid_cntr *cntr_priv;
cntr_priv = container_of(cntr, struct psmx2_fid_cntr, cntr);
- cntr_priv->counter += value;
+ atomic_add(&cntr_priv->counter, value);
psmx2_cntr_check_trigger(cntr_priv);
@@ -301,7 +309,7 @@ static int psmx2_cntr_set(struct fid_cntr *cntr, uint64_t value)
struct psmx2_fid_cntr *cntr_priv;
cntr_priv = container_of(cntr, struct psmx2_fid_cntr, cntr);
- cntr_priv->counter = value;
+ atomic_set(&cntr_priv->counter, value);
psmx2_cntr_check_trigger(cntr_priv);
@@ -322,7 +330,7 @@ static int psmx2_cntr_wait(struct fid_cntr *cntr, uint64_t threshold, int timeou
clock_gettime(CLOCK_REALTIME, &ts0);
- while (cntr_priv->counter < threshold) {
+ while (atomic_get(&cntr_priv->counter) < threshold) {
if (cntr_priv->wait) {
ret = fi_wait((struct fid_wait *)cntr_priv->wait,
timeout - msec_passed);
@@ -332,7 +340,7 @@ static int psmx2_cntr_wait(struct fid_cntr *cntr, uint64_t threshold, int timeou
psmx2_progress(cntr_priv->domain);
}
- if (cntr_priv->counter >= threshold)
+ if (atomic_get(&cntr_priv->counter) >= threshold)
break;
if (timeout < 0)
@@ -389,7 +397,10 @@ static int psmx2_cntr_control(fid_t fid, int command, void *arg)
break;
case FI_GETWAIT:
- ret = fi_control(&cntr->wait->wait_fid.fid, FI_GETWAIT, arg);
+ if (cntr->wait)
+ ret = fi_control(&cntr->wait->wait_fid.fid, FI_GETWAIT, arg);
+ else
+ return -FI_EINVAL;
break;
default:
return -FI_ENOSYS;
@@ -493,6 +504,8 @@ int psmx2_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
cntr_priv->cntr.fid.context = context;
cntr_priv->cntr.fid.ops = &psmx2_fi_ops;
cntr_priv->cntr.ops = &psmx2_cntr_ops;
+ atomic_initialize(&cntr_priv->counter, 0);
+ atomic_initialize(&cntr_priv->error_counter, 0);
pthread_mutex_init(&cntr_priv->trigger_lock, NULL);
diff --git a/prov/psm2/src/psmx2_cq.c b/prov/psm2/src/psmx2_cq.c
index 0567187..6d783fc 100644
--- a/prov/psm2/src/psmx2_cq.c
+++ b/prov/psm2/src/psmx2_cq.c
@@ -294,7 +294,7 @@ psmx2_cq_create_event_from_status(struct psmx2_fid_cq *cq,
event->cqe.tagged.buf = buf;
event->cqe.tagged.flags = flags;
event->cqe.tagged.len = psm2_status->nbytes;
- event->cqe.data.data =
+ event->cqe.tagged.data =
(psm2_status->msg_tag.tag2 & PSMX2_MSG_BIT) ?
PSMX2_GET_TAG64(psm2_status->msg_tag) : 0;
event->cqe.tagged.tag = PSMX2_GET_TAG64(psm2_status->msg_tag);
@@ -337,6 +337,7 @@ int psmx2_cq_poll_mq(struct psmx2_fid_cq *cq,
struct psmx2_fid_cq *tmp_cq;
struct psmx2_fid_cntr *tmp_cntr;
struct psmx2_cq_event *event;
+ struct psmx2_am_request *read_req;
int multi_recv;
int err;
int read_more = 1;
@@ -371,10 +372,21 @@ int psmx2_cq_poll_mq(struct psmx2_fid_cq *cq,
multi_recv = 0;
switch ((int)PSMX2_CTXT_TYPE(fi_context)) {
+ case PSMX2_SEND_CONTEXT:
+ case PSMX2_TSEND_CONTEXT:
+ tmp_cq = tmp_ep->send_cq;
+ /* Fall through */
case PSMX2_NOCOMP_SEND_CONTEXT:
tmp_cntr = tmp_ep->send_cntr;
break;
+ case PSMX2_RECV_CONTEXT:
+ case PSMX2_TRECV_CONTEXT:
+ if ((psm2_status.msg_tag.tag2 & PSMX2_IOV_BIT) &&
+ !psmx2_handle_sendv_req(tmp_ep, &psm2_status, 0))
+ continue;
+ tmp_cq = tmp_ep->recv_cq;
+ /* Fall through */
case PSMX2_NOCOMP_RECV_CONTEXT:
tmp_cntr = tmp_ep->recv_cntr;
break;
@@ -389,27 +401,29 @@ int psmx2_cq_poll_mq(struct psmx2_fid_cq *cq,
psmx2_ep_put_op_context(tmp_ep, fi_context);
break;
+ case PSMX2_WRITE_CONTEXT:
+ tmp_cq = tmp_ep->send_cq;
+ /* Fall through */
case PSMX2_NOCOMP_WRITE_CONTEXT:
tmp_cntr = tmp_ep->write_cntr;
break;
+ case PSMX2_READ_CONTEXT:
+ tmp_cq = tmp_ep->send_cq;
+ /* Fall throigh */
case PSMX2_NOCOMP_READ_CONTEXT:
+ read_req = container_of(fi_context, struct psmx2_am_request,
+ fi_context);
tmp_cntr = tmp_ep->read_cntr;
- break;
-
- case PSMX2_SEND_CONTEXT:
- case PSMX2_TSEND_CONTEXT:
- tmp_cq = tmp_ep->send_cq;
- tmp_cntr = tmp_ep->send_cntr;
- break;
-
- case PSMX2_RECV_CONTEXT:
- case PSMX2_TRECV_CONTEXT:
- if ((psm2_status.msg_tag.tag2 & PSMX2_IOV_BIT) &&
- !psmx2_handle_sendv_req(tmp_ep, &psm2_status, 0))
- continue;
- tmp_cq = tmp_ep->recv_cq;
- tmp_cntr = tmp_ep->recv_cntr;
+ if (read_req->op == PSMX2_AM_REQ_READV) {
+ read_req->read.len_read += psm2_status.nbytes;
+ if (read_req->read.len_read < read_req->read.len) {
+ FI_INFO(&psmx2_prov, FI_LOG_EP_DATA,
+ "readv: long protocol finishes early\n");
+ tmp_cq = NULL;
+ tmp_cntr = NULL;
+ }
+ }
break;
case PSMX2_MULTI_RECV_CONTEXT:
@@ -421,16 +435,6 @@ int psmx2_cq_poll_mq(struct psmx2_fid_cq *cq,
tmp_cntr = tmp_ep->recv_cntr;
break;
- case PSMX2_READ_CONTEXT:
- tmp_cq = tmp_ep->send_cq;
- tmp_cntr = tmp_ep->read_cntr;
- break;
-
- case PSMX2_WRITE_CONTEXT:
- tmp_cq = tmp_ep->send_cq;
- tmp_cntr = tmp_ep->write_cntr;
- break;
-
case PSMX2_REMOTE_WRITE_CONTEXT:
{
struct fi_context *fi_context = psm2_status.context;
@@ -457,7 +461,9 @@ int psmx2_cq_poll_mq(struct psmx2_fid_cq *cq,
if (event == event_buffer) {
read_count++;
read_more = --count;
- event_buffer = count ? event_buffer + cq->entry_size : NULL;
+ event_buffer = count ?
+ (uint8_t *)event_buffer + cq->entry_size :
+ NULL;
if (src_addr)
src_addr = count ? src_addr + 1 : NULL;
} else {
@@ -558,7 +564,9 @@ int psmx2_cq_poll_mq(struct psmx2_fid_cq *cq,
if (event == event_buffer) {
read_count++;
read_more = --count;
- event_buffer = count ? event_buffer + cq->entry_size : NULL;
+ event_buffer = count ?
+ (uint8_t *)event_buffer + cq->entry_size :
+ NULL;
if (src_addr)
src_addr = count ? src_addr + 1 : NULL;
} else {
@@ -574,15 +582,19 @@ int psmx2_cq_poll_mq(struct psmx2_fid_cq *cq,
if (multi_recv) {
struct psmx2_multi_recv *req;
psm2_mq_req_t psm2_req;
+ size_t len_remaining;
req = PSMX2_CTXT_USER(fi_context);
req->offset += psm2_status.nbytes;
- if (req->offset + req->min_buf_size <= req->len) {
+ len_remaining = req->len - req->offset;
+ if (len_remaining >= req->min_buf_size) {
+ if (len_remaining > PSMX2_MAX_MSG_SIZE)
+ len_remaining = PSMX2_MAX_MSG_SIZE;
err = psm2_mq_irecv2(tmp_ep->domain->psm2_mq,
req->src_addr, &req->tag,
&req->tagsel, req->flag,
req->buf + req->offset,
- req->len - req->offset,
+ len_remaining,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
@@ -614,6 +626,7 @@ static ssize_t psmx2_cq_readfrom(struct fid_cq *cq, void *buf, size_t count,
struct psmx2_cq_event *event;
int ret;
ssize_t read_count;
+ int i;
cq_priv = container_of(cq, struct psmx2_fid_cq, cq);
@@ -634,7 +647,7 @@ static ssize_t psmx2_cq_readfrom(struct fid_cq *cq, void *buf, size_t count,
return -FI_EINVAL;
read_count = 0;
- while (count--) {
+ for (i = 0; i < count; i++) {
event = psmx2_cq_dequeue_event(cq_priv);
if (event) {
if (!event->error) {
@@ -645,7 +658,7 @@ static ssize_t psmx2_cq_readfrom(struct fid_cq *cq, void *buf, size_t count,
psmx2_cq_free_event(cq_priv, event);
read_count++;
- buf += cq_priv->entry_size;
+ buf = (uint8_t *)buf + cq_priv->entry_size;
if (src_addr)
src_addr++;
continue;
@@ -660,7 +673,11 @@ static ssize_t psmx2_cq_readfrom(struct fid_cq *cq, void *buf, size_t count,
}
}
- if (!read_count && slist_empty(&cq_priv->event_queue))
+ /*
+ * Return 0 if and only if the input count is 0 and the CQ is not empty.
+ * This is used by the util poll code to check the poll state.
+ */
+ if (!read_count && (count || slist_empty(&cq_priv->event_queue)))
read_count = -FI_EAGAIN;
return read_count;
@@ -794,7 +811,10 @@ static int psmx2_cq_control(struct fid *fid, int command, void *arg)
switch (command) {
case FI_GETWAIT:
- ret = fi_control(&cq->wait->wait_fid.fid, FI_GETWAIT, arg);
+ if (cq->wait)
+ ret = fi_control(&cq->wait->wait_fid.fid, FI_GETWAIT, arg);
+ else
+ return -FI_EINVAL;
break;
default:
@@ -802,7 +822,6 @@ static int psmx2_cq_control(struct fid *fid, int command, void *arg)
}
return ret;
- return -FI_ENOSYS;
}
static struct fi_ops psmx2_fi_ops = {
diff --git a/prov/psm2/src/psmx2_domain.c b/prov/psm2/src/psmx2_domain.c
index 1ed6a3f..32ee000 100644
--- a/prov/psm2/src/psmx2_domain.c
+++ b/prov/psm2/src/psmx2_domain.c
@@ -187,7 +187,7 @@ static int psmx2_domain_close(fid_t fid)
psmx2_domain_release(domain);
- if (util_domain_close(&domain->util_domain))
+ if (ofi_domain_close(&domain->util_domain))
return 0;
if (domain->progress_thread_enabled)
@@ -389,11 +389,11 @@ int psmx2_domain_open(struct fid_fabric *fabric, struct fi_info *info,
goto err_out;
}
- err = fi_domain_init(fabric, info, &domain_priv->util_domain, context);
+ err = ofi_domain_init(fabric, info, &domain_priv->util_domain, context);
if (err)
goto err_out_free_domain;
- /* fclass & context are set in fi_domain_init */
+ /* fclass & context are set in ofi_domain_init */
domain_priv->util_domain.domain_fid.fid.ops = &psmx2_fi_ops;
domain_priv->util_domain.domain_fid.ops = &psmx2_domain_ops;
domain_priv->util_domain.domain_fid.mr = &psmx2_mr_ops;
@@ -415,7 +415,7 @@ int psmx2_domain_open(struct fid_fabric *fabric, struct fi_info *info,
return 0;
err_out_close_domain:
- util_domain_close(&domain_priv->util_domain);
+ ofi_domain_close(&domain_priv->util_domain);
err_out_free_domain:
free(domain_priv);
diff --git a/prov/psm2/src/psmx2_ep.c b/prov/psm2/src/psmx2_ep.c
index fa4513d..a645faf 100644
--- a/prov/psm2/src/psmx2_ep.c
+++ b/prov/psm2/src/psmx2_ep.c
@@ -98,7 +98,7 @@ static int psmx2_alloc_vlane(struct psmx2_fid_domain *domain, uint8_t *vl)
static void psmx2_ep_optimize_ops(struct psmx2_fid_ep *ep)
{
if (ep->ep.tagged) {
- if (ep->flags) {
+ if (ep->tx_flags | ep->rx_flags) {
ep->ep.tagged = &psmx2_tagged_ops;
FI_INFO(&psmx2_prov, FI_LOG_EP_DATA,
"generic tagged ops.\n");
@@ -239,6 +239,14 @@ static int psmx2_ep_close(fid_t fid)
ep = container_of(fid, struct psmx2_fid_ep, ep.fid);
+ if (ep->base_ep) {
+ atomic_dec(&ep->base_ep->ref);
+ return 0;
+ }
+
+ if (atomic_get(&ep->ref))
+ return -FI_EBUSY;
+
ep->domain->eps[ep->vlane] = NULL;
psmx2_free_vlane(ep->domain, ep->vlane);
psmx2_domain_release(ep->domain);
@@ -266,9 +274,10 @@ static int psmx2_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
int err;
ep = container_of(fid, struct psmx2_fid_ep, ep.fid);
+ err = ofi_ep_bind_valid(&psmx2_prov, bfid, flags);
+ if (err)
+ return err;
- if (!bfid)
- return -FI_EINVAL;
switch (bfid->fclass) {
case FI_CLASS_EQ:
return -FI_ENOSYS;
@@ -339,10 +348,44 @@ static int psmx2_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
return 0;
}
+static inline int psmx2_ep_set_flags(struct psmx2_fid_ep *ep, uint64_t flags)
+{
+ uint64_t real_flags = flags & ~(FI_TRANSMIT | FI_RECV);
+
+ if ((flags & FI_TRANSMIT) && (flags & FI_RECV))
+ return -EINVAL;
+ else if (flags & FI_TRANSMIT)
+ ep->tx_flags = real_flags;
+ else if (flags & FI_RECV)
+ ep->rx_flags = real_flags;
+
+ /* otherwise ok to leave the flags intact */
+
+ return 0;
+}
+
+static inline int psmx2_ep_get_flags(struct psmx2_fid_ep *ep, uint64_t *flags)
+{
+ uint64_t flags_in = *flags;
+
+ if ((flags_in & FI_TRANSMIT) && (flags_in & FI_RECV))
+ return -EINVAL;
+ else if (flags_in & FI_TRANSMIT)
+ *flags = ep->tx_flags;
+ else if (flags_in & FI_RECV)
+ *flags = ep->rx_flags;
+ else
+ return -EINVAL;
+
+ return 0;
+}
+
static int psmx2_ep_control(fid_t fid, int command, void *arg)
{
struct fi_alias *alias;
struct psmx2_fid_ep *ep, *new_ep;
+ int err;
+
ep = container_of(fid, struct psmx2_fid_ep, ep.fid);
switch (command) {
@@ -352,23 +395,34 @@ static int psmx2_ep_control(fid_t fid, int command, void *arg)
return -FI_ENOMEM;
alias = arg;
*new_ep = *ep;
- new_ep->flags = alias->flags;
+ err = psmx2_ep_set_flags(new_ep, alias->flags);
+ if (err) {
+ free(new_ep);
+ return err;
+ }
+ new_ep->base_ep = ep;
+ atomic_inc(&ep->ref);
psmx2_ep_optimize_ops(new_ep);
*alias->fid = &new_ep->ep.fid;
break;
- case FI_SETFIDFLAG:
- ep->flags = *(uint64_t *)arg;
+ case FI_SETOPSFLAG:
+ err = psmx2_ep_set_flags(ep, *(uint64_t *)arg);
+ if (err)
+ return err;
psmx2_ep_optimize_ops(ep);
break;
- case FI_GETFIDFLAG:
+ case FI_GETOPSFLAG:
if (!arg)
return -FI_EINVAL;
- *(uint64_t *)arg = ep->flags;
+ err = psmx2_ep_get_flags(ep, arg);
+ if (err)
+ return err;
break;
case FI_ENABLE:
+ ep->enabled = 1;
return 0;
default:
@@ -380,12 +434,24 @@ static int psmx2_ep_control(fid_t fid, int command, void *arg)
static ssize_t psmx2_rx_size_left(struct fid_ep *ep)
{
- return 0x7fffffff; /* a random choice */
+ struct psmx2_fid_ep *ep_priv;
+
+ ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
+ if (ep_priv->enabled)
+ return 0x7fffffff;
+ else
+ return -FI_EOPBADSTATE;
}
static ssize_t psmx2_tx_size_left(struct fid_ep *ep)
{
- return 0x7fffffff; /* a random choice */
+ struct psmx2_fid_ep *ep_priv;
+
+ ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
+ if (ep_priv->enabled)
+ return 0x7fffffff;
+ else
+ return -FI_EOPBADSTATE;
}
static struct fi_ops psmx2_fi_ops = {
@@ -449,6 +515,7 @@ int psmx2_ep_open(struct fid_domain *domain, struct fi_info *info,
ep_priv->ep.cm = &psmx2_cm_ops;
ep_priv->domain = domain_priv;
ep_priv->vlane = vlane;
+ atomic_initialize(&ep_priv->ref, 0);
PSMX2_CTXT_TYPE(&ep_priv->nocomp_send_context) = PSMX2_NOCOMP_SEND_CONTEXT;
PSMX2_CTXT_EP(&ep_priv->nocomp_send_context) = ep_priv;
@@ -475,9 +542,9 @@ int psmx2_ep_open(struct fid_domain *domain, struct fi_info *info,
if (info) {
if (info->tx_attr)
- ep_priv->flags = info->tx_attr->op_flags;
+ ep_priv->tx_flags = info->tx_attr->op_flags;
if (info->rx_attr)
- ep_priv->flags |= info->rx_attr->op_flags;
+ ep_priv->rx_flags = info->rx_attr->op_flags;
}
psmx2_ep_optimize_ops(ep_priv);
diff --git a/prov/psm2/src/psmx2_fabric.c b/prov/psm2/src/psmx2_fabric.c
index 30a9acf..8754acd 100644
--- a/prov/psm2/src/psmx2_fabric.c
+++ b/prov/psm2/src/psmx2_fabric.c
@@ -48,7 +48,7 @@ static int psmx2_fabric_close(fid_t fid)
psmx2_fabric_release(fabric);
- if (util_fabric_close(&fabric->util_fabric))
+ if (ofi_fabric_close(&fabric->util_fabric))
return 0;
if (psmx2_env.name_server &&
@@ -89,9 +89,9 @@ static struct fi_ops_fabric psmx2_fabric_ops = {
.size = sizeof(struct fi_ops_fabric),
.domain = psmx2_domain_open,
.passive_ep = fi_no_passive_ep,
- .eq_open = fi_eq_create,
+ .eq_open = ofi_eq_create,
.wait_open = psmx2_wait_open,
- .trywait = util_trywait
+ .trywait = psmx2_wait_trywait
};
static struct fi_fabric_attr psmx2_fabric_attr = {
@@ -120,15 +120,16 @@ int psmx2_fabric(struct fi_fabric_attr *attr,
if (!fabric_priv)
return -FI_ENOMEM;
- ret = fi_fabric_init(&psmx2_prov, &psmx2_fabric_attr, attr,
- &fabric_priv->util_fabric, context);
+ ret = ofi_fabric_init(&psmx2_prov, &psmx2_fabric_attr, attr,
+ &fabric_priv->util_fabric, context,
+ FI_MATCH_EXACT);
if (ret) {
- FI_INFO(&psmx2_prov, FI_LOG_CORE, "fi_fabric_init returns %d\n", ret);
+ FI_INFO(&psmx2_prov, FI_LOG_CORE, "ofi_fabric_init returns %d\n", ret);
free(fabric_priv);
return ret;
}
- /* fclass & context initialized in fi_fabric_init */
+ /* fclass & context initialized in ofi_fabric_init */
fabric_priv->util_fabric.fabric_fid.fid.ops = &psmx2_fabric_fi_ops;
fabric_priv->util_fabric.fabric_fid.ops = &psmx2_fabric_ops;
diff --git a/prov/psm2/src/psmx2_init.c b/prov/psm2/src/psmx2_init.c
index 55c3a64..af3114a 100644
--- a/prov/psm2/src/psmx2_init.c
+++ b/prov/psm2/src/psmx2_init.c
@@ -76,12 +76,18 @@ static int psmx2_getinfo(uint32_t version, const char *node,
uint64_t caps = PSMX2_CAPS;
uint64_t max_tag_value = -1ULL;
int err = -FI_ENODATA;
+ struct stat st;
FI_INFO(&psmx2_prov, FI_LOG_CORE,"\n");
*info = NULL;
- if (psm2_ep_num_devunits(&cnt) || !cnt) {
+ /*
+ * psm2_ep_num_devunits() may wait for 15 seconds before return
+ * when /dev/hfi1_0 is not present. Check the existence of this
+ * device first to avoid this delay.
+ */
+ if (stat("/dev/hfi1_0", &st) || psm2_ep_num_devunits(&cnt) || !cnt) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"no PSM device is found.\n");
return -FI_ENODATA;
@@ -91,14 +97,16 @@ static int psmx2_getinfo(uint32_t version, const char *node,
if (node && !(flags & FI_SOURCE)) {
dest_addr = psmx2_resolve_name(node, 0);
- if (dest_addr)
+ if (dest_addr) {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"node '%s' resolved to <epid=0x%llx, vl=%d>\n", node,
((struct psmx2_ep_name *)dest_addr)->epid,
((struct psmx2_ep_name *)dest_addr)->vlane);
- else
+ } else {
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"failed to resolve node '%s'.\n", node);
+ return -FI_ENODATA;
+ }
}
if (hints) {
@@ -413,7 +421,7 @@ static int psmx2_getinfo(uint32_t version, const char *node,
psmx2_info->domain_attr->ep_cnt = 65535;
psmx2_info->domain_attr->tx_ctx_cnt = 1;
psmx2_info->domain_attr->rx_ctx_cnt = 1;
- psmx2_info->domain_attr->max_ep_tx_ctx = 65535;
+ psmx2_info->domain_attr->max_ep_tx_ctx = 1;
psmx2_info->domain_attr->max_ep_rx_ctx = 1;
psmx2_info->domain_attr->max_ep_stx_ctx = 65535;
psmx2_info->domain_attr->max_ep_srx_ctx = 0;
diff --git a/prov/psm2/src/psmx2_mr.c b/prov/psm2/src/psmx2_mr.c
index 27cd8e6..688b97c 100644
--- a/prov/psm2/src/psmx2_mr.c
+++ b/prov/psm2/src/psmx2_mr.c
@@ -36,13 +36,16 @@ struct psmx2_fid_mr *psmx2_mr_get(struct psmx2_fid_domain *domain,
uint64_t key)
{
RbtIterator it;
- struct psmx2_fid_mr *mr;
+ struct psmx2_fid_mr *mr = NULL;
+ fastlock_acquire(&domain->mr_lock);
it = rbtFind(domain->mr_map, (void *)key);
if (!it)
- return NULL;
+ goto exit;
rbtKeyValue(domain->mr_map, it, (void **)&key, (void **)&mr);
+exit:
+ fastlock_release(&domain->mr_lock);
return mr;
}
@@ -174,6 +177,7 @@ static void psmx2_mr_normalize_iov(struct iovec *iov, size_t *count)
{
struct iovec tmp_iov;
int i, j, n, new_len;
+ uintptr_t iov_end_i, iov_end_j;
n = *count;
@@ -200,8 +204,10 @@ static void psmx2_mr_normalize_iov(struct iovec *iov, size_t *count)
if (iov[j].iov_len == 0)
continue;
- if (iov[i].iov_base + iov[i].iov_len >= iov[j].iov_base) {
- new_len = iov[j].iov_base + iov[j].iov_len - iov[i].iov_base;
+ iov_end_i = (uintptr_t)iov[i].iov_base + iov[i].iov_len;
+ iov_end_j = (uintptr_t)iov[j].iov_base + iov[j].iov_len;
+ if (iov_end_i >= (uintptr_t)iov[j].iov_base) {
+ new_len = iov_end_j - (uintptr_t)iov[i].iov_base;
if (new_len > iov[i].iov_len)
iov[i].iov_len = new_len;
iov[j].iov_len = 0;
@@ -262,10 +268,6 @@ static int psmx2_mr_reg(struct fid *fid, const void *buf, size_t len,
mr_priv->mr.fid.context = context;
mr_priv->mr.fid.ops = &psmx2_fi_ops;
mr_priv->mr.mem_desc = mr_priv;
- mr_priv->mr.fid.fclass = FI_CLASS_MR;
- mr_priv->mr.fid.context = context;
- mr_priv->mr.fid.ops = &psmx2_fi_ops;
- mr_priv->mr.mem_desc = mr_priv;
mr_priv->mr.key = key;
mr_priv->domain = domain_priv;
mr_priv->access = access;
diff --git a/prov/psm2/src/psmx2_msg.c b/prov/psm2/src/psmx2_msg.c
index d5efbb0..6d5c130 100644
--- a/prov/psm2/src/psmx2_msg.c
+++ b/prov/psm2/src/psmx2_msg.c
@@ -127,6 +127,8 @@ ssize_t psmx2_recv_generic(struct fid_ep *ep, void *buf, size_t len,
req->context = fi_context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_MULTI_RECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = req;
+ if (len > PSMX2_MAX_MSG_SIZE)
+ len = PSMX2_MAX_MSG_SIZE;
} else {
PSMX2_CTXT_TYPE(fi_context) = PSMX2_RECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = buf;
@@ -155,7 +157,7 @@ static ssize_t psmx2_recv(struct fid_ep *ep, void *buf, size_t len,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_recv_generic(ep, buf, len, desc, src_addr, context,
- ep_priv->flags);
+ ep_priv->rx_flags);
}
static ssize_t psmx2_recvmsg(struct fid_ep *ep, const struct fi_msg *msg,
@@ -168,7 +170,7 @@ static ssize_t psmx2_recvmsg(struct fid_ep *ep, const struct fi_msg *msg,
return -FI_EINVAL;
if (msg->iov_count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (msg->iov_count) {
buf = msg->msg_iov[0].iov_base;
len = msg->msg_iov[0].iov_len;
@@ -193,7 +195,7 @@ static ssize_t psmx2_recvv(struct fid_ep *ep, const struct iovec *iov,
return -FI_EINVAL;
if (count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -532,7 +534,7 @@ int psmx2_handle_sendv_req(struct psmx2_fid_ep *ep,
struct fi_context *fi_context;
struct fi_context *recv_context;
int i, err;
- void *recv_buf;
+ uint8_t *recv_buf;
size_t recv_len, len;
if (psm2_status->error_code != PSM2_OK)
@@ -622,7 +624,7 @@ static ssize_t psmx2_send(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_send_generic(ep, buf, len, desc, dest_addr, context,
- ep_priv->flags, 0);
+ ep_priv->tx_flags, 0);
}
static ssize_t psmx2_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
@@ -670,7 +672,7 @@ static ssize_t psmx2_sendv(struct fid_ep *ep, const struct iovec *iov,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_sendv_generic(ep, iov, desc, count, dest_addr,
- context, ep_priv->flags, 0);
+ context, ep_priv->tx_flags, 0);
} else if (count) {
buf = iov[0].iov_base;
len = iov[0].iov_len;
@@ -691,7 +693,7 @@ static ssize_t psmx2_inject(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_send_generic(ep, buf, len, NULL, dest_addr, NULL,
- ep_priv->flags | FI_INJECT | PSMX2_NO_COMPLETION,
+ ep_priv->tx_flags | FI_INJECT | PSMX2_NO_COMPLETION,
0);
}
@@ -704,7 +706,7 @@ static ssize_t psmx2_senddata(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_send_generic(ep, buf, len, desc, dest_addr, context,
- ep_priv->flags, data);
+ ep_priv->tx_flags, data);
}
static ssize_t psmx2_injectdata(struct fid_ep *ep, const void *buf, size_t len,
@@ -715,7 +717,7 @@ static ssize_t psmx2_injectdata(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_send_generic(ep, buf, len, NULL, dest_addr, NULL,
- ep_priv->flags | FI_INJECT | PSMX2_NO_COMPLETION,
+ ep_priv->tx_flags | FI_INJECT | PSMX2_NO_COMPLETION,
data);
}
diff --git a/prov/psm2/src/psmx2_rma.c b/prov/psm2/src/psmx2_rma.c
index 695c1de..0b5305a 100644
--- a/prov/psm2/src/psmx2_rma.c
+++ b/prov/psm2/src/psmx2_rma.c
@@ -40,6 +40,34 @@ static inline void psmx2_am_enqueue_rma(struct psmx2_fid_domain *domain,
fastlock_release(&domain->rma_queue.lock);
}
+static inline void psmx2_iov_copy(struct iovec *iov, size_t count,
+ size_t offset, const void *src,
+ size_t len)
+{
+ int i;
+ size_t copy_len;
+
+ for (i=0; i<count && len; i++) {
+ if (offset >= iov[i].iov_len) {
+ offset -= iov[i].iov_len;
+ continue;
+ }
+
+ copy_len = iov[i].iov_len - offset;
+
+ if (copy_len > len)
+ copy_len = len;
+
+ memcpy((uint8_t *)iov[i].iov_base + offset, src, copy_len);
+
+ src = (const uint8_t *)src + copy_len;
+ len -= copy_len;
+
+ if (offset)
+ offset = 0;
+ }
+}
+
/* RMA protocol:
*
* Write REQ:
@@ -74,7 +102,7 @@ int psmx2_am_rma_handler(psm2_am_token_t token, psm2_amarg_t *args,
int nargs, void *src, uint32_t len)
{
psm2_amarg_t rep_args[8];
- void *rma_addr;
+ uint8_t *rma_addr;
ssize_t rma_len;
uint64_t key;
int err = 0;
@@ -103,7 +131,7 @@ int psmx2_am_rma_handler(psm2_am_token_t token, psm2_amarg_t *args,
switch (cmd) {
case PSMX2_AM_REQ_WRITE:
rma_len = args[0].u32w1;
- rma_addr = (void *)(uintptr_t)args[2].u64;
+ rma_addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
mr = psmx2_mr_get(domain, key);
op_error = mr ?
@@ -152,7 +180,7 @@ int psmx2_am_rma_handler(psm2_am_token_t token, psm2_amarg_t *args,
case PSMX2_AM_REQ_WRITE_LONG:
src_vl = PSMX2_AM_GET_SRC(args[0].u32w0);
rma_len = args[0].u32w1;
- rma_addr = (void *)(uintptr_t)args[2].u64;
+ rma_addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
mr = psmx2_mr_get(domain, key);
op_error = mr ?
@@ -194,7 +222,7 @@ int psmx2_am_rma_handler(psm2_am_token_t token, psm2_amarg_t *args,
case PSMX2_AM_REQ_READ:
rma_len = args[0].u32w1;
- rma_addr = (void *)(uintptr_t)args[2].u64;
+ rma_addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
offset = args[4].u64;
mr = psmx2_mr_get(domain, key);
@@ -225,7 +253,7 @@ int psmx2_am_rma_handler(psm2_am_token_t token, psm2_amarg_t *args,
case PSMX2_AM_REQ_READ_LONG:
src_vl = PSMX2_AM_GET_SRC(args[0].u32w0);
rma_len = args[0].u32w1;
- rma_addr = (void *)(uintptr_t)args[2].u64;
+ rma_addr = (uint8_t *)(uintptr_t)args[2].u64;
key = args[3].u64;
mr = psmx2_mr_get(domain, key);
op_error = mr ?
@@ -296,16 +324,23 @@ int psmx2_am_rma_handler(psm2_am_token_t token, psm2_amarg_t *args,
case PSMX2_AM_REP_READ:
req = (struct psmx2_am_request *)(uintptr_t)args[1].u64;
- assert(req->op == PSMX2_AM_REQ_READ);
+ assert(req->op == PSMX2_AM_REQ_READ || req->op == PSMX2_AM_REQ_READV);
op_error = (int)args[0].u32w1;
offset = args[2].u64;
if (!req->error)
req->error = op_error;
if (!op_error) {
- memcpy(req->read.buf + offset, src, len);
+ if (req->op == PSMX2_AM_REQ_READ)
+ memcpy(req->read.buf + offset, src, len);
+ else
+ psmx2_iov_copy(req->iov, req->read.iov_count, offset, src, len);
+
req->read.len_read += len;
}
- if (eom) {
+ if (eom || req->read.len == req->read.len_read) {
+ if (!eom)
+ FI_INFO(&psmx2_prov, FI_LOG_EP_DATA,
+ "readv: short protocol finishes after long protocol.\n");
if (req->ep->send_cq && !req->no_event) {
event = psmx2_cq_create_event(
req->ep->send_cq,
@@ -352,7 +387,7 @@ static ssize_t psmx2_rma_self(int am_cmd,
int err = 0;
int op_error = 0;
int access;
- void *dst;
+ uint8_t *dst, *src;
uint64_t cq_flags;
struct iovec *iov = buf;
size_t iov_count = len;
@@ -374,6 +409,12 @@ static ssize_t psmx2_rma_self(int am_cmd,
access = FI_REMOTE_READ;
cq_flags = FI_READ | FI_RMA;
break;
+ case PSMX2_AM_REQ_READV:
+ access = FI_REMOTE_READ;
+ cq_flags = FI_READ | FI_RMA;
+ len = 0;
+ for (i=0; i<iov_count; i++)
+ len += iov[i].iov_len;
default:
return -FI_EINVAL;
}
@@ -411,6 +452,16 @@ static ssize_t psmx2_rma_self(int am_cmd,
cntr = dst_ep->remote_read_cntr;
memcpy(buf, (void *)addr, len);
break;
+
+ case PSMX2_AM_REQ_READV:
+ cntr = dst_ep->remote_read_cntr;
+ src = (void *)addr;
+ for (i=0; i<iov_count; i++)
+ if (iov[i].iov_len) {
+ memcpy(iov[i].iov_base, src, iov[i].iov_len);
+ src += iov[i].iov_len;
+ }
+ break;
}
if (cq) {
@@ -460,11 +511,13 @@ static ssize_t psmx2_rma_self(int am_cmd,
switch (am_cmd) {
case PSMX2_AM_REQ_WRITE:
+ case PSMX2_AM_REQ_WRITEV:
if (ep->write_cntr)
psmx2_cntr_inc(ep->write_cntr);
break;
case PSMX2_AM_REQ_READ:
+ case PSMX2_AM_REQ_READV:
if (ep->read_cntr)
psmx2_cntr_inc(ep->read_cntr);
break;
@@ -662,6 +715,171 @@ ssize_t psmx2_read_generic(struct fid_ep *ep, void *buf, size_t len,
return 0;
}
+ssize_t psmx2_readv_generic(struct fid_ep *ep, const struct iovec *iov,
+ void *desc, size_t count, fi_addr_t src_addr,
+ uint64_t addr, uint64_t key, void *context,
+ uint64_t flags)
+{
+ struct psmx2_fid_ep *ep_priv;
+ struct psmx2_fid_av *av;
+ struct psmx2_epaddr_context *epaddr_context;
+ struct psmx2_am_request *req;
+ psm2_amarg_t args[8];
+ int chunk_size;
+ size_t offset = 0;
+ psm2_epaddr_t psm2_epaddr;
+ uint8_t vlane;
+ psm2_mq_req_t psm2_req;
+ psm2_mq_tag_t psm2_tag, psm2_tagsel;
+ uint32_t tag32;
+ size_t idx;
+ size_t total_len, long_len, short_len;
+ void *long_buf;
+ int i;
+
+ ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
+
+ if (flags & FI_TRIGGER) {
+ struct psmx2_trigger *trigger;
+ struct fi_triggered_context *ctxt = context;
+
+ trigger = calloc(1, sizeof(*trigger));
+ if (!trigger)
+ return -FI_ENOMEM;
+
+ trigger->op = PSMX2_TRIGGERED_READV;
+ trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
+ struct psmx2_fid_cntr, cntr);
+ trigger->threshold = ctxt->trigger.threshold.threshold;
+ trigger->readv.ep = ep;
+ trigger->readv.iov = iov;
+ trigger->readv.count = count;
+ trigger->readv.desc = desc;
+ trigger->readv.src_addr = src_addr;
+ trigger->readv.addr = addr;
+ trigger->readv.key = key;
+ trigger->readv.context = context;
+ trigger->readv.flags = flags & ~FI_TRIGGER;
+
+ psmx2_cntr_add_trigger(trigger->cntr, trigger);
+ return 0;
+ }
+
+ av = ep_priv->av;
+ if (av && av->type == FI_AV_TABLE) {
+ idx = src_addr;
+ if (idx >= av->last)
+ return -FI_EINVAL;
+
+ psm2_epaddr = av->epaddrs[idx];
+ vlane = av->vlanes[idx];
+ } else {
+ if (!src_addr)
+ return -FI_EINVAL;
+
+ psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr);
+ vlane = PSMX2_ADDR_TO_VL(src_addr);
+ }
+
+ epaddr_context = psm2_epaddr_getctxt((void *)psm2_epaddr);
+ if (epaddr_context->epid == ep_priv->domain->psm2_epid)
+ return psmx2_rma_self(PSMX2_AM_REQ_READV, ep_priv,
+ ep_priv->domain->eps[vlane],
+ (void *)iov, count, desc, addr,
+ key, context, flags, 0);
+
+ total_len = 0;
+ for (i=0; i<count; i++)
+ total_len += iov[i].iov_len;
+
+ req = calloc(1, sizeof(*req) + count * sizeof(struct iovec));
+ if (!req)
+ return -FI_ENOMEM;
+
+ memcpy(req->iov, iov, count * sizeof(struct iovec));
+
+ req->op = PSMX2_AM_REQ_READV;
+ req->read.iov_count = count;
+ req->read.len = total_len;
+ req->read.addr = addr; /* needed? */
+ req->read.key = key; /* needed? */
+ req->read.context = context;
+ req->ep = ep_priv;
+ req->cq_flags = FI_READ | FI_RMA;
+ PSMX2_CTXT_TYPE(&req->fi_context) = PSMX2_READ_CONTEXT;
+ PSMX2_CTXT_USER(&req->fi_context) = context;
+ PSMX2_CTXT_EP(&req->fi_context) = ep_priv;
+
+ if (ep_priv->send_selective_completion && !(flags & FI_COMPLETION)) {
+ PSMX2_CTXT_TYPE(&req->fi_context) = PSMX2_NOCOMP_READ_CONTEXT;
+ req->no_event = 1;
+ }
+
+ chunk_size = psmx2_am_param.max_reply_short;
+
+ long_len = 0;
+ if (psmx2_env.tagged_rma) {
+ for (i=count-1; i>=0; i--) {
+ if (iov[i].iov_len > chunk_size) {
+ long_buf = iov[i].iov_base;
+ long_len = iov[i].iov_len;
+ break;
+ } else if (iov[i].iov_len) {
+ break;
+ }
+ }
+ }
+
+ short_len = total_len - long_len;
+
+ /* Use short protocol for all but the last segment (long_len) */
+ args[0].u32w0 = 0;
+ PSMX2_AM_SET_SRC(args[0].u32w0, ep_priv->vlane);
+ PSMX2_AM_SET_DST(args[0].u32w0, vlane);
+ PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_READ);
+ args[1].u64 = (uint64_t)(uintptr_t)req;
+ args[3].u64 = key;
+ while (short_len > chunk_size) {
+ args[0].u32w1 = chunk_size;
+ args[2].u64 = addr;
+ args[4].u64 = offset;
+ psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
+ args, 5, NULL, 0, 0, NULL, NULL);
+ addr += chunk_size;
+ short_len -= chunk_size;
+ offset += chunk_size;
+ }
+
+ if (!long_len)
+ PSMX2_AM_SET_FLAG(args[0].u32w0, PSMX2_AM_EOM);
+ args[0].u32w1 = short_len;
+ args[2].u64 = addr;
+ args[4].u64 = offset;
+ psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
+ args, 5, NULL, 0, 0, NULL, NULL);
+
+ /* Use the long protocol for the last segment */
+ if (long_len) {
+ tag32 = PSMX2_TAG32(PSMX2_RMA_BIT, vlane, ep_priv->vlane);
+ PSMX2_SET_TAG(psm2_tag, (uint64_t)req, tag32);
+ PSMX2_SET_TAG(psm2_tagsel, -1ULL, -1);
+ psm2_mq_irecv2(ep_priv->domain->psm2_mq, psm2_epaddr,
+ &psm2_tag, &psm2_tagsel, 0,
+ long_buf, long_len,
+ (void *)&req->fi_context, &psm2_req);
+
+ PSMX2_AM_SET_OP(args[0].u32w0, PSMX2_AM_REQ_READ_LONG);
+ args[0].u32w1 = long_len;
+ args[1].u64 = (uint64_t)req;
+ args[2].u64 = addr + short_len;
+ args[3].u64 = key;
+ psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER,
+ args, 4, NULL, 0, 0, NULL, NULL);
+ }
+
+ return 0;
+}
+
static ssize_t psmx2_read(struct fid_ep *ep, void *buf, size_t len,
void *desc, fi_addr_t src_addr,
uint64_t addr, uint64_t key, void *context)
@@ -671,16 +889,25 @@ static ssize_t psmx2_read(struct fid_ep *ep, void *buf, size_t len,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_read_generic(ep, buf, len, desc, src_addr, addr,
- key, context, ep_priv->flags);
+ key, context, ep_priv->tx_flags);
}
static ssize_t psmx2_readmsg(struct fid_ep *ep,
const struct fi_msg_rma *msg,
uint64_t flags)
{
- if (!msg || msg->iov_count != 1 || !msg->msg_iov || !msg->rma_iov)
+ if (!msg || !msg->iov_count || !msg->msg_iov ||
+ !msg->rma_iov || msg->rma_iov_count != 1)
return -FI_EINVAL;
+ if (msg->iov_count > 1)
+ return psmx2_readv_generic(ep, msg->msg_iov,
+ msg->desc ? msg->desc[0] : NULL,
+ msg->iov_count, msg->addr,
+ msg->rma_iov[0].addr,
+ msg->rma_iov[0].key,
+ msg->context, flags);
+
return psmx2_read_generic(ep, msg->msg_iov[0].iov_base,
msg->msg_iov[0].iov_len,
msg->desc ? msg->desc[0] : NULL,
@@ -693,9 +920,18 @@ static ssize_t psmx2_readv(struct fid_ep *ep, const struct iovec *iov,
void **desc, size_t count, fi_addr_t src_addr,
uint64_t addr, uint64_t key, void *context)
{
- if (!iov || count != 1)
+ struct psmx2_fid_ep *ep_priv;
+
+ ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
+
+ if (!iov || !count)
return -FI_EINVAL;
+ if (count > 1)
+ return psmx2_readv_generic(ep, iov, desc ? desc[0] : NULL,
+ count, src_addr, addr, key,
+ context, ep_priv->tx_flags);
+
return psmx2_read(ep, iov->iov_base, iov->iov_len,
desc ? desc[0] : NULL, src_addr, addr, key, context);
}
@@ -788,9 +1024,9 @@ ssize_t psmx2_write_generic(struct fid_ep *ep, const void *buf, size_t len,
if (!req)
return -FI_ENOMEM;
- memset((void *)req, 0, sizeof(*req));
- memcpy((void *)req + sizeof(*req), (void *)buf, len);
- buf = (void *)req + sizeof(*req);
+ memset(req, 0, sizeof(*req));
+ memcpy((uint8_t *)req + sizeof(*req), (void *)buf, len);
+ buf = (uint8_t *)req + sizeof(*req);
} else {
req = calloc(1, sizeof(*req));
if (!req)
@@ -860,7 +1096,7 @@ ssize_t psmx2_write_generic(struct fid_ep *ep, const void *buf, size_t len,
psm2_am_request_short(psm2_epaddr, PSMX2_AM_RMA_HANDLER, args,
nargs, (void *)buf, chunk_size, am_flags,
NULL, NULL);
- buf += chunk_size;
+ buf = (const uint8_t *)buf + chunk_size;
addr += chunk_size;
len -= chunk_size;
}
@@ -904,7 +1140,7 @@ ssize_t psmx2_writev_generic(struct fid_ep *ep, const struct iovec *iov,
void *psm2_context;
int no_event;
size_t total_len, len, len_sent;
- void *buf, *p;
+ uint8_t *buf, *p;
int i;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
@@ -974,15 +1210,15 @@ ssize_t psmx2_writev_generic(struct fid_ep *ep, const struct iovec *iov,
if (!req)
return -FI_ENOMEM;
- memset((void *)req, 0, sizeof(*req));
- p = (void *)req + sizeof(*req);
+ memset(req, 0, sizeof(*req));
+ p = (uint8_t *)req + sizeof(*req);
for (i=0; i<count; i++) {
if (iov[i].iov_len) {
memcpy(p, iov[i].iov_base, iov[i].iov_len);
p += iov[i].iov_len;
}
}
- buf = (void *)req + sizeof(*req);
+ buf = (uint8_t *)req + sizeof(*req);
len = total_len;
req->no_event = no_event;
@@ -1137,7 +1373,7 @@ static ssize_t psmx2_write(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_write_generic(ep, buf, len, desc, dest_addr, addr,
- key, context, ep_priv->flags, 0);
+ key, context, ep_priv->tx_flags, 0);
}
static ssize_t psmx2_writemsg(struct fid_ep *ep,
@@ -1175,11 +1411,11 @@ static ssize_t psmx2_writev(struct fid_ep *ep, const struct iovec *iov,
if (count > 1)
return psmx2_writev_generic(ep, iov, desc, count, dest_addr,
- addr, key, context, ep_priv->flags, 0);
+ addr, key, context, ep_priv->tx_flags, 0);
return psmx2_write_generic(ep, iov->iov_base, iov->iov_len,
desc ? desc[0] : NULL, dest_addr, addr, key,
- context, ep_priv->flags, 0);
+ context, ep_priv->tx_flags, 0);
}
static ssize_t psmx2_inject(struct fid_ep *ep, const void *buf, size_t len,
@@ -1190,7 +1426,7 @@ static ssize_t psmx2_inject(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_write_generic(ep, buf, len, NULL, dest_addr, addr, key, NULL,
- ep_priv->flags | FI_INJECT | PSMX2_NO_COMPLETION,
+ ep_priv->tx_flags | FI_INJECT | PSMX2_NO_COMPLETION,
0);
}
@@ -1203,7 +1439,7 @@ static ssize_t psmx2_writedata(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_write_generic(ep, buf, len, desc, dest_addr, addr, key,
- context, ep_priv->flags | FI_REMOTE_CQ_DATA,
+ context, ep_priv->tx_flags | FI_REMOTE_CQ_DATA,
data);
}
@@ -1216,7 +1452,7 @@ static ssize_t psmx2_injectdata(struct fid_ep *ep, const void *buf, size_t len,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_write_generic(ep, buf, len, NULL, dest_addr, addr, key, NULL,
- ep_priv->flags | FI_INJECT | PSMX2_NO_COMPLETION,
+ ep_priv->tx_flags | FI_INJECT | PSMX2_NO_COMPLETION,
data);
}
diff --git a/prov/psm2/src/psmx2_tagged.c b/prov/psm2/src/psmx2_tagged.c
index 6ee4d21..785e054 100644
--- a/prov/psm2/src/psmx2_tagged.c
+++ b/prov/psm2/src/psmx2_tagged.c
@@ -473,7 +473,7 @@ static ssize_t psmx2_tagged_recv(struct fid_ep *ep, void *buf, size_t len,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_tagged_recv_generic(ep, buf, len, desc, src_addr, tag,
- ignore, context, ep_priv->flags);
+ ignore, context, ep_priv->rx_flags);
}
static ssize_t psmx2_tagged_recvmsg(struct fid_ep *ep,
@@ -487,7 +487,7 @@ static ssize_t psmx2_tagged_recvmsg(struct fid_ep *ep,
return -FI_EINVAL;
if (msg->iov_count > 1) {
- return -FI_EINVAL;
+ return -FI_ENOSYS;
} else if (msg->iov_count) {
buf = msg->msg_iov[0].iov_base;
len = msg->msg_iov[0].iov_len;
@@ -511,8 +511,10 @@ psmx2_tagged_recvv##suffix(struct fid_ep *ep, const struct iovec *iov, \
{ \
void *buf; \
size_t len; \
- if ((count && !iov) || (count > 1)) \
+ if (count && !iov) \
return -FI_EINVAL; \
+ if (count > 1) \
+ return -FI_ENOSYS; \
if (count) { \
buf = iov[0].iov_base; \
len = iov[0].iov_len; \
@@ -1101,7 +1103,7 @@ static ssize_t psmx2_tagged_send(struct fid_ep *ep,
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
return psmx2_tagged_send_generic(ep, buf, len, desc, dest_addr,
- tag, context, ep_priv->flags, 0);
+ tag, context, ep_priv->tx_flags, 0);
}
static ssize_t psmx2_tagged_sendmsg(struct fid_ep *ep,
@@ -1153,7 +1155,7 @@ psmx2_tagged_sendv##suffix(struct fid_ep *ep, const struct iovec *iov, \
return psmx2_tagged_sendv_generic(ep, iov, desc, count, \
dest_addr, tag, \
context, \
- ep_priv->flags, 0); \
+ ep_priv->tx_flags, 0);\
} else if (count) { \
buf = iov[0].iov_base; \
len = iov[0].iov_len; \
@@ -1182,7 +1184,7 @@ static ssize_t psmx2_tagged_inject(struct fid_ep *ep,
return psmx2_tagged_send_generic(ep, buf, len, NULL, dest_addr,
tag, NULL,
- ep_priv->flags | FI_INJECT | PSMX2_NO_COMPLETION,
+ ep_priv->tx_flags | FI_INJECT | PSMX2_NO_COMPLETION,
0);
}
diff --git a/prov/psm2/src/psmx2_wait.c b/prov/psm2/src/psmx2_wait.c
index 68c49f9..dcdae16 100644
--- a/prov/psm2/src/psmx2_wait.c
+++ b/prov/psm2/src/psmx2_wait.c
@@ -137,7 +137,7 @@ int psmx2_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
struct fid_wait *wait;
int err;
- err = fi_wait_fd_open(fabric, attr, &wait);
+ err = ofi_wait_fd_open(fabric, attr, &wait);
if (err)
return err;
@@ -150,3 +150,36 @@ int psmx2_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
return 0;
}
+int psmx2_wait_trywait(struct fid_fabric *fabric, struct fid **fids, int count)
+{
+ struct psmx2_fid_cq *cq_priv;
+ struct util_eq *eq;
+ struct util_wait *wait;
+ int i, ret;
+
+ for (i = 0; i < count; i++) {
+ switch (fids[i]->fclass) {
+ case FI_CLASS_CQ:
+ cq_priv = container_of(fids[i], struct psmx2_fid_cq, cq);
+ wait = cq_priv->wait;
+ break;
+ case FI_CLASS_EQ:
+ eq = container_of(fids[i], struct util_eq, eq_fid.fid);
+ wait = eq->wait;
+ break;
+ case FI_CLASS_CNTR:
+ return -FI_ENOSYS;
+ case FI_CLASS_WAIT:
+ wait = container_of(fids[i], struct util_wait, wait_fid.fid);
+ break;
+ default:
+ return -FI_EINVAL;
+ }
+
+ ret = wait->try(wait);
+ if (ret)
+ return ret;
+ }
+ return 0;
+}
+
diff --git a/prov/rxd/Makefile.include b/prov/rxd/Makefile.include
new file mode 100644
index 0000000..5ca1acd
--- /dev/null
+++ b/prov/rxd/Makefile.include
@@ -0,0 +1,28 @@
+if HAVE_RXD
+_rxd_files = \
+ prov/rxd/src/rxd_attr.c \
+ prov/rxd/src/rxd_init.c \
+ prov/rxd/src/rxd_fabric.c \
+ prov/rxd/src/rxd_domain.c \
+ prov/rxd/src/rxd_av.c \
+ prov/rxd/src/rxd_cq.c \
+ prov/rxd/src/rxd_ep.c \
+ prov/rxd/src/rxd_rma.c \
+ prov/rxd/src/rxd.h
+
+if HAVE_RXD_DL
+pkglib_LTLIBRARIES += librxd-fi.la
+librxd_fi_la_SOURCES = $(_rxd_files) $(common_srcs)
+librxd_fi_la_LIBADD = $(linkback) $(rxd_shm_LIBS)
+librxd_fi_la_LDFLAGS = -module -avoid-version -shared -export-dynamic
+librxd_fi_la_DEPENDENCIES = $(linkback)
+else !HAVE_RXD_DL
+src_libfabric_la_SOURCES += $(_rxd_files)
+src_libfabric_la_LIBADD += $(rxd_shm_LIBS)
+endif !HAVE_RXD_DL
+
+#prov_install_man_pages += man/man7/fi_rxd.7
+
+endif HAVE_RXD
+
+#prov_dist_man_pages += man/man7/fi_rxd.7
diff --git a/prov/rxd/configure.m4 b/prov/rxd/configure.m4
new file mode 100644
index 0000000..9907901
--- /dev/null
+++ b/prov/rxd/configure.m4
@@ -0,0 +1,15 @@
+dnl Configury specific to the libfabric rxd provider
+
+dnl Called to configure this provider
+dnl
+dnl Arguments:
+dnl
+dnl $1: action if configured successfully
+dnl $2: action if not configured successfully
+dnl
+AC_DEFUN([FI_RXD_CONFIGURE],[
+ # Determine if we can support the rxd provider
+ rxd_h_happy=0
+ AS_IF([test x"$enable_rxd" != x"no"], [rxd_h_happy=1])
+ AS_IF([test $rxd_h_happy -eq 1], [$1], [$2])
+])
diff --git a/prov/rxd/src/rxd.h b/prov/rxd/src/rxd.h
new file mode 100644
index 0000000..78a74c6
--- /dev/null
+++ b/prov/rxd/src/rxd.h
@@ -0,0 +1,471 @@
+/*
+ * Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <pthread.h>
+#include <rdma/fabric.h>
+#include <rdma/fi_atomic.h>
+#include <rdma/fi_cm.h>
+#include <rdma/fi_domain.h>
+#include <rdma/fi_endpoint.h>
+#include <rdma/fi_eq.h>
+#include <rdma/fi_errno.h>
+#include <rdma/fi_rma.h>
+#include <rdma/fi_tagged.h>
+#include <rdma/fi_trigger.h>
+
+#include <fi.h>
+#include <fi_proto.h>
+#include <fi_enosys.h>
+#include <fi_rbuf.h>
+#include <fi_list.h>
+#include <fi_util.h>
+
+#ifndef _RXD_H_
+#define _RXD_H_
+
+#define RXD_MAJOR_VERSION (1)
+#define RXD_MINOR_VERSION (0)
+#define RXD_PROTOCOL_VERSION (1)
+#define RXD_FI_VERSION FI_VERSION(1,3)
+
+#define RXD_IOV_LIMIT (4)
+#define RXD_DEF_CQ_CNT (8)
+#define RXD_DEF_EP_CNT (8)
+#define RXD_AV_DEF_COUNT (128)
+
+#define RXD_MAX_TX_BITS (10)
+#define RXD_MAX_RX_BITS (10)
+#define RXD_TX_ID(seq, id) (((seq) << RXD_MAX_TX_BITS) | id)
+#define RXD_RX_ID(seq, id) (((seq) << RXD_MAX_RX_BITS) | id)
+#define RXD_TX_IDX_BITS ((1ULL << RXD_MAX_TX_BITS) - 1)
+#define RXD_RX_IDX_BITS ((1ULL << RXD_MAX_RX_BITS) - 1)
+
+#define RXD_BUF_POOL_ALIGNMENT (16)
+#define RXD_TX_POOL_CHUNK_CNT (1024)
+#define RXD_RX_POOL_CHUNK_CNT (1024)
+
+#define RXD_MAX_RX_WIN (16)
+#define RXD_MAX_OUT_TX_MSG (8)
+#define RXD_MAX_UNACKED (128)
+
+#define RXD_EP_MAX_UNEXP_PKT (512)
+#define RXD_EP_MAX_UNEXP_MSG (128)
+
+#define RXD_USE_OP_FLAGS (1ULL << 61)
+#define RXD_NO_COMPLETION (1ULL << 62)
+#define RXD_UNEXP_ENTRY (1ULL << 63)
+
+
+#define RXD_RETRY_TIMEOUT (900)
+#define RXD_WAIT_TIMEOUT (2000)
+#define RXD_MAX_PKT_RETRY (50)
+
+#define RXD_PKT_LOCAL_ACK (1)
+#define RXD_PKT_REMOTE_ACK (1 << 1)
+#define RXD_PKT_DONE (RXD_PKT_LOCAL_ACK | RXD_PKT_REMOTE_ACK)
+
+#define RXD_PKT_MARK_LOCAL_ACK(_pkt) ((_pkt)->ref |= RXD_PKT_LOCAL_ACK)
+#define RXD_PKT_MARK_REMOTE_ACK(_pkt) ((_pkt)->ref |= RXD_PKT_REMOTE_ACK)
+#define RXD_PKT_IS_COMPLETE(_pkt) ((_pkt)->ref == RXD_PKT_DONE)
+
+#define RXD_COPY_IOV_TO_BUF (1)
+#define RXD_COPY_BUF_TO_IOV (2)
+
+extern struct fi_provider rxd_prov;
+extern struct fi_info rxd_info;
+extern struct fi_fabric_attr rxd_fabric_attr;
+extern struct util_prov rxd_util_prov;
+extern struct fi_ops_rma rxd_ops_rma;
+
+enum {
+ RXD_PKT_ORDR_OK = 0,
+ RXD_PKT_ORDR_UNEXP,
+ RXD_PKT_ORDR_DUP,
+};
+
+enum {
+ RXD_TX_CONN = 0,
+ RXD_TX_MSG,
+ RXD_TX_TAG,
+ RXD_TX_WRITE,
+ RXD_TX_READ_REQ,
+ RXD_TX_READ_RSP,
+};
+
+enum {
+ RXD_PKT_STRT = 0,
+ RXD_PKT_DATA,
+ RXD_PKT_LAST,
+};
+
+struct rxd_fabric {
+ struct util_fabric util_fabric;
+ struct fid_fabric *dg_fabric;
+};
+
+struct rxd_domain {
+ struct util_domain util_domain;
+ struct fid_domain *dg_domain;
+
+ size_t addrlen;
+ ssize_t max_mtu_sz;
+ uint64_t dg_mode;
+ int do_progress;
+ pthread_t progress_thread;
+ fastlock_t lock;
+
+ struct dlist_entry ep_list;
+ struct dlist_entry cq_list;
+ struct ofi_util_mr *mr_heap;
+ fastlock_t mr_lock;
+};
+
+struct rxd_av {
+ struct fid_av *dg_av;
+ fastlock_t lock;
+
+ struct util_av util_av;
+ int dg_av_used;
+ size_t addrlen;
+ size_t size;
+};
+
+struct rxd_cq;
+typedef int (*rxd_cq_write_fn)(struct rxd_cq *cq,
+ struct fi_cq_tagged_entry *cq_entry);
+struct rxd_cq {
+ struct util_cq util_cq;
+ struct fid_cq *dg_cq;
+ struct rxd_domain *domain;
+ rxd_cq_write_fn write_fn;
+ struct dlist_entry dom_entry;
+ struct util_buf_pool *unexp_pool;
+ struct dlist_entry unexp_list;
+ fastlock_t lock;
+};
+
+struct rxd_peer {
+ uint64_t nxt_msg_id;
+ uint64_t exp_msg_id;
+ uint64_t conn_data;
+
+ uint8_t addr_published;
+ uint8_t conn_initiated;
+ uint16_t num_msg_out;
+ uint8_t pad[4];
+};
+
+struct rxd_ep {
+ struct fid_ep ep;
+ struct fid_ep *dg_ep;
+
+ struct rxd_domain *domain;
+ struct rxd_cq *rx_cq;
+ struct rxd_cq *tx_cq;
+ struct rxd_av *av;
+
+ struct rxd_peer *peer_info;
+ size_t max_peers;
+
+ void *name;
+ size_t addrlen;
+ int conn_data_set;
+ uint64_t conn_data;
+
+ size_t rx_size;
+ size_t credits;
+ uint64_t num_out;
+
+ int do_local_mr;
+ uint64_t caps;
+
+ struct dlist_entry dom_entry;
+ struct dlist_entry wait_rx_list;
+
+ struct dlist_entry unexp_tag_list;
+ struct dlist_entry unexp_msg_list;
+ uint16_t num_unexp_pkt;
+ uint16_t num_unexp_msg;
+
+ struct util_buf_pool *tx_pkt_pool;
+ struct util_buf_pool *rx_pkt_pool;
+ struct slist rx_pkt_list;
+
+ struct rxd_tx_entry_fs *tx_entry_fs;
+ struct dlist_entry tx_entry_list;
+
+ struct rxd_rx_entry_fs *rx_entry_fs;
+ struct dlist_entry rx_entry_list;
+
+ struct rxd_recv_fs *recv_fs;
+ struct dlist_entry recv_list;
+
+ struct rxd_trecv_fs *trecv_fs;
+ struct dlist_entry trecv_list;
+ fastlock_t lock;
+};
+
+struct rxd_unexp_cq_entry {
+ struct dlist_entry entry;
+ struct fi_cq_msg_entry cq_entry;
+};
+
+struct rxd_rx_buf {
+ struct fi_context context;
+ struct slist_entry entry;
+ struct rxd_ep *ep;
+ struct fid_mr *mr;
+ char buf[];
+};
+
+struct rxd_rx_entry {
+ struct ofi_op_hdr op_hdr;
+ uint32_t exp_seg_no;
+ uint64_t msg_id;
+ uint64_t key;
+ uint64_t done;
+ uint64_t peer;
+ uint16_t window;
+ uint32_t last_win_seg;
+ fi_addr_t source;
+ struct rxd_peer *peer_info;
+ struct rxd_rx_buf *unexp_buf;
+ uint64_t nack_stamp;
+ struct dlist_entry entry;
+
+ union {
+ struct rxd_recv_entry *recv;
+ struct rxd_trecv_entry *trecv;
+
+ struct {
+ struct iovec iov[RXD_IOV_LIMIT];
+ } write;
+
+ struct {
+ struct rxd_tx_entry *tx_entry;
+ } read_rsp;
+ };
+
+ union {
+ struct dlist_entry wait_entry;
+ struct dlist_entry unexp_entry;
+ };
+};
+DECLARE_FREESTACK(struct rxd_rx_entry, rxd_rx_entry_fs);
+
+struct rxd_tx_entry {
+ fi_addr_t peer;
+ uint64_t msg_id;
+ uint64_t flags;
+ uint64_t done;
+ uint64_t rx_key;
+ uint32_t nxt_seg_no;
+ uint32_t win_sz;
+ int num_unacked;
+ int is_waiting;
+ uint64_t retry_stamp;
+
+ struct dlist_entry entry;
+ struct dlist_entry pkt_list;
+
+ uint8_t op_type;
+ struct ofi_op_hdr op_hdr;
+
+ union {
+ struct {
+ struct fi_msg msg;
+ struct iovec msg_iov[RXD_IOV_LIMIT];
+ } msg;
+
+ struct {
+ struct fi_msg_tagged tmsg;
+ struct iovec msg_iov[RXD_IOV_LIMIT];
+ } tmsg;
+
+ struct {
+ struct fi_msg_rma msg;
+ struct iovec src_iov[RXD_IOV_LIMIT];
+ struct fi_rma_iov dst_iov[RXD_IOV_LIMIT];
+ } write;
+
+ struct {
+ struct fi_msg_rma msg;
+ struct fi_rma_iov src_iov[RXD_IOV_LIMIT];
+ struct iovec dst_iov[RXD_IOV_LIMIT];
+ } read_req;
+
+ struct {
+ uint64_t peer_msg_id;
+ uint8_t iov_count;
+ struct iovec src_iov[RXD_IOV_LIMIT];
+ } read_rsp;
+ };
+};
+DECLARE_FREESTACK(struct rxd_tx_entry, rxd_tx_entry_fs);
+
+struct rxd_recv_entry {
+ struct dlist_entry entry;
+ struct fi_msg msg;
+ uint64_t flags;
+ struct iovec iov[RXD_IOV_LIMIT];
+ void *desc[RXD_IOV_LIMIT];
+};
+DECLARE_FREESTACK(struct rxd_recv_entry, rxd_recv_fs);
+
+struct rxd_trecv_entry {
+ struct dlist_entry entry;
+ struct fi_msg_tagged msg;
+ uint64_t flags;
+ struct rxd_rx_entry *rx_entry;
+ struct iovec iov[RXD_IOV_LIMIT];
+ void *desc[RXD_IOV_LIMIT];
+};
+DECLARE_FREESTACK(struct rxd_trecv_entry, rxd_trecv_fs);
+
+struct rxd_pkt_data_start {
+ struct ofi_ctrl_hdr ctrl;
+ struct ofi_op_hdr op;
+ char data[];
+};
+#define RXD_START_DATA_PKT_SZ (sizeof(struct rxd_pkt_data_start))
+#define RXD_MAX_STRT_DATA_PKT_SZ(ep) (ep->domain->max_mtu_sz - RXD_START_DATA_PKT_SZ)
+
+struct rxd_pkt_data {
+ struct ofi_ctrl_hdr ctrl;
+ char data[];
+};
+#define RXD_DATA_PKT_SZ (sizeof(struct rxd_pkt_data))
+#define RXD_MAX_DATA_PKT_SZ(ep) (ep->domain->max_mtu_sz - RXD_DATA_PKT_SZ)
+
+struct rxd_pkt_meta {
+ struct fi_context context;
+ struct dlist_entry entry;
+ struct rxd_tx_entry *tx_entry;
+ struct rxd_ep *ep;
+ struct fid_mr *mr;
+ uint64_t us_stamp;
+ uint8_t ref;
+ uint8_t type;
+ uint8_t retries;
+ uint8_t pad[5];
+
+ char pkt_data[]; /* rxd_pkt, followed by data */
+};
+
+int rxd_alter_layer_info(struct fi_info *layer_info, struct fi_info *base_info);
+int rxd_alter_base_info(struct fi_info *base_info, struct fi_info *layer_info);
+
+int rxd_fabric(struct fi_fabric_attr *attr,
+ struct fid_fabric **fabric, void *context);
+int rxd_domain_open(struct fid_fabric *fabric, struct fi_info *info,
+ struct fid_domain **dom, void *context);
+int rxd_av_create(struct fid_domain *domain_fid, struct fi_av_attr *attr,
+ struct fid_av **av, void *context);
+int rxd_endpoint(struct fid_domain *domain, struct fi_info *info,
+ struct fid_ep **ep, void *context);
+int rxd_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
+ struct fid_cq **cq_fid, void *context);
+
+
+/* AV sub-functions */
+fi_addr_t rxd_av_get_dg_addr(struct rxd_av *av, fi_addr_t fi_addr);
+int rxd_av_insert_dg_av(struct rxd_av *av, const void *addr);
+fi_addr_t rxd_av_get_fi_addr(struct rxd_av *av, fi_addr_t dg_addr);
+int rxd_av_dg_reverse_lookup(struct rxd_av *av, uint64_t start_idx,
+ const void *addr, size_t addrlen, uint64_t *idx);
+
+/* EP sub-functions */
+void rxd_ep_lock_if_required(struct rxd_ep *rxd_ep);
+void rxd_ep_unlock_if_required(struct rxd_ep *rxd_ep);
+int rxd_ep_repost_buff(struct rxd_rx_buf *rx_buf);
+void rxd_ep_progress(struct rxd_ep *ep);
+int rxd_ep_reply_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *in_ctrl,
+ uint8_t type, uint16_t seg_size, uint64_t rx_key,
+ uint64_t source, fi_addr_t dest);
+int rxd_ep_reply_nack(struct rxd_ep *ep, struct ofi_ctrl_hdr *in_ctrl,
+ uint32_t seg_no, uint64_t rx_key,
+ uint64_t source, fi_addr_t dest);
+int rxd_ep_reply_discard(struct rxd_ep *ep, struct ofi_ctrl_hdr *in_ctrl,
+ uint32_t seg_no, uint64_t rx_key,
+ uint64_t source, fi_addr_t dest);
+struct rxd_peer *rxd_ep_getpeer_info(struct rxd_ep *rxd_ep, fi_addr_t addr);
+
+void rxd_ep_check_unexp_msg_list(struct rxd_ep *ep, struct rxd_recv_entry *recv_entry);
+void rxd_ep_check_unexp_tag_list(struct rxd_ep *ep, struct rxd_trecv_entry *trecv_entry);
+void rxd_ep_handle_data_msg(struct rxd_ep *ep, struct rxd_peer *peer,
+ struct rxd_rx_entry *rx_entry,
+ struct iovec *iov, size_t iov_count,
+ struct ofi_ctrl_hdr *ctrl, void *data,
+ struct rxd_rx_buf *rx_buf);
+void rxd_ep_free_acked_pkts(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry,
+ uint32_t seg_no);
+uint64_t rxd_ep_copy_iov_buf(const struct iovec *iov, size_t iov_count,
+ void *buf, uint64_t data_sz, uint64_t skip, int dir);
+int rxd_ep_retry_pkt(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry,
+ struct rxd_pkt_meta *pkt);
+void rxd_ep_copy_msg_iov(const struct iovec *src_iov,
+ struct iovec *dst_iov, size_t iov_count);
+void rxd_ep_copy_rma_iov(const struct fi_rma_iov *src_iov,
+ struct fi_rma_iov *dst_iov, size_t iov_count);
+ssize_t rxd_ep_post_start_msg(struct rxd_ep *ep, struct rxd_peer *peer,
+ uint8_t op, struct rxd_tx_entry *tx_entry);
+ssize_t rxd_ep_post_conn_msg(struct rxd_ep *ep, struct rxd_peer *peer,
+ fi_addr_t addr);
+int rxd_mr_verify(struct rxd_domain *rxd_domain, ssize_t len,
+ uintptr_t *io_addr, uint64_t key, uint64_t access);
+
+
+/* Tx/Rx entry sub-functions */
+struct rxd_tx_entry *rxd_tx_entry_acquire(struct rxd_ep *ep, struct rxd_peer *peer);
+struct rxd_tx_entry *rxd_tx_entry_acquire_fast(struct rxd_ep *ep, struct rxd_peer *peer);
+int rxd_tx_entry_progress(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry,
+ struct ofi_ctrl_hdr *ack);
+void rxd_tx_entry_discard(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry);
+void rxd_tx_entry_release(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry);
+void rxd_tx_entry_done(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry);
+
+struct rxd_pkt_meta *rxd_tx_pkt_acquire(struct rxd_ep *ep);
+void rxd_tx_pkt_release(struct rxd_pkt_meta *pkt_meta);
+void rxd_rx_entry_release(struct rxd_ep *ep, struct rxd_rx_entry *rx_entry);
+
+
+/* CQ sub-functions */
+void rxd_cq_progress(struct util_cq *util_cq);
+void rxd_cq_report_error(struct rxd_cq *cq, struct fi_cq_err_entry *err_entry);
+void rxd_cq_report_tx_comp(struct rxd_cq *cq, struct rxd_tx_entry *tx_entry);
+void rxd_cq_report_rx_comp(struct rxd_cq *cq, struct rxd_rx_entry *rx_entry);
+
+#endif
diff --git a/prov/udp/src/udpx_attr.c b/prov/rxd/src/rxd_attr.c
similarity index 57%
copy from prov/udp/src/udpx_attr.c
copy to prov/rxd/src/rxd_attr.c
index b320423..d338490 100644
--- a/prov/udp/src/udpx_attr.c
+++ b/prov/rxd/src/rxd_attr.c
@@ -30,61 +30,70 @@
* SOFTWARE.
*/
-#include "udpx.h"
+#include "rxd.h"
-
-struct fi_tx_attr udpx_tx_attr = {
- .caps = FI_MSG | FI_SEND,
+struct fi_tx_attr rxd_tx_attr = {
+ .caps = FI_MSG | FI_TAGGED | FI_SEND | FI_RMA | FI_WRITE |
+ FI_READ | FI_RMA_EVENT | FI_REMOTE_READ | FI_REMOTE_WRITE,
.comp_order = FI_ORDER_STRICT,
- .inject_size = 1472,
- .size = 1024,
- .iov_limit = UDPX_IOV_LIMIT
+ .inject_size = 0,
+ .size = (1ULL << RXD_MAX_TX_BITS),
+ .iov_limit = RXD_IOV_LIMIT,
};
-struct fi_rx_attr udpx_rx_attr = {
- .caps = FI_MSG | FI_RECV | FI_SOURCE,
+struct fi_rx_attr rxd_rx_attr = {
+ .caps = FI_MSG | FI_TAGGED | FI_RECV | FI_SOURCE | FI_RMA_EVENT,
.comp_order = FI_ORDER_STRICT,
- .total_buffered_recv = (1 << 16),
- .size = 1024,
- .iov_limit = UDPX_IOV_LIMIT
+ .total_buffered_recv = 0,
+ .size = (1ULL << RXD_MAX_RX_BITS),
+ .iov_limit = RXD_IOV_LIMIT
};
-struct fi_ep_attr udpx_ep_attr = {
- .type = FI_EP_DGRAM,
- .protocol = FI_PROTO_UDP,
- .protocol_version = 0,
- .max_msg_size = 1472,
+struct fi_ep_attr rxd_ep_attr = {
+ .type = FI_EP_RDM,
+ .protocol = FI_PROTO_RXD,
+ .protocol_version = 1,
+ .max_msg_size = SIZE_MAX,
.tx_ctx_cnt = 1,
.rx_ctx_cnt = 1
};
-struct fi_domain_attr udpx_domain_attr = {
- .name = "udp",
+struct fi_domain_attr rxd_domain_attr = {
+ .name = "rxd",
.threading = FI_THREAD_SAFE,
.control_progress = FI_PROGRESS_AUTO,
.data_progress = FI_PROGRESS_AUTO,
.resource_mgmt = FI_RM_ENABLED,
.av_type = FI_AV_UNSPEC,
.mr_mode = FI_MR_SCALABLE,
- .cq_cnt = (1 << 16),
- .ep_cnt = (1 << 15),
- .tx_ctx_cnt = (1 << 15),
- .rx_ctx_cnt = (1 << 15),
+ .cq_cnt = RXD_DEF_CQ_CNT,
+ .ep_cnt = RXD_DEF_EP_CNT,
+ .tx_ctx_cnt = 1,
+ .rx_ctx_cnt = 1,
.max_ep_tx_ctx = 1,
.max_ep_rx_ctx = 1
};
-struct fi_fabric_attr udpx_fabric_attr = {
- .name = "UDP-IP",
- .prov_version = FI_VERSION(UDPX_MAJOR_VERSION, UDPX_MINOR_VERSION)
+struct fi_fabric_attr rxd_fabric_attr = {
+ .name = "",
+ .prov_version = FI_VERSION(RXD_MAJOR_VERSION, RXD_MINOR_VERSION),
+ .prov_name = "rxd",
+};
+
+struct fi_info rxd_info = {
+ .caps = FI_MSG | FI_SEND | FI_RECV | FI_SOURCE | FI_TAGGED |
+ FI_RMA | FI_WRITE | FI_READ | FI_RMA_EVENT |
+ FI_REMOTE_WRITE | FI_REMOTE_READ,
+ .addr_format = FI_SOCKADDR,
+ .tx_attr = &rxd_tx_attr,
+ .rx_attr = &rxd_rx_attr,
+ .ep_attr = &rxd_ep_attr,
+ .domain_attr = &rxd_domain_attr,
+ .fabric_attr = &rxd_fabric_attr
};
-struct fi_info udpx_info = {
- .caps = FI_MSG | FI_SEND | FI_RECV | FI_SOURCE, /* | FI_MULTI_RECV, */
- .addr_format = FI_SOCKADDR_IN,
- .tx_attr = &udpx_tx_attr,
- .rx_attr = &udpx_rx_attr,
- .ep_attr = &udpx_ep_attr,
- .domain_attr = &udpx_domain_attr,
- .fabric_attr = &udpx_fabric_attr
+struct util_prov rxd_util_prov = {
+ .prov = &rxd_prov,
+ .info = &rxd_info,
+ .flags = 0,
};
diff --git a/prov/rxd/src/rxd_av.c b/prov/rxd/src/rxd_av.c
new file mode 100644
index 0000000..5d73434
--- /dev/null
+++ b/prov/rxd/src/rxd_av.c
@@ -0,0 +1,335 @@
+/*
+ * Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "rxd.h"
+
+fi_addr_t rxd_av_get_dg_addr(struct rxd_av *av, fi_addr_t fi_addr)
+{
+ uint64_t *dg_idx;
+ dg_idx = ofi_av_get_addr(&av->util_av, (int) fi_addr);
+ return *dg_idx;
+}
+
+fi_addr_t rxd_av_get_fi_addr(struct rxd_av *av, fi_addr_t dg_addr)
+{
+ return (fi_addr_t) ofi_av_lookup_index(&av->util_av,
+ &dg_addr, (int) dg_addr);
+}
+
+int rxd_av_insert_dg_av(struct rxd_av *av, const void *addr)
+{
+ int ret;
+ fastlock_acquire(&av->lock);
+ ret = fi_av_insert(av->dg_av, addr, 1, NULL, 0, NULL);
+ if (ret != 1)
+ goto out;
+ av->dg_av_used++;
+out:
+ fastlock_release(&av->lock);
+ return ret;
+}
+
+int rxd_av_dg_reverse_lookup(struct rxd_av *av, uint64_t start_idx,
+ const void *addr, size_t addrlen, uint64_t *idx)
+{
+ int ret;
+ size_t i, len;
+ void *curr_addr;
+
+ len = addrlen;
+ curr_addr = calloc(1, av->addrlen);
+ if (!curr_addr)
+ return -FI_ENOMEM;
+
+ for (i = 0; i < av->dg_av_used; i++) {
+ ret = fi_av_lookup(av->dg_av, (i + start_idx) % av->dg_av_used,
+ curr_addr, &len);
+ if (ret)
+ continue;
+ if (len == addrlen && memcmp(curr_addr, addr, len) == 0) {
+ *idx = (i + start_idx) % av->dg_av_used;
+ goto out;
+ }
+ }
+ ret = -FI_ENODATA;
+out:
+ free(curr_addr);
+ return ret;
+}
+
+size_t rxd_av_insert_check(struct rxd_av *av, const void *addr, size_t count,
+ fi_addr_t *fi_addr, uint64_t flags, void *context)
+{
+ size_t i, success_cnt = 0;
+ int ret, index;
+ void *curr_addr;
+ uint64_t dg_av_idx;
+
+ for (i = 0; i < count; i++) {
+ curr_addr = (char *) addr + av->addrlen * i;
+ ret = rxd_av_dg_reverse_lookup(av, i, curr_addr, av->addrlen, &dg_av_idx);
+ if (ret == -FI_ENODATA) {
+ ret = fi_av_insert(av->dg_av, curr_addr, 1, &dg_av_idx,
+ flags, context);
+ if (ret <= 0) {
+ if (av->util_av.eq)
+ ofi_av_write_event(&av->util_av, i,
+ (ret == 0) ? FI_EINVAL : -ret,
+ context);
+ if (fi_addr)
+ fi_addr[i] = FI_ADDR_NOTAVAIL;
+ continue;
+ }
+ }
+
+ ret = ofi_av_insert_addr(&av->util_av, &dg_av_idx, dg_av_idx, &index);
+ if (ret) {
+ if (av->util_av.eq)
+ ofi_av_write_event(&av->util_av, i, -ret, context);
+ } else {
+ success_cnt++;
+ }
+
+ if (fi_addr)
+ fi_addr[i] = (ret == 0) ? index : FI_ADDR_NOTAVAIL;
+ }
+ av->dg_av_used += success_cnt;
+ if (av->util_av.eq) {
+ ofi_av_write_event(&av->util_av, success_cnt, 0, context);
+ ret = 0;
+ } else {
+ ret = success_cnt;
+ }
+ return ret;
+}
+
+size_t rxd_av_insert_fast(struct rxd_av *av, const void *addr, size_t count,
+ fi_addr_t *fi_addr, uint64_t flags, void *context)
+{
+ size_t i, num, ret, success_cnt = 0;
+ int index;
+ fi_addr_t *fi_addrs;
+
+ fi_addrs = calloc(count, sizeof(fi_addr_t));
+ if (!fi_addrs)
+ return -FI_ENOMEM;
+
+ num = fi_av_insert(av->dg_av, addr, count, fi_addrs, flags, context);
+ if (num != count) {
+ free(fi_addrs);
+ return rxd_av_insert_check(av, addr, count, fi_addr,
+ flags, context);
+ }
+
+ for (i = 0; i < num; i++) {
+ ret = ofi_av_insert_addr(&av->util_av, &fi_addrs[i],
+ fi_addrs[i], &index);
+ if (ret) {
+ if (av->util_av.eq)
+ ofi_av_write_event(&av->util_av, i, -ret, context);
+ } else {
+ success_cnt++;
+ }
+
+ if (fi_addr)
+ fi_addr[i] = (ret == 0) ? index : FI_ADDR_NOTAVAIL;
+ }
+
+ free(fi_addrs);
+ av->dg_av_used += success_cnt;
+
+ if (av->util_av.eq) {
+ ofi_av_write_event(&av->util_av, success_cnt, 0, context);
+ ret = 0;
+ } else {
+ ret = success_cnt;
+ }
+
+ return ret;
+}
+
+static int rxd_av_insert(struct fid_av *av_fid, const void *addr, size_t count,
+ fi_addr_t *fi_addr, uint64_t flags, void *context)
+{
+ struct rxd_av *av;
+ av = container_of(av_fid, struct rxd_av, util_av.av_fid);
+ return av->dg_av_used ?
+ rxd_av_insert_check(av, addr, count, fi_addr, flags, context) :
+ rxd_av_insert_fast(av, addr, count, fi_addr, flags, context);
+}
+
+static int rxd_av_insertsvc(struct fid_av *av, const char *node,
+ const char *service, fi_addr_t *fi_addr,
+ uint64_t flags, void *context)
+{
+ return -FI_ENOSYS;
+}
+
+static int rxd_av_insertsym(struct fid_av *av_fid, const char *node, size_t nodecnt,
+ const char *service, size_t svccnt, fi_addr_t *fi_addr,
+ uint64_t flags, void *context)
+{
+ return -FI_ENOSYS;
+}
+
+static int rxd_av_remove(struct fid_av *av_fid, fi_addr_t *fi_addr, size_t count,
+ uint64_t flags)
+{
+ int ret = 0;
+ size_t i;
+ fi_addr_t dg_idx;
+ struct rxd_av *av;
+
+ av = container_of(av_fid, struct rxd_av, util_av.av_fid);
+ for (i = 0; i < count; i++) {
+ dg_idx = rxd_av_get_dg_addr(av, fi_addr[i]);
+ ret = fi_av_remove(av->dg_av, &dg_idx, 1, flags);
+ if (ret)
+ break;
+ av->dg_av_used--;
+ }
+ return ret;
+}
+
+static const char * rxd_av_straddr(struct fid_av *av, const void *addr, char *buf, size_t *len)
+{
+ struct rxd_av *rxd_av;
+ rxd_av = container_of(av, struct rxd_av, util_av.av_fid);
+ return rxd_av->dg_av->ops->straddr(rxd_av->dg_av, addr, buf, len);
+}
+
+int rxd_av_lookup(struct fid_av *av, fi_addr_t fi_addr, void *addr, size_t *addrlen)
+{
+ struct rxd_av *rxd_av;
+ fi_addr_t dg_addr;
+ rxd_av = container_of(av, struct rxd_av, util_av.av_fid);
+ dg_addr = rxd_av_get_dg_addr(rxd_av, fi_addr);
+ return fi_av_lookup(rxd_av->dg_av, dg_addr, addr, addrlen);
+}
+
+static struct fi_ops_av rxd_av_ops = {
+ .size = sizeof(struct fi_ops_av),
+ .insert = rxd_av_insert,
+ .insertsvc = rxd_av_insertsvc,
+ .insertsym = rxd_av_insertsym,
+ .remove = rxd_av_remove,
+ .lookup = rxd_av_lookup,
+ .straddr = rxd_av_straddr,
+};
+
+int rxd_av_close(struct fid *fid)
+{
+ int ret;
+ struct rxd_av *av;
+ av = container_of(fid, struct rxd_av, util_av.av_fid);
+ ret = fi_close(&av->dg_av->fid);
+ if (ret)
+ return ret;
+
+ ret = ofi_av_close(&av->util_av);
+ if (ret)
+ return ret;
+
+ fastlock_destroy(&av->lock);
+ free(av);
+ return 0;
+}
+
+int rxd_av_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
+{
+ return ofi_av_bind(fid, bfid, flags);
+}
+
+static struct fi_ops rxd_av_fi_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = rxd_av_close,
+ .bind = rxd_av_bind,
+ .control = fi_no_control,
+ .ops_open = fi_no_ops_open,
+};
+
+int rxd_av_create(struct fid_domain *domain_fid, struct fi_av_attr *attr,
+ struct fid_av **av_fid, void *context)
+{
+ int ret;
+ struct rxd_av *av;
+ struct rxd_domain *domain;
+ struct util_av_attr util_attr;
+ struct fi_av_attr av_attr;
+
+ if (!attr)
+ return -FI_EINVAL;
+
+ if (attr->name)
+ return -FI_ENOSYS;
+
+ domain = container_of(domain_fid, struct rxd_domain, util_domain.domain_fid);
+ av = calloc(1, sizeof(*av));
+ if (!av)
+ return -FI_ENOMEM;
+
+ util_attr.addrlen = sizeof(fi_addr_t);
+ util_attr.overhead = attr->count;
+ util_attr.flags = FI_SOURCE;
+ av->size = attr->count ? attr->count : RXD_AV_DEF_COUNT;
+ if (attr->type == FI_AV_UNSPEC)
+ attr->type = FI_AV_TABLE;
+
+ ret = ofi_av_init(&domain->util_domain, attr, &util_attr,
+ &av->util_av, context);
+ if (ret)
+ goto err1;
+
+ av->size = av->util_av.count;
+ av_attr = *attr;
+ av_attr.type = FI_AV_TABLE;
+ av_attr.count = 0;
+ av_attr.flags = 0;
+ ret = fi_av_open(domain->dg_domain, &av_attr, &av->dg_av, context);
+ if (ret)
+ goto err2;
+
+ fastlock_init(&av->lock);
+ av->addrlen = domain->addrlen;
+
+ *av_fid = &av->util_av.av_fid;
+ (*av_fid)->fid.fclass = FI_CLASS_AV;
+ (*av_fid)->fid.ops = &rxd_av_fi_ops;
+ (*av_fid)->ops = &rxd_av_ops;
+ return 0;
+
+err2:
+ ofi_av_close(&av->util_av);
+err1:
+ free(av);
+ return ret;
+}
diff --git a/prov/rxd/src/rxd_cq.c b/prov/rxd/src/rxd_cq.c
new file mode 100644
index 0000000..f73f04e
--- /dev/null
+++ b/prov/rxd/src/rxd_cq.c
@@ -0,0 +1,1355 @@
+/*
+ * Copyright (c) 2013-2016 Intel Corporation. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <inttypes.h>
+#include "rxd.h"
+
+static int rxd_cq_write_ctx(struct rxd_cq *cq,
+ struct fi_cq_tagged_entry *cq_entry)
+{
+ struct fi_cq_tagged_entry *comp;
+ if (cirque_isfull(cq->util_cq.cirq))
+ return -FI_ENOSPC;
+
+ comp = cirque_tail(cq->util_cq.cirq);
+ comp->op_context = cq_entry->op_context;
+ cirque_commit(cq->util_cq.cirq);
+ return 0;
+}
+
+static int rxd_cq_write_ctx_signal(struct rxd_cq *cq,
+ struct fi_cq_tagged_entry *cq_entry)
+{
+ int ret = rxd_cq_write_ctx(cq, cq_entry);
+ cq->util_cq.wait->signal(cq->util_cq.wait);
+ return ret;
+}
+
+static int rxd_cq_write_msg(struct rxd_cq *cq,
+ struct fi_cq_tagged_entry *cq_entry)
+{
+ struct fi_cq_tagged_entry *comp;
+ if (cirque_isfull(cq->util_cq.cirq))
+ return -FI_ENOSPC;
+
+ comp = cirque_tail(cq->util_cq.cirq);
+ comp->op_context = cq_entry->op_context;
+ comp->flags = cq_entry->flags;
+ comp->len = cq_entry->len;
+ cirque_commit(cq->util_cq.cirq);
+ return 0;
+}
+
+static int rxd_cq_write_msg_signal(struct rxd_cq *cq,
+ struct fi_cq_tagged_entry *cq_entry)
+{
+ int ret = rxd_cq_write_msg(cq, cq_entry);
+ cq->util_cq.wait->signal(cq->util_cq.wait);
+ return ret;
+}
+
+static int rxd_cq_write_data(struct rxd_cq *cq,
+ struct fi_cq_tagged_entry *cq_entry)
+{
+ struct fi_cq_tagged_entry *comp;
+ if (cirque_isfull(cq->util_cq.cirq))
+ return -FI_ENOSPC;
+
+ comp = cirque_tail(cq->util_cq.cirq);
+ comp->op_context = cq_entry->op_context;
+ comp->flags = cq_entry->flags;
+ comp->len = cq_entry->len;
+ comp->buf = cq_entry->buf;
+ comp->data = cq_entry->data;
+ cirque_commit(cq->util_cq.cirq);
+ return 0;
+}
+
+static int rxd_cq_write_data_signal(struct rxd_cq *cq,
+ struct fi_cq_tagged_entry *cq_entry)
+{
+ int ret = rxd_cq_write_data(cq, cq_entry);
+ cq->util_cq.wait->signal(cq->util_cq.wait);
+ return ret;
+}
+
+static int rxd_cq_write_tagged(struct rxd_cq *cq,
+ struct fi_cq_tagged_entry *cq_entry)
+{
+ struct fi_cq_tagged_entry *comp;
+ if (cirque_isfull(cq->util_cq.cirq))
+ return -FI_ENOSPC;
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
+ "report completion: %p\n", cq_entry->tag);
+
+ comp = cirque_tail(cq->util_cq.cirq);
+ *comp = *cq_entry;
+ cirque_commit(cq->util_cq.cirq);
+ return 0;
+}
+
+static int rxd_cq_write_tagged_signal(struct rxd_cq *cq,
+ struct fi_cq_tagged_entry *cq_entry)
+{
+ int ret = rxd_cq_write_tagged(cq, cq_entry);
+ cq->util_cq.wait->signal(cq->util_cq.wait);
+ return ret;
+}
+
+static int rxd_check_start_pkt_order(struct rxd_ep *ep, struct rxd_peer *peer,
+ struct ofi_ctrl_hdr *ctrl,
+ struct fi_cq_msg_entry *comp)
+{
+ uint64_t msg_id;
+ msg_id = ctrl->msg_id >> RXD_MAX_TX_BITS;
+ if (peer->exp_msg_id == msg_id)
+ return RXD_PKT_ORDR_OK;
+
+ return (peer->exp_msg_id > msg_id) ? RXD_PKT_ORDR_DUP : RXD_PKT_ORDR_UNEXP;
+}
+
+static int rxd_rx_entry_match(struct dlist_entry *item, const void *arg)
+{
+ const struct ofi_ctrl_hdr *ctrl = arg;
+ struct rxd_rx_entry *rx_entry;
+
+ rx_entry = container_of(item, struct rxd_rx_entry, entry);
+ return (rx_entry->msg_id == ctrl->msg_id && rx_entry->peer == ctrl->conn_id);
+}
+
+static void rxd_handle_dup_datastart(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
+ struct rxd_rx_buf *rx_buf)
+{
+ struct dlist_entry *item;
+ struct rxd_rx_entry *rx_entry;
+ struct rxd_peer *peer;
+
+ item = dlist_find_first_match(&ep->rx_entry_list,
+ rxd_rx_entry_match, ctrl);
+ if (!item)
+ return;
+
+ FI_INFO(&rxd_prov, FI_LOG_EP_CTRL,
+ "duplicate start-data: msg_id: %" PRIu64 ", seg_no: %d\n",
+ ctrl->msg_id, ctrl->seg_no);
+
+ rx_entry = container_of(item, struct rxd_rx_entry, entry);
+ peer = rxd_ep_getpeer_info(ep, ctrl->conn_id);
+ rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, rx_entry->window, rx_entry->key,
+ peer->conn_data, ctrl->conn_id);
+ return;
+}
+
+int rxd_handle_conn_req(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
+ struct fi_cq_msg_entry *comp,
+ struct rxd_rx_buf *rx_buf)
+{
+ int ret;
+ void *addr;
+ size_t addrlen;
+ uint64_t peer;
+ struct rxd_pkt_data *pkt_data;
+ struct rxd_peer *peer_info;
+
+ rxd_ep_lock_if_required(ep);
+
+ pkt_data = (struct rxd_pkt_data *) ctrl;
+ addr = pkt_data->data;
+ addrlen = ctrl->seg_size;
+
+ ret = rxd_av_dg_reverse_lookup(ep->av, ctrl->rx_key, addr, addrlen, &peer);
+ if (ret == -FI_ENODATA) {
+ ret = rxd_av_insert_dg_av(ep->av, addr);
+ assert(ret == 1);
+
+ ret = rxd_av_dg_reverse_lookup(ep->av, ctrl->rx_key, addr, addrlen, &peer);
+ assert(ret == 0);
+ }
+
+ peer_info = rxd_ep_getpeer_info(ep, peer);
+ if (!peer_info->addr_published) {
+ peer_info->addr_published = 1;
+ peer_info->conn_initiated = 1;
+ peer_info->conn_data = ctrl->conn_id;
+ peer_info->exp_msg_id++;
+ }
+
+ rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_connresp, 0, ctrl->conn_id, peer, peer);
+ rxd_ep_repost_buff(rx_buf);
+ rxd_ep_unlock_if_required(ep);
+ return ret;
+}
+
+int rxd_tx_pkt_match(struct dlist_entry *item, const void *arg)
+{
+ const struct ofi_ctrl_hdr *pkt_ctrl, *ack_ctrl = arg;
+ struct rxd_pkt_meta *tx_pkt;
+
+ tx_pkt = container_of(item, struct rxd_pkt_meta, entry);
+ pkt_ctrl = (struct ofi_ctrl_hdr *) tx_pkt->pkt_data;
+ return (ack_ctrl->seg_no == pkt_ctrl->seg_no) ? 1 : 0;
+}
+
+int rxd_handle_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
+ struct rxd_rx_buf *rx_buf)
+{
+ int ret = 0;
+ uint64_t idx;
+ struct rxd_tx_entry *tx_entry;
+ struct dlist_entry *item;
+ struct rxd_pkt_meta *pkt;
+
+ rxd_ep_lock_if_required(ep);
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "got ack: msg: %p - %d\n",
+ ctrl->msg_id, ctrl->seg_no);
+
+ idx = ctrl->msg_id & RXD_TX_IDX_BITS;
+ tx_entry = &ep->tx_entry_fs->buf[idx];
+ if (tx_entry->msg_id != ctrl->msg_id)
+ goto out;
+
+ item = dlist_find_first_match(&tx_entry->pkt_list, rxd_tx_pkt_match, ctrl);
+ if (!item)
+ goto out;
+
+ pkt = container_of(item, struct rxd_pkt_meta, entry);
+ switch (pkt->type) {
+
+ case RXD_PKT_STRT:
+ case RXD_PKT_DATA:
+ ret = rxd_tx_entry_progress(ep, tx_entry, ctrl);
+ break;
+
+ case RXD_PKT_LAST:
+ rxd_ep_free_acked_pkts(ep, tx_entry, ctrl->seg_no);
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "reporting TX completion : %p\n", tx_entry);
+ if (tx_entry->op_type != RXD_TX_READ_REQ) {
+ rxd_cq_report_tx_comp(ep->tx_cq, tx_entry);
+ rxd_tx_entry_done(ep, tx_entry);
+ }
+ break;
+ default:
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid pkt type\n");
+ break;
+ }
+out:
+ rxd_ep_repost_buff(rx_buf);
+ rxd_ep_unlock_if_required(ep);
+ return ret;
+}
+
+int rxd_handle_nack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
+ struct rxd_rx_buf *rx_buf)
+{
+ int ret = 0;
+ uint64_t idx;
+ struct rxd_tx_entry *tx_entry;
+
+ rxd_ep_lock_if_required(ep);
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "got NACK: msg: %p - %d\n",
+ ctrl->msg_id, ctrl->seg_no);
+
+ idx = ctrl->msg_id & RXD_TX_IDX_BITS;
+ tx_entry = &ep->tx_entry_fs->buf[idx];
+ if (tx_entry->msg_id != ctrl->msg_id)
+ goto out;
+
+ ret = rxd_tx_entry_progress(ep, tx_entry, ctrl);
+out:
+ rxd_ep_repost_buff(rx_buf);
+ rxd_ep_unlock_if_required(ep);
+ return ret;
+}
+
+void rxd_handle_discard(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
+ struct rxd_rx_buf *rx_buf)
+{
+ uint64_t idx;
+ struct rxd_tx_entry *tx_entry;
+
+ rxd_ep_lock_if_required(ep);
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "got Reject: msg: %p - %d\n",
+ ctrl->msg_id, ctrl->seg_no);
+
+ idx = ctrl->msg_id & RXD_TX_IDX_BITS;
+ tx_entry = &ep->tx_entry_fs->buf[idx];
+ if (tx_entry->msg_id != ctrl->msg_id)
+ goto out;
+
+ rxd_tx_entry_discard(ep, tx_entry);
+out:
+ rxd_ep_repost_buff(rx_buf);
+ rxd_ep_unlock_if_required(ep);
+}
+
+void rxd_tx_pkt_release(struct rxd_pkt_meta *pkt_meta)
+{
+ if (RXD_PKT_IS_COMPLETE(pkt_meta)) {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "Releasing buf: %p, num_out: %d\n",
+ pkt_meta, pkt_meta->ep->num_out);
+ pkt_meta->ep->num_out--;
+ util_buf_release(pkt_meta->ep->tx_pkt_pool, pkt_meta);
+ }
+}
+
+void rxd_tx_entry_done(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry)
+{
+ struct rxd_pkt_meta *pkt_meta;
+ struct dlist_entry *item;
+
+ while (!dlist_empty(&tx_entry->pkt_list)) {
+ item = tx_entry->pkt_list.next;
+ pkt_meta = container_of(item, struct rxd_pkt_meta, entry);
+ dlist_remove(&pkt_meta->entry);
+ RXD_PKT_MARK_REMOTE_ACK(pkt_meta);
+ rxd_tx_pkt_release(pkt_meta);
+ }
+ rxd_tx_entry_release(ep, tx_entry);
+}
+
+static int rxd_conn_msg_match(struct dlist_entry *item, const void *arg)
+{
+ struct rxd_tx_entry *tx_entry;
+ struct ofi_ctrl_hdr *ctrl = (struct ofi_ctrl_hdr *) arg;
+ tx_entry = container_of(item, struct rxd_tx_entry, entry);
+ return (tx_entry->op_type == RXD_TX_CONN &&
+ tx_entry->peer == ctrl->rx_key);
+}
+
+void rxd_handle_connect_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
+ struct rxd_rx_buf *rx_buf)
+{
+ struct rxd_peer *peer;
+ struct dlist_entry *match;
+ struct rxd_tx_entry *tx_entry;
+
+ rxd_ep_lock_if_required(ep);
+ match = dlist_find_first_match(&ep->tx_entry_list, rxd_conn_msg_match, ctrl);
+ if (!match)
+ goto out;
+
+ tx_entry = container_of(match, struct rxd_tx_entry, entry);
+ peer = rxd_ep_getpeer_info(ep, tx_entry->peer);
+ peer->addr_published = 1;
+ peer->conn_data = ctrl->conn_id;
+
+ dlist_remove(match);
+ rxd_tx_entry_done(ep, tx_entry);
+out:
+ rxd_ep_repost_buff(rx_buf);
+ rxd_ep_unlock_if_required(ep);
+}
+
+static inline uint16_t rxd_get_window_sz(struct rxd_ep *ep, uint64_t rem)
+{
+ uint16_t num_pkts, avail;
+
+ num_pkts = (rem + RXD_MAX_DATA_PKT_SZ(ep) - 1) / RXD_MAX_DATA_PKT_SZ(ep);
+ avail = MIN(ep->credits, num_pkts);
+ return MIN(avail, RXD_MAX_RX_WIN);
+}
+
+struct rxd_rx_entry *rxd_get_rx_entry(struct rxd_ep *ep)
+{
+ struct rxd_rx_entry *rx_entry;
+ if (freestack_isempty(ep->rx_entry_fs))
+ return NULL;
+
+ rx_entry = freestack_pop(ep->rx_entry_fs);
+ rx_entry->key = rx_entry - &ep->rx_entry_fs->buf[0];
+ dlist_init(&rx_entry->entry);
+ dlist_init(&rx_entry->wait_entry);
+ dlist_insert_tail(&rx_entry->entry, &ep->rx_entry_list);
+ return rx_entry;
+}
+
+static void rxd_progress_wait_rx(struct rxd_ep *ep, struct rxd_rx_entry *rx_entry)
+{
+ struct ofi_ctrl_hdr ctrl;
+
+ rx_entry->window = rxd_get_window_sz(ep, rx_entry->op_hdr.size - rx_entry->done);
+
+ if (!rx_entry->window)
+ return;
+
+ rx_entry->last_win_seg += rx_entry->window;
+ dlist_remove(&rx_entry->wait_entry);
+
+ ep->credits -= rx_entry->window;
+
+ ctrl.msg_id = rx_entry->msg_id;
+ ctrl.seg_no = rx_entry->exp_seg_no - 1;
+ ctrl.conn_id = rx_entry->peer;
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "rx-entry wait over [%p], window: %d\n",
+ rx_entry->msg_id, rx_entry->window);
+ rxd_ep_reply_ack(ep, &ctrl, ofi_ctrl_ack, rx_entry->window,
+ rx_entry->key, rx_entry->peer_info->conn_data,
+ ctrl.conn_id);
+}
+
+static void rxd_check_waiting_rx(struct rxd_ep *ep)
+{
+ struct dlist_entry *entry;
+ struct rxd_rx_entry *rx_entry;
+
+ if (!ep->credits)
+ return;
+
+ while(!dlist_empty(&ep->wait_rx_list) && ep->credits) {
+ entry = ep->wait_rx_list.next;
+ rx_entry = container_of(entry, struct rxd_rx_entry, wait_entry);
+ rxd_progress_wait_rx(ep, rx_entry);
+ }
+}
+
+void rxd_rx_entry_release(struct rxd_ep *ep, struct rxd_rx_entry *rx_entry)
+{
+ rx_entry->key = -1;
+ dlist_remove(&rx_entry->entry);
+ freestack_push(ep->rx_entry_fs, rx_entry);
+
+ if (ep->credits && !dlist_empty(&ep->wait_rx_list))
+ rxd_check_waiting_rx(ep);
+}
+
+static int rxd_match_recv_entry(struct dlist_entry *item, const void *arg)
+{
+ struct rxd_rx_entry *rx_entry = (struct rxd_rx_entry *) arg;
+ struct rxd_recv_entry *recv_entry;
+ recv_entry = container_of(item, struct rxd_recv_entry, entry);
+ return (recv_entry->msg.addr == FI_ADDR_UNSPEC ||
+ rx_entry->source == FI_ADDR_UNSPEC ||
+ recv_entry->msg.addr == rx_entry->source);
+}
+
+struct rxd_recv_entry *rxd_get_recv_entry(struct rxd_ep *ep, struct rxd_rx_entry *rx_entry)
+{
+ struct dlist_entry *match;
+ struct rxd_recv_entry *recv_entry;
+
+ match = dlist_find_first_match(&ep->recv_list, &rxd_match_recv_entry,
+ (void *) rx_entry);
+ if (!match) {
+ /*todo: queue the pkt */
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "no matching recv entry\n");
+ return NULL;
+ }
+
+ dlist_remove(match);
+ recv_entry = container_of(match, struct rxd_recv_entry, entry);
+ return recv_entry;
+}
+
+static int rxd_match_trecv_entry(struct dlist_entry *item, const void *arg)
+{
+ struct rxd_rx_entry *rx_entry = (struct rxd_rx_entry *) arg;
+ struct rxd_trecv_entry *trecv_entry;
+
+ trecv_entry = container_of(item, struct rxd_trecv_entry, entry);
+ return ((trecv_entry->msg.tag | trecv_entry->msg.ignore) ==
+ (rx_entry->op_hdr.tag | trecv_entry->msg.ignore) &&
+ ((trecv_entry->msg.addr == FI_ADDR_UNSPEC) ||
+ (rx_entry->source == FI_ADDR_UNSPEC) ||
+ (trecv_entry->msg.addr == rx_entry->source)));
+ return 0;
+}
+
+struct rxd_trecv_entry *rxd_get_trecv_entry(struct rxd_ep *ep,
+ struct rxd_rx_entry *rx_entry)
+{
+ struct dlist_entry *match;
+ struct rxd_trecv_entry *trecv_entry;
+
+ match = dlist_find_first_match(&ep->trecv_list, &rxd_match_trecv_entry,
+ (void *)rx_entry);
+ if (!match) {
+ /*todo: queue the pkt */
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "no matching trecv entry, tag: %p\n",
+ rx_entry->op_hdr.tag);
+ return NULL;
+ }
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "matched - tag: %p\n", rx_entry->op_hdr.tag);
+
+ dlist_remove(match);
+ trecv_entry = container_of(match, struct rxd_trecv_entry, entry);
+ trecv_entry->rx_entry = rx_entry;
+ return trecv_entry;
+}
+
+void rxd_report_rx_comp(struct rxd_cq *cq, struct rxd_rx_entry *rx_entry)
+{
+ struct fi_cq_tagged_entry cq_entry = {0};
+
+ /* todo: handle FI_COMPLETION */
+ if (rx_entry->op_hdr.flags & OFI_REMOTE_CQ_DATA)
+ cq_entry.flags |= FI_REMOTE_CQ_DATA;
+
+ switch(rx_entry->op_hdr.op) {
+ case ofi_op_msg:
+ cq_entry.flags |= FI_RECV;
+ cq_entry.op_context = rx_entry->recv->msg.context;
+ cq_entry.len = rx_entry->done;
+ cq_entry.buf = rx_entry->recv->iov[0].iov_base;
+ cq_entry.data = rx_entry->op_hdr.data;
+ break;
+
+ case ofi_op_tagged:
+ cq_entry.flags |= (FI_RECV | FI_TAGGED);
+ cq_entry.op_context = rx_entry->trecv->msg.context;
+ cq_entry.len = rx_entry->done;
+ cq_entry.buf = rx_entry->trecv->iov[0].iov_base;
+ cq_entry.data = rx_entry->op_hdr.data;
+ cq_entry.tag = rx_entry->trecv->msg.tag;
+ break;
+
+ case ofi_op_atomic:
+ cq_entry.flags |= FI_ATOMIC;
+ break;
+
+ case ofi_op_write:
+ if (!(rx_entry->op_hdr.flags & OFI_REMOTE_CQ_DATA))
+ return;
+
+ cq_entry.flags |= (FI_RMA | FI_REMOTE_WRITE);
+ cq_entry.op_context = rx_entry->trecv->msg.context;
+ cq_entry.len = rx_entry->done;
+ cq_entry.buf = rx_entry->write.iov[0].iov_base;
+ cq_entry.data = rx_entry->op_hdr.data;
+ break;
+
+ case ofi_op_read_rsp:
+ return;
+
+ default:
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid op type: %d\n",
+ rx_entry->op_hdr.op);
+ break;
+ }
+
+ cq->write_fn(cq, &cq_entry);
+}
+
+void rxd_cq_report_error(struct rxd_cq *cq, struct fi_cq_err_entry *err_entry)
+{
+ struct fi_cq_tagged_entry cq_entry = {0};
+ struct util_cq_err_entry *entry = calloc(1, sizeof(*entry));
+ if (!entry) {
+ FI_WARN(&rxd_prov, FI_LOG_CQ,
+ "out of memory, cannot report CQ error\n");
+ return;
+ }
+
+ entry->err_entry = *err_entry;
+ slist_insert_tail(&entry->list_entry, &cq->util_cq.err_list);
+ cq_entry.flags = UTIL_FLAG_ERROR;
+ cq->write_fn(cq, &cq_entry);
+}
+
+void rxd_cq_report_tx_comp(struct rxd_cq *cq, struct rxd_tx_entry *tx_entry)
+{
+ struct fi_cq_tagged_entry cq_entry = {0};
+
+ /* todo: handle FI_COMPLETION */
+ switch(tx_entry->op_type) {
+ case RXD_TX_MSG:
+ cq_entry.flags = (FI_TRANSMIT | FI_MSG);
+ cq_entry.op_context = tx_entry->msg.msg.context;
+ cq_entry.len = tx_entry->op_hdr.size;
+ cq_entry.buf = tx_entry->msg.msg_iov[0].iov_base;
+ cq_entry.data = tx_entry->op_hdr.data;
+ break;
+
+ case RXD_TX_TAG:
+ cq_entry.flags = (FI_TRANSMIT | FI_TAGGED);
+ cq_entry.op_context = tx_entry->tmsg.tmsg.context;
+ cq_entry.len = tx_entry->op_hdr.size;
+ cq_entry.buf = tx_entry->tmsg.msg_iov[0].iov_base;
+ cq_entry.data = tx_entry->op_hdr.data;
+ cq_entry.tag = tx_entry->tmsg.tmsg.tag;
+ break;
+
+ case RXD_TX_WRITE:
+ cq_entry.flags = (FI_TRANSMIT | FI_RMA | FI_WRITE);
+ cq_entry.op_context = tx_entry->write.msg.context;
+ cq_entry.len = tx_entry->op_hdr.size;
+ cq_entry.buf = tx_entry->write.msg.msg_iov[0].iov_base;
+ cq_entry.data = tx_entry->op_hdr.data;
+ break;
+
+ case RXD_TX_READ_REQ:
+ cq_entry.flags = (FI_TRANSMIT | FI_RMA | FI_READ);
+ cq_entry.op_context = tx_entry->read_req.msg.context;
+ cq_entry.len = tx_entry->op_hdr.size;
+ cq_entry.buf = tx_entry->read_req.msg.msg_iov[0].iov_base;
+ cq_entry.data = tx_entry->op_hdr.data;
+ break;
+
+ case RXD_TX_READ_RSP:
+ return;
+
+ default:
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid op type\n");
+ return;
+ }
+
+ cq->write_fn(cq, &cq_entry);
+}
+
+void rxd_ep_handle_data_msg(struct rxd_ep *ep, struct rxd_peer *peer,
+ struct rxd_rx_entry *rx_entry,
+ struct iovec *iov, size_t iov_count,
+ struct ofi_ctrl_hdr *ctrl, void *data,
+ struct rxd_rx_buf *rx_buf)
+{
+
+ uint64_t done;
+
+ ep->credits++;
+ done = rxd_ep_copy_iov_buf(iov, iov_count, data, ctrl->seg_size,
+ rx_entry->done, RXD_COPY_BUF_TO_IOV);
+ rx_entry->done += done;
+ rx_entry->window--;
+ rx_entry->exp_seg_no++;
+
+ if (done != ctrl->seg_size) {
+ /* todo: generate truncation error */
+ /* inform peer */
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "TODO: message truncated\n");
+ }
+
+ if (rx_entry->window == 0) {
+ rx_entry->window = rxd_get_window_sz(ep, rx_entry->op_hdr.size - rx_entry->done);
+
+ rx_entry->last_win_seg += rx_entry->window;
+ ep->credits -= rx_entry->window;
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "replying ack [%p] - %d\n",
+ ctrl->msg_id, ctrl->seg_no);
+
+ rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, rx_entry->window,
+ rx_entry->key, peer->conn_data, ctrl->conn_id);
+ }
+
+ if (rx_entry->op_hdr.size != rx_entry->done) {
+ if (rx_entry->window == 0) {
+ dlist_init(&rx_entry->wait_entry);
+ dlist_insert_tail(&rx_entry->wait_entry, &ep->wait_rx_list);
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "rx-entry %p - %d enqueued\n",
+ ctrl->msg_id, ctrl->seg_no);
+ } else {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
+ "rx_entry->op_hdr.size: %d, rx_entry->done: %d\n",
+ rx_entry->op_hdr.size, rx_entry->done);
+ }
+ return;
+ }
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "reporting RX completion event\n");
+ rxd_report_rx_comp(ep->rx_cq, rx_entry);
+
+ switch(rx_entry->op_hdr.op) {
+ case ofi_op_msg:
+ freestack_push(ep->recv_fs, rx_entry->recv);
+ break;
+
+ case ofi_op_tagged:
+ freestack_push(ep->trecv_fs, rx_entry->trecv);
+ break;
+
+ case ofi_op_read_rsp:
+ rxd_cq_report_tx_comp(ep->tx_cq, rx_entry->read_rsp.tx_entry);
+ rxd_tx_entry_done(ep, rx_entry->read_rsp.tx_entry);
+ break;
+
+ default:
+ break;
+ }
+ rxd_rx_entry_release(ep, rx_entry);
+}
+
+static int rxd_check_data_pkt_order(struct rxd_ep *ep,
+ struct rxd_peer *peer,
+ struct ofi_ctrl_hdr *ctrl,
+ struct rxd_rx_entry *rx_entry)
+{
+ if (rx_entry->msg_id != ctrl->msg_id)
+ return RXD_PKT_ORDR_DUP;
+
+ if (rx_entry->exp_seg_no == ctrl->seg_no)
+ return RXD_PKT_ORDR_OK;
+
+ return (rx_entry->exp_seg_no > ctrl->seg_no) ?
+ RXD_PKT_ORDR_DUP : RXD_PKT_ORDR_UNEXP;
+}
+
+static inline void rxd_ep_enqueue_pkt(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
+ struct fi_cq_msg_entry *comp)
+{
+ struct rxd_unexp_cq_entry *unexp;
+ if (comp->flags & RXD_UNEXP_ENTRY ||
+ ep->num_unexp_pkt > RXD_EP_MAX_UNEXP_PKT)
+ return;
+
+ unexp = util_buf_alloc(ep->rx_cq->unexp_pool);
+ assert(unexp);
+ unexp->cq_entry = *comp;
+ unexp->cq_entry.flags |= RXD_UNEXP_ENTRY;
+
+ dlist_init(&unexp->entry);
+ dlist_insert_tail(&ep->rx_cq->unexp_list, &unexp->entry);
+ FI_INFO(&rxd_prov, FI_LOG_EP_CTRL,
+ "enqueuing unordered pkt: %p, seg_no: %d\n",
+ ctrl->msg_id, ctrl->seg_no);
+ ep->num_unexp_pkt++;
+}
+
+static inline void rxd_release_unexp_entry(struct rxd_cq *cq,
+ struct fi_cq_msg_entry *comp)
+{
+ struct rxd_unexp_cq_entry *unexp;
+ unexp = container_of(comp, struct rxd_unexp_cq_entry, cq_entry);
+ dlist_remove(&unexp->entry);
+ util_buf_release(cq->unexp_pool, unexp);
+}
+
+
+static int rxd_match_unexp_msg(struct dlist_entry *item, const void *arg)
+{
+ const struct rxd_recv_entry *recv_entry = arg;
+ struct rxd_rx_entry *rx_entry;
+
+ rx_entry = container_of(item, struct rxd_rx_entry, unexp_entry);
+ return (recv_entry->msg.addr == FI_ADDR_UNSPEC ||
+ rx_entry->source == FI_ADDR_UNSPEC ||
+ rx_entry->source == recv_entry->msg.addr);
+}
+
+void rxd_ep_check_unexp_msg_list(struct rxd_ep *ep, struct rxd_recv_entry *recv_entry)
+{
+ struct dlist_entry *match;
+ struct rxd_rx_entry *rx_entry;
+ struct rxd_pkt_data_start *pkt_start;
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "ep->num_unexp_msg: %d\n", ep->num_unexp_msg);
+ match = dlist_remove_first_match(&ep->unexp_msg_list, &rxd_match_unexp_msg,
+ (void *) recv_entry);
+ if (match) {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "progressing unexp msg entry\n");
+ dlist_remove(&recv_entry->entry);
+ ep->num_unexp_msg--;
+
+ rx_entry = container_of(match, struct rxd_rx_entry, unexp_entry);
+ rx_entry->recv = recv_entry;
+
+ pkt_start = (struct rxd_pkt_data_start *) rx_entry->unexp_buf->buf;
+ rxd_ep_handle_data_msg(ep, rx_entry->peer_info, rx_entry, rx_entry->recv->iov,
+ rx_entry->recv->msg.iov_count, &pkt_start->ctrl,
+ pkt_start->data, rx_entry->unexp_buf);
+ rxd_ep_repost_buff(rx_entry->unexp_buf);
+ }
+}
+
+static int rxd_match_unexp_tag(struct dlist_entry *item, const void *arg)
+{
+ const struct rxd_trecv_entry *trecv_entry = arg;
+ struct rxd_rx_entry *rx_entry;
+
+ rx_entry = container_of(item, struct rxd_rx_entry, unexp_entry);
+ return ((trecv_entry->msg.tag | trecv_entry->msg.ignore) ==
+ (rx_entry->op_hdr.tag | trecv_entry->msg.ignore) &&
+ ((trecv_entry->msg.addr == FI_ADDR_UNSPEC) ||
+ (rx_entry->source == FI_ADDR_UNSPEC) ||
+ (trecv_entry->msg.addr == rx_entry->source)));
+}
+
+void rxd_ep_check_unexp_tag_list(struct rxd_ep *ep, struct rxd_trecv_entry *trecv_entry)
+{
+ struct dlist_entry *match;
+ struct rxd_rx_entry *rx_entry;
+ struct rxd_pkt_data_start *pkt_start;
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "ep->num_unexp_msg: %d\n", ep->num_unexp_msg);
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "ep->num_unexp_pkt: %d\n", ep->num_unexp_pkt);
+ match = dlist_find_first_match(&ep->unexp_tag_list, &rxd_match_unexp_tag,
+ (void *) trecv_entry);
+ if (match) {
+ dlist_remove(match);
+ dlist_remove(&trecv_entry->entry);
+ ep->num_unexp_msg--;
+
+ rx_entry = container_of(match, struct rxd_rx_entry, unexp_entry);
+ rx_entry->trecv = trecv_entry;
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "progressing unexp tagged recv [%p]\n",
+ rx_entry->msg_id);
+
+ pkt_start = (struct rxd_pkt_data_start *) rx_entry->unexp_buf->buf;
+ rxd_ep_handle_data_msg(ep, rx_entry->peer_info, rx_entry, rx_entry->trecv->iov,
+ rx_entry->trecv->msg.iov_count, &pkt_start->ctrl,
+ pkt_start->data, rx_entry->unexp_buf);
+ rxd_ep_repost_buff(rx_entry->unexp_buf);
+ }
+}
+
+void rxd_handle_data(struct rxd_ep *ep, struct rxd_peer *peer,
+ struct ofi_ctrl_hdr *ctrl, struct fi_cq_msg_entry *comp,
+ struct rxd_rx_buf *rx_buf)
+{
+ int ret;
+ struct rxd_rx_entry *rx_entry;
+ struct rxd_tx_entry *tx_entry;
+ struct rxd_pkt_data *pkt_data = (struct rxd_pkt_data *) ctrl;
+ uint16_t win_sz;
+ uint64_t curr_stamp;
+
+ rxd_ep_lock_if_required(ep);
+ rx_entry = &ep->rx_entry_fs->buf[ctrl->rx_key];
+
+ ret = rxd_check_data_pkt_order(ep, peer, ctrl, rx_entry);
+ if (ret == RXD_PKT_ORDR_DUP) {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
+ "duplicate pkt: %d expected:%d, rx-key:%d, ctrl_msg_id: %p\n",
+ ctrl->seg_no, rx_entry->exp_seg_no, ctrl->rx_key, ctrl->msg_id);
+
+ win_sz = (rx_entry->msg_id == ctrl->msg_id &&
+ rx_entry->last_win_seg == ctrl->seg_no) ? rx_entry->window : 0;
+ rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, win_sz,
+ ctrl->rx_key, peer->conn_data, ctrl->conn_id);
+
+ goto repost;
+ } else if (ret == RXD_PKT_ORDR_UNEXP) {
+ if (!(comp->flags & RXD_UNEXP_ENTRY)) {
+ curr_stamp = fi_gettime_us();
+ if (rx_entry->nack_stamp == 0 ||
+ (curr_stamp > rx_entry->nack_stamp &&
+ curr_stamp - rx_entry->nack_stamp > RXD_RETRY_TIMEOUT)) {
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
+ "unexpected pkt, sending NACK: %d\n", ctrl->seg_no);
+
+ rx_entry->nack_stamp = curr_stamp;
+ rxd_ep_reply_nack(ep, ctrl, rx_entry->exp_seg_no,
+ ctrl->rx_key, peer->conn_data,
+ ctrl->conn_id);
+ }
+ rxd_ep_enqueue_pkt(ep, ctrl, comp);
+ }
+ goto out;
+ }
+
+ rx_entry->nack_stamp = 0;
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "expected pkt: %d\n", ctrl->seg_no);
+ switch (rx_entry->op_hdr.op) {
+ case ofi_op_msg:
+ rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->recv->iov,
+ rx_entry->recv->msg.iov_count, ctrl,
+ pkt_data->data, rx_buf);
+ break;
+
+ case ofi_op_tagged:
+ rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->trecv->iov,
+ rx_entry->trecv->msg.iov_count, ctrl,
+ pkt_data->data, rx_buf);
+ break;
+
+ case ofi_op_write:
+ rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->write.iov,
+ rx_entry->op_hdr.iov_count, ctrl,
+ pkt_data->data, rx_buf);
+ break;
+
+ case ofi_op_read_rsp:
+ tx_entry = rx_entry->read_rsp.tx_entry;
+ rxd_ep_handle_data_msg(ep, peer, rx_entry, tx_entry->read_req.dst_iov,
+ tx_entry->read_req.msg.iov_count, ctrl,
+ pkt_data->data, rx_buf);
+ break;
+
+ case ofi_op_atomic:
+ default:
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid op type\n");
+ }
+
+repost:
+ if (comp->flags & RXD_UNEXP_ENTRY) {
+ rxd_release_unexp_entry(ep->rx_cq, comp);
+ ep->num_unexp_pkt--;
+ }
+
+ rxd_ep_repost_buff(rx_buf);
+out:
+ rxd_ep_unlock_if_required(ep);
+}
+
+void rxd_ep_handle_read_req(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry,
+ struct rxd_peer *peer)
+{
+ int ret;
+
+ dlist_init(&tx_entry->pkt_list);
+ tx_entry->op_type = RXD_TX_READ_RSP;
+ ret = rxd_ep_post_start_msg(ep, peer, ofi_op_read_rsp, tx_entry);
+ if (ret)
+ goto err;
+
+ dlist_insert_tail(&tx_entry->entry, &ep->tx_entry_list);
+ return;
+err:
+ rxd_tx_entry_release(ep, tx_entry);
+ return;
+}
+
+int rxd_process_start_data(struct rxd_ep *ep, struct rxd_rx_entry *rx_entry,
+ struct rxd_peer *peer, struct ofi_ctrl_hdr *ctrl,
+ struct fi_cq_msg_entry *comp,
+ struct rxd_rx_buf *rx_buf)
+{
+ uint64_t idx;
+ int i, offset, ret;
+ struct ofi_rma_iov *rma_iov;
+ struct rxd_pkt_data_start *pkt_start;
+ struct rxd_tx_entry *tx_entry;
+ pkt_start = (struct rxd_pkt_data_start *) ctrl;
+
+ switch (rx_entry->op_hdr.op) {
+ case ofi_op_msg:
+ rx_entry->recv = rxd_get_recv_entry(ep, rx_entry);
+ if (!rx_entry->recv) {
+ if (ep->num_unexp_msg < RXD_EP_MAX_UNEXP_MSG) {
+ dlist_insert_tail(&rx_entry->unexp_entry, &ep->unexp_msg_list);
+ rx_entry->unexp_buf = rx_buf;
+ ep->num_unexp_msg++;
+ return -FI_ENOENT;
+ } else {
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "dropping msg\n");
+ return -FI_ENOMEM;
+ }
+ }
+
+ rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->recv->iov,
+ rx_entry->recv->msg.iov_count, ctrl,
+ pkt_start->data, rx_buf);
+ break;
+
+ case ofi_op_tagged:
+ rx_entry->trecv = rxd_get_trecv_entry(ep, rx_entry);
+ if (!rx_entry->trecv) {
+ if (ep->num_unexp_msg < RXD_EP_MAX_UNEXP_MSG) {
+ dlist_insert_tail(&rx_entry->unexp_entry, &ep->unexp_tag_list);
+ rx_entry->unexp_buf = rx_buf;
+ ep->num_unexp_msg++;
+ return -FI_ENOENT;
+ } else {
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "dropping msg\n");
+ return -FI_ENOMEM;
+ }
+ }
+
+ rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->trecv->iov,
+ rx_entry->trecv->msg.iov_count, ctrl,
+ pkt_start->data, rx_buf);
+ break;
+
+ case ofi_op_write:
+ rma_iov = (struct ofi_rma_iov *) pkt_start->data;
+ for (i = 0; i < rx_entry->op_hdr.iov_count; i++) {
+ ret = rxd_mr_verify(ep->domain,
+ rma_iov[i].len,
+ (uintptr_t *) &rma_iov[i].addr,
+ rma_iov[i].key, FI_WRITE);
+ if (ret) {
+ /* todo: handle invalid key case */
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid key/access permissions\n");
+ return -FI_EACCES;
+ }
+
+ rx_entry->write.iov[i].iov_base = (void *) (uintptr_t) rma_iov[i].addr;
+ rx_entry->write.iov[i].iov_len = rma_iov[i].len;
+ }
+
+ offset = sizeof(struct ofi_rma_iov) * rx_entry->op_hdr.iov_count;
+ ctrl->seg_size -= offset;
+ rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->write.iov,
+ rx_entry->op_hdr.iov_count, ctrl,
+ pkt_start->data + offset, rx_buf);
+ break;
+
+ case ofi_op_read_req:
+ rma_iov = (struct ofi_rma_iov *) pkt_start->data;
+ tx_entry = rxd_tx_entry_acquire_fast(ep, peer);
+ if (!tx_entry) {
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "no free tx-entry\n");
+ return -FI_ENOMEM;
+ }
+
+ tx_entry->peer = rx_entry->peer;
+ tx_entry->read_rsp.iov_count = rx_entry->op_hdr.iov_count;
+ for (i = 0; i < rx_entry->op_hdr.iov_count; i++) {
+ ret = rxd_mr_verify(ep->domain,
+ rma_iov[i].len,
+ (uintptr_t *) &rma_iov[i].addr,
+ rma_iov[i].key, FI_READ);
+ if (ret) {
+ /* todo: handle invalid key case */
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid key/access permissions\n");
+ return -FI_EACCES;
+ }
+
+ tx_entry->read_rsp.src_iov[i].iov_base = (void *) (uintptr_t) rma_iov[i].addr;
+ tx_entry->read_rsp.src_iov[i].iov_len = rma_iov[i].len;
+ }
+ tx_entry->read_rsp.peer_msg_id = ctrl->msg_id;
+ rxd_ep_handle_read_req(ep, tx_entry, peer);
+ rxd_rx_entry_release(ep, rx_entry);
+ break;
+
+ case ofi_op_read_rsp:
+ idx = rx_entry->op_hdr.remote_idx & RXD_TX_IDX_BITS;
+ tx_entry = &ep->tx_entry_fs->buf[idx];
+ if (tx_entry->msg_id != rx_entry->op_hdr.remote_idx)
+ return -FI_ENOMEM;
+
+ rx_entry->read_rsp.tx_entry = tx_entry;
+ rxd_ep_handle_data_msg(ep, peer, rx_entry, tx_entry->read_req.dst_iov,
+ tx_entry->read_req.msg.iov_count, ctrl,
+ pkt_start->data, rx_buf);
+ break;
+
+ case ofi_op_atomic:
+ default:
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid op type\n");
+ return -FI_EINVAL;
+ }
+ return 0;
+}
+
+void rxd_handle_start_data(struct rxd_ep *ep, struct rxd_peer *peer,
+ struct ofi_ctrl_hdr *ctrl,
+ struct fi_cq_msg_entry *comp,
+ struct rxd_rx_buf *rx_buf)
+{
+ int ret;
+ struct rxd_rx_entry *rx_entry;
+ struct rxd_pkt_data_start *pkt_start;
+ pkt_start = (struct rxd_pkt_data_start *) ctrl;
+
+ rxd_ep_lock_if_required(ep);
+ if (pkt_start->op.version != OFI_OP_VERSION) {
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "op version mismatch\n");
+ goto repost;
+ }
+
+ ret = rxd_check_start_pkt_order(ep, peer, ctrl, comp);
+ if (ret == RXD_PKT_ORDR_DUP) {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "duplicate pkt: %d\n", ctrl->seg_no);
+ rxd_handle_dup_datastart(ep, ctrl, rx_buf);
+ goto repost;
+ } else if (ret == RXD_PKT_ORDR_UNEXP) {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "unexpected pkt: %d\n", ctrl->seg_no);
+ rxd_ep_enqueue_pkt(ep, ctrl, comp);
+ goto out;
+ }
+
+ rx_entry = rxd_get_rx_entry(ep);
+ if (!rx_entry)
+ goto repost;
+
+ rx_entry->peer_info = peer;
+ rx_entry->op_hdr = pkt_start->op;
+ rx_entry->exp_seg_no = 0;
+ rx_entry->msg_id = ctrl->msg_id;
+ rx_entry->done = 0;
+ rx_entry->peer = ctrl->conn_id;
+ rx_entry->source = (ep->caps & FI_DIRECTED_RECV) ?
+ rxd_av_get_fi_addr(ep->av, ctrl->conn_id) : FI_ADDR_UNSPEC;
+ rx_entry->window = 1;
+ rx_entry->last_win_seg = 1;
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "Assign rx_entry :%d for %p\n",
+ rx_entry->key, rx_entry->msg_id);
+
+ ep->credits--;
+ ret = rxd_process_start_data(ep, rx_entry, peer, ctrl, comp, rx_buf);
+ if (ret == -FI_ENOMEM)
+ rxd_rx_entry_release(ep, rx_entry);
+ else if (ret == -FI_ENOENT) {
+ peer->exp_msg_id++;
+
+ /* reply ack, with win_sz = 0 */
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "Sending wait-ACK [%p] - %d\n",
+ ctrl->msg_id, ctrl->seg_no);
+ goto out;
+ } else {
+ peer->exp_msg_id++;
+ }
+
+repost:
+ if (comp->flags & RXD_UNEXP_ENTRY) {
+ rxd_release_unexp_entry(ep->rx_cq, comp);
+ ep->num_unexp_pkt--;
+ }
+ rxd_ep_repost_buff(rx_buf);
+out:
+ rxd_ep_unlock_if_required(ep);
+ return;
+}
+
+void rxd_handle_recv_comp(struct rxd_cq *cq, struct fi_cq_msg_entry *comp,
+ int is_unexpected)
+{
+ struct rxd_ep *ep;
+ struct ofi_ctrl_hdr *ctrl;
+ struct rxd_rx_buf *rx_buf;
+ struct rxd_peer *peer;
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "got recv completion\n");
+
+ rx_buf = container_of(comp->op_context, struct rxd_rx_buf, context);
+ ctrl = (struct ofi_ctrl_hdr *) rx_buf->buf;
+ ep = rx_buf->ep;
+ peer = rxd_ep_getpeer_info(ep, ctrl->conn_id);
+
+ if (ctrl->type != ofi_ctrl_ack && ctrl->type != ofi_ctrl_nack) {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
+ "got data pkt - msg_id:[%p - %d], type: %d on buf %p [unexp: %d]\n",
+ ctrl->msg_id, ctrl->seg_no, ctrl->type, rx_buf, is_unexpected);
+ } else {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
+ "got ack pkt - msg_id:[%p - %d], type: %d on buf %p [unexp: %d]\n",
+ ctrl->msg_id, ctrl->seg_no, ctrl->type, rx_buf, is_unexpected);
+ }
+ if (ctrl->version != OFI_CTRL_VERSION) {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "ctrl version mismatch\n");
+ return;
+ }
+
+ switch(ctrl->type) {
+ case ofi_ctrl_connreq:
+ rxd_handle_conn_req(ep, ctrl, comp, rx_buf);
+ break;
+
+ case ofi_ctrl_ack:
+ rxd_handle_ack(ep, ctrl, rx_buf);
+ break;
+
+ case ofi_ctrl_nack:
+ rxd_handle_nack(ep, ctrl, rx_buf);
+ break;
+
+ case ofi_ctrl_discard:
+ rxd_handle_discard(ep, ctrl, rx_buf);
+ break;
+
+ case ofi_ctrl_connresp:
+ rxd_handle_connect_ack(ep, ctrl, rx_buf);
+ break;
+
+ case ofi_ctrl_start_data:
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
+ "start data msg for tx: %p\n", ctrl->msg_id);
+ rxd_handle_start_data(ep, peer, ctrl, comp, rx_buf);
+ break;
+
+ case ofi_ctrl_data:
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
+ "data msg for tx: %p, %d \n", ctrl->msg_id, ctrl->seg_no);
+ rxd_handle_data(ep, peer, ctrl, comp, rx_buf);
+ break;
+
+ default:
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL,
+ "invalid ctrl type \n", ctrl->type);
+ }
+
+ rxd_ep_lock_if_required(ep);
+ rxd_check_waiting_rx(ep);
+ rxd_ep_unlock_if_required(ep);
+ return;
+}
+
+static inline void rxd_handle_send_comp(struct fi_cq_msg_entry *comp)
+{
+ struct rxd_pkt_meta *pkt_meta;
+ pkt_meta = container_of(comp->op_context, struct rxd_pkt_meta, context);
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "Send completion for: %p\n", pkt_meta);
+ rxd_ep_lock_if_required(pkt_meta->ep);
+ RXD_PKT_MARK_LOCAL_ACK(pkt_meta);
+ rxd_tx_pkt_release(pkt_meta);
+ rxd_ep_unlock_if_required(pkt_meta->ep);
+}
+
+void rxd_cq_progress(struct util_cq *util_cq)
+{
+ ssize_t ret = 0;
+ struct rxd_cq *cq;
+ struct fi_cq_msg_entry cq_entry;
+ struct dlist_entry *item, *next;
+ struct rxd_unexp_cq_entry *unexp;
+
+ cq = container_of(util_cq, struct rxd_cq, util_cq);
+ fastlock_acquire(&cq->lock);
+
+ do {
+ ret = fi_cq_read(cq->dg_cq, &cq_entry, 1);
+ if (ret == -FI_EAGAIN)
+ break;
+
+ if (cq_entry.flags & FI_SEND) {
+ rxd_handle_send_comp(&cq_entry);
+ } else if (cq_entry.flags & FI_RECV) {
+ rxd_handle_recv_comp(cq, &cq_entry, 0);
+ } else
+ assert (0);
+ } while (ret > 0);
+
+ for (item = cq->unexp_list.next; item != &cq->unexp_list;) {
+ unexp = container_of(item, struct rxd_unexp_cq_entry, entry);
+ next = item->next;
+ rxd_handle_recv_comp(cq, &unexp->cq_entry, 1);
+ item = next;
+ }
+
+ fastlock_release(&cq->lock);
+}
+
+static int rxd_cq_close(struct fid *fid)
+{
+ int ret;
+ struct rxd_cq *cq;
+
+ cq = container_of(fid, struct rxd_cq, util_cq.cq_fid.fid);
+
+ fastlock_acquire(&cq->domain->lock);
+ dlist_remove(&cq->dom_entry);
+ fastlock_release(&cq->domain->lock);
+ fastlock_destroy(&cq->lock);
+
+ ret = fi_close(&cq->dg_cq->fid);
+ if (ret)
+ return ret;
+
+ ret = ofi_cq_cleanup(&cq->util_cq);
+ if (ret)
+ return ret;
+ util_buf_pool_destroy(cq->unexp_pool);
+ free(cq);
+ return 0;
+}
+
+static struct fi_ops rxd_cq_fi_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = rxd_cq_close,
+ .bind = fi_no_bind,
+ .control = fi_no_control,
+ .ops_open = fi_no_ops_open,
+};
+
+int rxd_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
+ struct fid_cq **cq_fid, void *context)
+{
+ int ret;
+ struct rxd_cq *cq;
+ struct rxd_domain *rxd_domain;
+
+ cq = calloc(1, sizeof(*cq));
+ if (!cq)
+ return -FI_ENOMEM;
+
+ ret = ofi_cq_init(&rxd_prov, domain, attr, &cq->util_cq,
+ &rxd_cq_progress, context);
+ if (ret)
+ goto err1;
+
+ switch (attr->format) {
+ case FI_CQ_FORMAT_UNSPEC:
+ case FI_CQ_FORMAT_CONTEXT:
+ cq->write_fn = cq->util_cq.wait ?
+ rxd_cq_write_ctx_signal : rxd_cq_write_ctx;
+ break;
+ case FI_CQ_FORMAT_MSG:
+ cq->write_fn = cq->util_cq.wait ?
+ rxd_cq_write_msg_signal : rxd_cq_write_msg;
+ break;
+ case FI_CQ_FORMAT_DATA:
+ cq->write_fn = cq->util_cq.wait ?
+ rxd_cq_write_data_signal : rxd_cq_write_data;
+ break;
+ case FI_CQ_FORMAT_TAGGED:
+ cq->write_fn = cq->util_cq.wait ?
+ rxd_cq_write_tagged_signal : rxd_cq_write_tagged;
+ break;
+ default:
+ ret = -FI_EINVAL;
+ goto err2;
+ }
+
+ rxd_domain = container_of(domain, struct rxd_domain, util_domain.domain_fid);
+ attr->format = FI_CQ_FORMAT_MSG;
+ ret = fi_cq_open(rxd_domain->dg_domain, attr, &cq->dg_cq, context);
+ if (ret)
+ goto err2;
+
+ cq->unexp_pool = util_buf_pool_create(
+ RXD_EP_MAX_UNEXP_PKT * sizeof (struct rxd_unexp_cq_entry),
+ RXD_BUF_POOL_ALIGNMENT, 0, RXD_EP_MAX_UNEXP_PKT);
+ if (!cq->unexp_pool) {
+ ret = -FI_ENOMEM;
+ goto err3;
+ }
+
+ dlist_init(&cq->dom_entry);
+ dlist_init(&cq->unexp_list);
+ fastlock_init(&cq->lock);
+
+ fastlock_acquire(&rxd_domain->lock);
+ dlist_insert_tail(&cq->dom_entry, &rxd_domain->cq_list);
+ fastlock_release(&rxd_domain->lock);
+
+ *cq_fid = &cq->util_cq.cq_fid;
+ (*cq_fid)->fid.ops = &rxd_cq_fi_ops;
+ *cq_fid = &cq->util_cq.cq_fid;
+ cq->domain = rxd_domain;
+ return 0;
+
+err3:
+ ofi_cq_cleanup(&cq->util_cq);
+err2:
+ fi_close(&cq->dg_cq->fid);
+err1:
+ free(cq);
+ return ret;
+}
diff --git a/prov/rxd/src/rxd_domain.c b/prov/rxd/src/rxd_domain.c
new file mode 100644
index 0000000..312df84
--- /dev/null
+++ b/prov/rxd/src/rxd_domain.c
@@ -0,0 +1,303 @@
+/*
+ * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+
+#include "rxd.h"
+
+
+static struct fi_ops_domain rxd_domain_ops = {
+ .size = sizeof(struct fi_ops_domain),
+ .av_open = rxd_av_create,
+ .cq_open = rxd_cq_open,
+ .endpoint = rxd_endpoint,
+ .scalable_ep = fi_no_scalable_ep,
+ .cntr_open = fi_no_cntr_open,
+ .poll_open = fi_poll_create,
+ .stx_ctx = fi_no_stx_context,
+ .srx_ctx = fi_no_srx_context,
+};
+
+static int rxd_domain_close(fid_t fid)
+{
+ int ret;
+ struct rxd_domain *rxd_domain;
+
+ rxd_domain = container_of(fid, struct rxd_domain, util_domain.domain_fid.fid);
+
+ ret = fi_close(&rxd_domain->dg_domain->fid);
+ if (ret)
+ return ret;
+
+ ret = ofi_domain_close(&rxd_domain->util_domain);
+ if (ret)
+ return ret;
+
+ ofi_mr_close(rxd_domain->mr_heap);
+ rxd_domain->do_progress = 0;
+ pthread_join(rxd_domain->progress_thread, NULL);
+ fastlock_destroy(&rxd_domain->lock);
+ fastlock_destroy(&rxd_domain->mr_lock);
+ free(rxd_domain);
+ return 0;
+}
+
+static struct fi_ops rxd_domain_fi_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = rxd_domain_close,
+ .bind = fi_no_bind,
+ .control = fi_no_control,
+ .ops_open = fi_no_ops_open,
+};
+
+void *rxd_progress(void *arg)
+{
+ struct rxd_cq *cq;
+ struct rxd_ep *ep;
+ struct dlist_entry *item;
+ struct rxd_domain *domain = arg;
+
+ while(domain->do_progress) {
+ fastlock_acquire(&domain->lock);
+ dlist_foreach(&domain->cq_list, item) {
+ cq = container_of(item, struct rxd_cq, dom_entry);
+ rxd_cq_progress(&cq->util_cq);
+ }
+
+ dlist_foreach(&domain->ep_list, item) {
+ ep = container_of(item, struct rxd_ep, dom_entry);
+ rxd_ep_progress(ep);
+ }
+ fastlock_release(&domain->lock);
+ }
+ return NULL;
+}
+struct rxd_mr_entry {
+ struct fid_mr mr_fid;
+ struct rxd_domain *domain;
+ uint64_t key;
+ uint64_t flags;
+};
+
+static int rxd_mr_close(struct fid *fid)
+{
+ struct rxd_domain *dom;
+ struct rxd_mr_entry *mr;
+ uint64_t mr_key;
+ int err = 0;
+
+ mr = container_of(fid, struct rxd_mr_entry, mr_fid.fid);
+ dom = mr->domain;
+ mr_key = mr->key;
+
+ fastlock_acquire(&dom->lock);
+ err = ofi_mr_erase(dom->mr_heap, mr_key);
+ if (err)
+ return err;
+
+ fastlock_release(&dom->lock);
+ atomic_dec(&dom->util_domain.ref);
+ free(mr);
+ return 0;
+}
+
+static struct fi_ops rxd_mr_fi_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = rxd_mr_close,
+ .control = fi_no_control,
+ .ops_open = fi_no_ops_open,
+};
+
+
+static int rxd_mr_regattr(struct fid *fid, const struct fi_mr_attr *attr,
+ uint64_t flags, struct fid_mr **mr)
+{
+ struct rxd_domain *dom;
+ struct rxd_mr_entry *_mr;
+ uint64_t key;
+ int ret = 0;
+
+ if (fid->fclass != FI_CLASS_DOMAIN || !attr || attr->iov_count <= 0) {
+ return -FI_EINVAL;
+ }
+
+ dom = container_of(fid, struct rxd_domain, util_domain.domain_fid.fid);
+ _mr = calloc(1, sizeof(*_mr));
+ if (!_mr)
+ return -FI_ENOMEM;
+
+ fastlock_acquire(&dom->mr_lock);
+
+ _mr->mr_fid.fid.fclass = FI_CLASS_MR;
+ _mr->mr_fid.fid.context = attr->context;
+ _mr->mr_fid.fid.ops = &rxd_mr_fi_ops;
+
+ _mr->domain = dom;
+ _mr->flags = flags;
+
+ ret = ofi_mr_insert(dom->mr_heap, attr, &key, _mr);
+ if (ret != 0) {
+ goto err;
+ }
+
+ _mr->mr_fid.key = _mr->key = key;
+ _mr->mr_fid.mem_desc = (void *) (uintptr_t) key;
+ fastlock_release(&dom->mr_lock);
+
+ *mr = &_mr->mr_fid;
+ atomic_inc(&dom->util_domain.ref);
+
+ return 0;
+err:
+ fastlock_release(&dom->mr_lock);
+ free(_mr);
+ return ret;
+}
+
+static int rxd_mr_regv(struct fid *fid, const struct iovec *iov,
+ size_t count, uint64_t access,
+ uint64_t offset, uint64_t requested_key,
+ uint64_t flags, struct fid_mr **mr, void *context)
+{
+ struct fi_mr_attr attr;
+
+ attr.mr_iov = iov;
+ attr.iov_count = count;
+ attr.access = access;
+ attr.offset = offset;
+ attr.requested_key = requested_key;
+ attr.context = context;
+ return rxd_mr_regattr(fid, &attr, flags, mr);
+}
+
+static int rxd_mr_reg(struct fid *fid, const void *buf, size_t len,
+ uint64_t access, uint64_t offset, uint64_t requested_key,
+ uint64_t flags, struct fid_mr **mr, void *context)
+{
+ struct iovec iov;
+
+ iov.iov_base = (void *) buf;
+ iov.iov_len = len;
+ return rxd_mr_regv(fid, &iov, 1, access, offset, requested_key,
+ flags, mr, context);
+}
+
+static struct fi_ops_mr rxd_mr_ops = {
+ .size = sizeof(struct fi_ops_mr),
+ .reg = rxd_mr_reg,
+ .regv = rxd_mr_regv,
+ .regattr = rxd_mr_regattr,
+};
+
+int rxd_mr_verify(struct rxd_domain *rxd_domain, ssize_t len,
+ uintptr_t *io_addr, uint64_t key, uint64_t access)
+{
+ int ret;
+ fastlock_acquire(&rxd_domain->mr_lock);
+ ret = ofi_mr_retrieve_and_verify(rxd_domain->mr_heap, len,
+ io_addr, key, access, NULL);
+ fastlock_release(&rxd_domain->mr_lock);
+ return ret;
+}
+
+int rxd_domain_open(struct fid_fabric *fabric, struct fi_info *info,
+ struct fid_domain **domain, void *context)
+{
+ int ret;
+ struct fi_info *dg_info;
+ struct rxd_domain *rxd_domain;
+ struct rxd_fabric *rxd_fabric;
+
+ ret = fi_check_info(&rxd_util_prov, info, FI_MATCH_PREFIX);
+ if (ret)
+ return ret;
+
+ rxd_domain = calloc(1, sizeof(*rxd_domain));
+ if (!rxd_domain)
+ return -FI_ENOMEM;
+
+ ret = ofix_getinfo(rxd_prov.version, NULL, NULL, 0, &rxd_util_prov,
+ info, rxd_alter_layer_info,
+ rxd_alter_base_info, 1, &dg_info);
+ if (ret)
+ goto err1;
+
+
+ rxd_fabric = container_of(fabric, struct rxd_fabric, util_fabric.fabric_fid);
+ ret = fi_domain(rxd_fabric->dg_fabric, dg_info, &rxd_domain->dg_domain, context);
+ if (ret)
+ goto err2;
+
+ rxd_domain->max_mtu_sz = dg_info->ep_attr->max_msg_size;
+ rxd_domain->dg_mode = dg_info->mode;
+ rxd_domain->addrlen = (info->src_addr) ? info->src_addrlen : info->dest_addrlen;
+
+ ret = ofi_domain_init(fabric, info, &rxd_domain->util_domain, context);
+ if (ret) {
+ goto err3;
+ }
+
+ dlist_init(&rxd_domain->ep_list);
+ dlist_init(&rxd_domain->cq_list);
+ fastlock_init(&rxd_domain->lock);
+ fastlock_init(&rxd_domain->mr_lock);
+
+ ret = ofi_mr_init(&rxd_prov, info->domain_attr->mr_mode, &rxd_domain->mr_heap);
+ if (ret)
+ goto err4;
+
+ rxd_domain->do_progress = 1;
+ if (pthread_create(&rxd_domain->progress_thread, NULL,
+ rxd_progress, rxd_domain)) {
+ goto err5;
+ }
+
+ *domain = &rxd_domain->util_domain.domain_fid;
+ (*domain)->fid.ops = &rxd_domain_fi_ops;
+ (*domain)->ops = &rxd_domain_ops;
+ (*domain)->mr = &rxd_mr_ops;
+ fi_freeinfo(dg_info);
+ return 0;
+err5:
+ ofi_mr_close(rxd_domain->mr_heap);
+err4:
+ ofi_domain_close(&rxd_domain->util_domain);
+err3:
+ fi_close(&rxd_domain->dg_domain->fid);
+err2:
+ fi_freeinfo(dg_info);
+err1:
+ free(rxd_domain);
+ return ret;
+}
diff --git a/prov/rxd/src/rxd_ep.c b/prov/rxd/src/rxd_ep.c
new file mode 100644
index 0000000..afc8c78
--- /dev/null
+++ b/prov/rxd/src/rxd_ep.c
@@ -0,0 +1,1905 @@
+/*
+ * Copyright (c) 2013-2016 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <fi_mem.h>
+#include "rxd.h"
+
+static ssize_t rxd_ep_cancel(fid_t fid, void *context)
+{
+ struct rxd_ep *ep;
+ struct dlist_entry *entry, *next;
+ struct rxd_recv_entry *recv_entry;
+ struct rxd_trecv_entry *trecv_entry;
+ struct fi_cq_err_entry err_entry = {0};
+
+ ep = container_of(fid, struct rxd_ep, ep.fid);
+ rxd_ep_lock_if_required(ep);
+ for (entry = ep->recv_list.next; entry != &ep->recv_list; entry = next) {
+ next = entry->next;
+ recv_entry = container_of(entry, struct rxd_recv_entry, entry);
+ if (recv_entry->msg.context != context)
+ continue;
+
+ dlist_remove(entry);
+ err_entry.op_context = recv_entry->msg.context;
+ err_entry.flags = (FI_MSG | FI_RECV);
+ err_entry.err = FI_ECANCELED;
+ err_entry.prov_errno = -FI_ECANCELED;
+ rxd_cq_report_error(ep->rx_cq, &err_entry);
+ goto out;
+ }
+
+ for (entry = ep->trecv_list.next; entry != &ep->trecv_list; entry = next) {
+ next = entry->next;
+ trecv_entry = container_of(entry, struct rxd_trecv_entry, entry);
+ if (trecv_entry->msg.context != context)
+ continue;
+
+ dlist_remove(entry);
+ err_entry.op_context = trecv_entry->msg.context;
+ err_entry.flags = (FI_MSG | FI_RECV | FI_TAGGED);
+ err_entry.tag = trecv_entry->msg.tag;
+ err_entry.err = FI_ECANCELED;
+ err_entry.prov_errno = -FI_ECANCELED;
+ rxd_cq_report_error(ep->rx_cq, &err_entry);
+ goto out;
+ }
+
+out:
+ rxd_ep_unlock_if_required(ep);
+ return 0;
+}
+
+static int rxd_ep_getopt(fid_t fid, int level, int optname,
+ void *optval, size_t *optlen)
+{
+ return -FI_ENOSYS;
+}
+
+static int rxd_ep_setopt(fid_t fid, int level, int optname,
+ const void *optval, size_t optlen)
+{
+ return -FI_ENOSYS;
+}
+
+struct fi_ops_ep rxd_ops_ep = {
+ .size = sizeof(struct fi_ops_ep),
+ .cancel = rxd_ep_cancel,
+ .getopt = rxd_ep_getopt,
+ .setopt = rxd_ep_setopt,
+ .tx_ctx = fi_no_tx_ctx,
+ .rx_ctx = fi_no_rx_ctx,
+ .rx_size_left = fi_no_rx_size_left,
+ .tx_size_left = fi_no_tx_size_left,
+};
+
+static ssize_t rxd_ep_recvmsg(struct fid_ep *ep, const struct fi_msg *msg,
+ uint64_t flags)
+{
+ ssize_t ret = 0, i;
+ struct rxd_ep *rxd_ep;
+ struct rxd_recv_entry *recv_entry;
+
+ rxd_ep = container_of(ep, struct rxd_ep, ep);
+
+ rxd_ep_lock_if_required(rxd_ep);
+ if (freestack_isempty(rxd_ep->recv_fs)) {
+ ret = -FI_EAGAIN;
+ goto out;
+ }
+
+ recv_entry = freestack_pop(rxd_ep->recv_fs);
+ recv_entry->msg = *msg;
+ recv_entry->flags = flags;
+ recv_entry->msg.addr = (rxd_ep->caps & FI_DIRECTED_RECV) ?
+ recv_entry->msg.addr : FI_ADDR_UNSPEC;
+ for (i = 0; i < msg->iov_count; i++) {
+ recv_entry->iov[i].iov_base = msg->msg_iov[i].iov_base;
+ recv_entry->iov[i].iov_len = msg->msg_iov[i].iov_len;
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "post recv: %u\n",
+ msg->msg_iov[i].iov_len);
+ }
+
+ dlist_init(&recv_entry->entry);
+ dlist_insert_tail(&recv_entry->entry, &rxd_ep->recv_list);
+
+ if (!dlist_empty(&rxd_ep->unexp_msg_list)) {
+ rxd_ep_check_unexp_msg_list(rxd_ep, recv_entry);
+ }
+out:
+ rxd_ep_unlock_if_required(rxd_ep);
+ return ret;
+}
+
+static ssize_t rxd_ep_recv(struct fid_ep *ep, void *buf, size_t len, void *desc,
+ fi_addr_t src_addr, void *context)
+{
+ struct fi_msg msg;
+ struct iovec msg_iov;
+ memset(&msg, 0, sizeof(msg));
+ msg_iov.iov_base = buf;
+ msg_iov.iov_len = len;
+
+ msg.msg_iov = &msg_iov;
+ msg.desc = &desc;
+ msg.iov_count = 1;
+ msg.addr = src_addr;
+ msg.context = context;
+ msg.data = 0;
+ return rxd_ep_recvmsg(ep, &msg, RXD_USE_OP_FLAGS);
+}
+
+static ssize_t rxd_ep_recvv(struct fid_ep *ep, const struct iovec *iov, void **desc,
+ size_t count, fi_addr_t src_addr, void *context)
+{
+ struct fi_msg msg;
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_iov = iov;
+ msg.desc = desc;
+ msg.iov_count = count;
+ msg.addr = src_addr;
+ msg.context = context;
+ msg.data = 0;
+ return rxd_ep_recvmsg(ep, &msg, RXD_USE_OP_FLAGS);
+}
+
+static inline void *rxd_mr_desc(struct fid_mr *mr, struct rxd_ep *ep)
+{
+ return (ep->do_local_mr) ? fi_mr_desc(mr) : NULL;
+}
+
+int rxd_ep_repost_buff(struct rxd_rx_buf *buf)
+{
+ int ret;
+ ret = fi_recv(buf->ep->dg_ep, buf->buf, buf->ep->domain->max_mtu_sz,
+ rxd_mr_desc(buf->mr, buf->ep),
+ FI_ADDR_UNSPEC, &buf->context);
+ if (ret)
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "failed to repost\n");
+ return ret;
+}
+
+static int rxd_ep_set_conn_id(struct rxd_ep *ep)
+{
+ int ret;
+ ep->name = calloc(1, ep->addrlen);
+ if (!ep->name)
+ return -FI_ENOMEM;
+
+ ret = fi_getname(&ep->dg_ep->fid, ep->name, &ep->addrlen);
+ if (ret)
+ return -FI_EINVAL;
+
+ ret = rxd_av_dg_reverse_lookup(ep->av, 0, ep->name, ep->addrlen,
+ &ep->conn_data);
+ if (!ret)
+ ep->conn_data_set = 1;
+ return 0;
+}
+
+int rxd_ep_enable(struct rxd_ep *ep)
+{
+ ssize_t i, ret;
+ void *mr = NULL;
+ struct rxd_rx_buf *rx_buf;
+
+ ret = fi_enable(ep->dg_ep);
+ if (ret)
+ return ret;
+
+ rxd_ep_lock_if_required(ep);
+ ret = rxd_ep_set_conn_id(ep);
+ if (ret)
+ goto out;
+
+ ep->credits = ep->rx_size;
+ for (i = 0; i < ep->rx_size; i++) {
+ rx_buf = ep->do_local_mr ?
+ util_buf_get_ex(ep->rx_pkt_pool, &mr) :
+ util_buf_get(ep->rx_pkt_pool);
+
+ if (!rx_buf) {
+ ret = -FI_ENOMEM;
+ goto out;
+ }
+
+ rx_buf->mr = (struct fid_mr *) mr;
+ rx_buf->ep = ep;
+ ret = rxd_ep_repost_buff(rx_buf);
+ if (ret)
+ goto out;
+ slist_insert_tail(&rx_buf->entry, &ep->rx_pkt_list);
+ }
+out:
+ rxd_ep_unlock_if_required(ep);
+ if (ret == 0) {
+ fastlock_acquire(&ep->domain->lock);
+ dlist_insert_tail(&ep->dom_entry, &ep->domain->ep_list);
+ fastlock_release(&ep->domain->lock);
+ }
+ return ret;
+}
+
+struct rxd_peer *rxd_ep_getpeer_info(struct rxd_ep *ep, fi_addr_t addr)
+{
+ return &ep->peer_info[addr];
+}
+
+void rxd_ep_lock_if_required(struct rxd_ep *ep)
+{
+ /* todo: do locking based on threading model */
+ fastlock_acquire(&ep->lock);
+}
+
+void rxd_ep_unlock_if_required(struct rxd_ep *ep)
+{
+ /* todo: do unlocking based on threading model */
+ fastlock_release(&ep->lock);
+}
+
+size_t rxd_get_msg_len(const struct iovec *iov, size_t iov_count)
+{
+ size_t i, ret = 0;
+ for (i = 0; i < iov_count; i++)
+ ret += iov[i].iov_len;
+ return ret;
+}
+
+static void rxd_init_ctrl_hdr(struct ofi_ctrl_hdr *ctrl,
+ uint8_t type, uint16_t seg_size,
+ uint32_t seg_no, uint64_t msg_id,
+ uint64_t rx_key, uint64_t source)
+{
+ ctrl->version = OFI_CTRL_VERSION;
+ ctrl->type = type;
+ ctrl->seg_size = seg_size;
+ ctrl->seg_no = seg_no;
+ ctrl->msg_id = msg_id;
+ ctrl->rx_key = rx_key;
+ ctrl->conn_id = source;
+}
+
+static void rxd_init_op_hdr(struct ofi_op_hdr *op, uint64_t data,
+ uint64_t msg_sz, uint8_t rx_index,
+ uint8_t op_type, uint64_t tag, uint32_t flags)
+{
+ op->version = OFI_OP_VERSION;
+ op->rx_index = rx_index;
+ op->op = op_type;
+ op->op_data = 0; /* unused */
+ op->flags = flags;
+ op->size = msg_sz;
+ op->data = data;
+ op->tag = tag;
+}
+
+static uint32_t rxd_prepare_tx_flags(uint64_t flags)
+{
+ uint32_t tx_flags = 0;
+ if (flags & FI_REMOTE_CQ_DATA)
+ tx_flags = OFI_REMOTE_CQ_DATA;
+ if (flags & FI_TRANSMIT_COMPLETE)
+ tx_flags |= OFI_TRANSMIT_COMPLETE;
+ if (flags & FI_DELIVERY_COMPLETE)
+ tx_flags |= OFI_DELIVERY_COMPLETE;
+ return tx_flags;
+}
+
+uint64_t rxd_ep_copy_iov_buf(const struct iovec *iov, size_t iov_count,
+ void *buf, uint64_t data_sz, uint64_t skip, int dir)
+{
+ int i;
+ uint64_t rem, offset, len, iov_offset;
+ offset = 0, rem = data_sz, iov_offset = 0;
+
+ for (i = 0; i < iov_count; i++) {
+ len = iov[i].iov_len;
+ iov_offset = 0;
+
+ if (skip) {
+ iov_offset = MIN(skip, len);
+ skip -= iov_offset;
+ len -= iov_offset;
+ }
+
+ len = MIN(rem, len);
+ if (dir == RXD_COPY_BUF_TO_IOV)
+ memcpy((char *) iov[i].iov_base + iov_offset,
+ (char *) buf + offset, len);
+ else if (dir == RXD_COPY_IOV_TO_BUF)
+ memcpy((char *) buf + offset,
+ (char *) iov[i].iov_base + iov_offset, len);
+ rem -= len, offset += len;
+ }
+ return offset;
+}
+
+struct rxd_pkt_meta *rxd_tx_pkt_acquire(struct rxd_ep *ep)
+{
+ struct rxd_pkt_meta *pkt_meta;
+ void *mr = NULL;
+
+ pkt_meta = ep->do_local_mr ?
+ util_buf_alloc_ex(ep->tx_pkt_pool, &mr) :
+ util_buf_alloc(ep->tx_pkt_pool);
+
+ if (!pkt_meta) {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "No free tx pkt\n");
+ return NULL;
+ }
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "Acquired tx pkt: %p\n", pkt_meta);
+ pkt_meta->ep = ep;
+ pkt_meta->retries = 0;
+ pkt_meta->mr = (struct fid_mr *) mr;
+ pkt_meta->ref = 0;
+ return pkt_meta;
+}
+
+static uint64_t rxd_ep_copy_data(struct rxd_tx_entry *tx_entry,
+ struct rxd_pkt_data *pkt, uint64_t data_sz)
+{
+ uint64_t done;
+ switch(tx_entry->op_hdr.op) {
+ case ofi_op_msg:
+ done = rxd_ep_copy_iov_buf(tx_entry->msg.msg_iov,
+ tx_entry->msg.msg.iov_count,
+ pkt->data, data_sz, tx_entry->done,
+ RXD_COPY_IOV_TO_BUF);
+ break;
+
+ case ofi_op_tagged:
+ done = rxd_ep_copy_iov_buf(tx_entry->tmsg.msg_iov,
+ tx_entry->tmsg.tmsg.iov_count,
+ pkt->data, data_sz, tx_entry->done,
+ RXD_COPY_IOV_TO_BUF);
+ break;
+
+ case ofi_op_write:
+ done = rxd_ep_copy_iov_buf(tx_entry->write.src_iov,
+ tx_entry->write.msg.iov_count,
+ pkt->data, data_sz, tx_entry->done,
+ RXD_COPY_IOV_TO_BUF);
+ break;
+
+ case ofi_op_read_rsp:
+ done = rxd_ep_copy_iov_buf(tx_entry->read_rsp.src_iov,
+ tx_entry->read_rsp.iov_count,
+ pkt->data, data_sz, tx_entry->done,
+ RXD_COPY_IOV_TO_BUF);
+ break;
+
+ default:
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid op-type\n",
+ tx_entry->op_hdr.op);
+ done = 0;
+ }
+ return done;
+}
+
+ssize_t rxd_ep_post_data_msg(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry)
+{
+ int ret;
+ uint64_t data_sz, done;
+ struct rxd_pkt_meta *pkt_meta;
+ struct rxd_pkt_data *pkt;
+ struct rxd_peer *peer;
+
+ peer = rxd_ep_getpeer_info(ep, tx_entry->peer);
+ pkt_meta = rxd_tx_pkt_acquire(ep);
+ if (!pkt_meta)
+ return -FI_ENOMEM;
+
+ pkt = (struct rxd_pkt_data *)pkt_meta->pkt_data;
+ data_sz = MIN(RXD_MAX_DATA_PKT_SZ(ep), tx_entry->op_hdr.size - tx_entry->done);
+
+ rxd_init_ctrl_hdr(&pkt->ctrl, ofi_ctrl_data, data_sz, tx_entry->nxt_seg_no,
+ tx_entry->msg_id, tx_entry->rx_key, peer->conn_data);
+ done = rxd_ep_copy_data(tx_entry, pkt, data_sz);
+
+ pkt_meta->tx_entry = tx_entry;
+ pkt_meta->type = (tx_entry->op_hdr.size == tx_entry->done + done) ?
+ RXD_PKT_LAST : RXD_PKT_DATA;
+ pkt_meta->us_stamp = fi_gettime_us();
+
+ ret = fi_send(ep->dg_ep, pkt, data_sz + RXD_DATA_PKT_SZ,
+ rxd_mr_desc(pkt_meta->mr, ep), tx_entry->peer, &pkt_meta->context);
+ if (ret) {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "send %d failed\n", pkt->ctrl.seg_no);
+ util_buf_release(ep->tx_pkt_pool, pkt_meta);
+ return ret;
+ }
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "sent data %p, %d [on buf: %p]\n",
+ pkt->ctrl.msg_id, pkt->ctrl.seg_no, pkt_meta);
+ tx_entry->done += done;
+ tx_entry->win_sz--;
+ tx_entry->nxt_seg_no++;
+ tx_entry->num_unacked++;
+
+ dlist_insert_tail(&pkt_meta->entry, &tx_entry->pkt_list);
+ ep->num_out++;
+ return 0;
+}
+
+void rxd_ep_free_acked_pkts(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry,
+ uint32_t seg_no)
+{
+ struct dlist_entry *next, *curr;
+ struct rxd_pkt_meta *pkt;
+ struct ofi_ctrl_hdr *ctrl;
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "freeing all [%p] pkts <= %d\n",
+ tx_entry->msg_id, seg_no);
+ for (curr = tx_entry->pkt_list.next; curr != &tx_entry->pkt_list;) {
+ next = curr->next;
+ pkt = container_of(curr, struct rxd_pkt_meta, entry);
+ ctrl = (struct ofi_ctrl_hdr *) pkt->pkt_data;
+ if (ctrl->seg_no <= seg_no) {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "freeing [%p] pkt:%d\n",
+ tx_entry->msg_id, ctrl->seg_no);
+ dlist_remove(curr);
+ RXD_PKT_MARK_REMOTE_ACK(pkt);
+ rxd_tx_pkt_release(pkt);
+ tx_entry->num_unacked--;
+ } else {
+ break;
+ }
+ curr = next;
+ }
+}
+
+void rxd_tx_entry_update_ts(struct rxd_tx_entry *tx_entry, uint32_t seg_no)
+{
+ struct dlist_entry *curr;
+ struct rxd_pkt_meta *pkt;
+ struct ofi_ctrl_hdr *ctrl;
+
+ for (curr = tx_entry->pkt_list.next; curr != &tx_entry->pkt_list;
+ curr = curr->next) {
+ pkt = container_of(curr, struct rxd_pkt_meta, entry);
+ ctrl = (struct ofi_ctrl_hdr *) pkt->pkt_data;
+ if (ctrl->seg_no == seg_no) {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "updating TS of [%p] pkt:%d\n",
+ tx_entry->msg_id, seg_no);
+ pkt->us_stamp += RXD_WAIT_TIMEOUT;
+ break;
+ }
+ }
+}
+
+int rxd_progress_tx(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry)
+{
+ int ret = 0;
+
+ switch (tx_entry->op_hdr.op) {
+ case ofi_op_msg:
+ case ofi_op_tagged:
+ case ofi_op_write:
+ case ofi_op_read_rsp:
+ ret = rxd_ep_post_data_msg(ep, tx_entry);
+ break;
+
+ default:
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid pkt type\n");
+ break;
+ }
+ return ret;
+}
+
+void rxd_tx_entry_discard(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry)
+{
+ rxd_cq_report_tx_comp(ep->tx_cq, tx_entry);
+ rxd_tx_entry_done(ep, tx_entry);
+}
+
+void rxd_resend_pkt(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry,
+ uint32_t seg_no)
+{
+ struct dlist_entry *pkt_item;
+ struct rxd_pkt_meta *pkt;
+ struct ofi_ctrl_hdr *ctrl;
+ uint64_t curr_stamp = fi_gettime_us();
+
+ dlist_foreach(&tx_entry->pkt_list, pkt_item) {
+ pkt = container_of(pkt_item, struct rxd_pkt_meta, entry);
+ ctrl = (struct ofi_ctrl_hdr *)pkt->pkt_data;
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "check pkt %d, %p\n",
+ ctrl->seg_no, ctrl->msg_id);
+
+ if (ctrl->seg_no == seg_no) {
+
+ if (curr_stamp < pkt->us_stamp ||
+ (curr_stamp - pkt->us_stamp) <
+ (((uint64_t) 1) << ((uint64_t) pkt->retries + 1)) *
+ RXD_RETRY_TIMEOUT) {
+ break;
+ }
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "resending pkt %d, %p\n",
+ ctrl->seg_no, ctrl->msg_id);
+
+ pkt->us_stamp = fi_gettime_us();
+ rxd_ep_retry_pkt(ep, tx_entry, pkt);
+ break;
+ }
+ }
+}
+
+int rxd_tx_entry_progress(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry,
+ struct ofi_ctrl_hdr *ack)
+{
+ if (ack) {
+ tx_entry->rx_key = ack->rx_key;
+ tx_entry->win_sz += ack->seg_size;
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "tx: %p [%p] - avail_win: %d\n",
+ tx_entry, ack->msg_id, ack->seg_size);
+
+ if (ack->type == ofi_ctrl_nack) {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "got NACK for %d, %p\n",
+ ack->seg_no, ack->msg_id);
+ if (ack->seg_no > 0)
+ rxd_ep_free_acked_pkts(ep, tx_entry, ack->seg_no - 1);
+ rxd_resend_pkt(ep, tx_entry, ack->seg_no);
+ } else {
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "got ACK for %d, %p\n",
+ ack->seg_no, ack->msg_id);
+
+ rxd_ep_free_acked_pkts(ep, tx_entry, ack->seg_no);
+ if (ack->seg_size == 0 &&
+ tx_entry->done != tx_entry->op_hdr.size) {
+ tx_entry->is_waiting = 1;
+ tx_entry->retry_stamp = fi_gettime_us();
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL,
+ "Marking [%p] as waiting\n", tx_entry->msg_id);
+ }
+ }
+ }
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "tx: %p [%p] - num_unacked: %d\n",
+ tx_entry, tx_entry->msg_id, tx_entry->num_unacked);
+
+ while (tx_entry->win_sz && tx_entry->num_unacked < RXD_MAX_UNACKED &&
+ tx_entry->done != tx_entry->op_hdr.size) {
+ if (rxd_progress_tx(ep, tx_entry))
+ break;
+ }
+ return 0;
+}
+
+int rxd_ep_reply_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *in_ctrl,
+ uint8_t type, uint16_t seg_size, uint64_t rx_key,
+ uint64_t source, fi_addr_t dest)
+{
+ ssize_t ret;
+ struct rxd_pkt_meta *pkt_meta;
+ struct rxd_pkt_data *pkt;
+
+ pkt_meta = rxd_tx_pkt_acquire(ep);
+ if (!pkt_meta)
+ return -FI_ENOMEM;
+
+ pkt = (struct rxd_pkt_data *)pkt_meta->pkt_data;
+ rxd_init_ctrl_hdr(&pkt->ctrl, type, seg_size, in_ctrl->seg_no,
+ in_ctrl->msg_id, rx_key, source);
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "sending ack [%p] - %d, %d\n",
+ in_ctrl->msg_id, in_ctrl->seg_no, seg_size);
+
+ RXD_PKT_MARK_REMOTE_ACK(pkt_meta);
+ pkt_meta->us_stamp = fi_gettime_us();
+ ret = fi_send(ep->dg_ep, pkt, RXD_DATA_PKT_SZ,
+ rxd_mr_desc(pkt_meta->mr, ep),
+ dest, &pkt_meta->context);
+ if (ret)
+ goto err;
+ ep->num_out++;
+ return 0;
+err:
+ util_buf_release(ep->tx_pkt_pool, pkt_meta);
+ return ret;
+}
+
+int rxd_ep_reply_nack(struct rxd_ep *ep, struct ofi_ctrl_hdr *in_ctrl,
+ uint32_t seg_no, uint64_t rx_key,
+ uint64_t source, fi_addr_t dest)
+{
+ ssize_t ret;
+ struct rxd_pkt_meta *pkt_meta;
+ struct rxd_pkt_data *pkt;
+
+ pkt_meta = rxd_tx_pkt_acquire(ep);
+ if (!pkt_meta)
+ return -FI_ENOMEM;
+
+ pkt = (struct rxd_pkt_data *)pkt_meta->pkt_data;
+ rxd_init_ctrl_hdr(&pkt->ctrl, ofi_ctrl_nack, 0, seg_no,
+ in_ctrl->msg_id, rx_key, source);
+
+ RXD_PKT_MARK_REMOTE_ACK(pkt_meta);
+ pkt_meta->us_stamp = fi_gettime_us();
+ ret = fi_send(ep->dg_ep, pkt, RXD_DATA_PKT_SZ,
+ rxd_mr_desc(pkt_meta->mr, ep),
+ dest, &pkt_meta->context);
+ if (ret)
+ goto err;
+ ep->num_out++;
+ return 0;
+err:
+ util_buf_release(ep->tx_pkt_pool, pkt_meta);
+ return ret;
+}
+
+int rxd_ep_reply_discard(struct rxd_ep *ep, struct ofi_ctrl_hdr *in_ctrl,
+ uint32_t seg_no, uint64_t rx_key,
+ uint64_t source, fi_addr_t dest)
+{
+ ssize_t ret;
+ struct rxd_pkt_meta *pkt_meta;
+ struct rxd_pkt_data *pkt;
+
+ pkt_meta = rxd_tx_pkt_acquire(ep);
+ if (!pkt_meta)
+ return -FI_ENOMEM;
+
+ pkt = (struct rxd_pkt_data *)pkt_meta->pkt_data;
+ rxd_init_ctrl_hdr(&pkt->ctrl, ofi_ctrl_discard, 0, seg_no,
+ in_ctrl->msg_id, rx_key, source);
+
+ RXD_PKT_MARK_REMOTE_ACK(pkt_meta);
+ pkt_meta->us_stamp = fi_gettime_us();
+ ret = fi_send(ep->dg_ep, pkt, RXD_DATA_PKT_SZ,
+ rxd_mr_desc(pkt_meta->mr, ep),
+ dest, &pkt_meta->context);
+ if (ret)
+ goto err;
+ ep->num_out++;
+ return 0;
+err:
+ util_buf_release(ep->tx_pkt_pool, pkt_meta);
+ return ret;
+}
+
+#define RXD_TX_ENTRY_ID(ep, tx_entry) (tx_entry - &ep->tx_entry_fs->buf[0])
+
+void rxd_ep_init_start_hdr(struct rxd_ep *ep, struct rxd_peer *peer,
+ uint8_t op, struct rxd_tx_entry *tx_entry,
+ struct rxd_pkt_data_start *pkt, uint32_t flags,
+ uint64_t *msg_sz, uint64_t *data_sz)
+{
+ size_t curr_offset, i, iov_sz;
+ struct ofi_rma_iov rma_iov;
+
+ switch(op) {
+ case ofi_op_msg:
+ *msg_sz = rxd_get_msg_len(tx_entry->msg.msg_iov, tx_entry->msg.msg.iov_count);
+ *data_sz = MIN(RXD_MAX_STRT_DATA_PKT_SZ(ep), *msg_sz);
+ rxd_init_op_hdr(&pkt->op, tx_entry->msg.msg.data, *msg_sz, 0, op, 0, flags);
+ tx_entry->done = rxd_ep_copy_iov_buf(tx_entry->msg.msg_iov,
+ tx_entry->msg.msg.iov_count,
+ pkt->data, *data_sz, 0, RXD_COPY_IOV_TO_BUF);
+ break;
+
+ case ofi_op_tagged:
+ *msg_sz = rxd_get_msg_len(tx_entry->tmsg.msg_iov, tx_entry->tmsg.tmsg.iov_count);
+ *data_sz = MIN(RXD_MAX_STRT_DATA_PKT_SZ(ep), *msg_sz);
+ rxd_init_op_hdr(&pkt->op, tx_entry->tmsg.tmsg.data, *msg_sz, 0, op,
+ tx_entry->tmsg.tmsg.tag, flags);
+ tx_entry->done = rxd_ep_copy_iov_buf(tx_entry->tmsg.msg_iov,
+ tx_entry->tmsg.tmsg.iov_count,
+ pkt->data, *data_sz, 0, RXD_COPY_IOV_TO_BUF);
+ break;
+
+ case ofi_op_write:
+ *msg_sz = rxd_get_msg_len(tx_entry->write.msg.msg_iov, tx_entry->write.msg.iov_count);
+ iov_sz = sizeof(struct ofi_rma_iov) * tx_entry->write.msg.rma_iov_count;
+ *data_sz = MIN(RXD_MAX_STRT_DATA_PKT_SZ(ep), *msg_sz);
+ *data_sz -= iov_sz;
+
+ rxd_init_op_hdr(&pkt->op, tx_entry->write.msg.data, *msg_sz, 0, op, 0, flags);
+ pkt->op.iov_count = tx_entry->write.msg.rma_iov_count;
+
+ curr_offset = 0;
+ for (i = 0; i < pkt->op.iov_count; i++) {
+ rma_iov.addr = tx_entry->write.dst_iov[i].addr;
+ rma_iov.len = tx_entry->write.dst_iov[i].len;
+ rma_iov.key = tx_entry->write.dst_iov[i].key;
+ memcpy(pkt->data + curr_offset, &rma_iov, sizeof(struct ofi_rma_iov));
+ curr_offset += sizeof(struct ofi_rma_iov);
+ }
+ tx_entry->done = rxd_ep_copy_iov_buf(tx_entry->write.msg.msg_iov,
+ tx_entry->write.msg.iov_count,
+ pkt->data + curr_offset, *data_sz, 0,
+ RXD_COPY_IOV_TO_BUF);
+ *data_sz = tx_entry->done + iov_sz;
+ break;
+
+ case ofi_op_read_req:
+ *msg_sz = rxd_get_msg_len(tx_entry->read_req.msg.msg_iov,
+ tx_entry->read_req.msg.iov_count);
+
+ rxd_init_op_hdr(&pkt->op, tx_entry->read_req.msg.data, *msg_sz,
+ 0, op, 0, flags);
+ pkt->op.iov_count = tx_entry->read_req.msg.rma_iov_count;
+
+ tx_entry->done = 0;
+ curr_offset = 0;
+ *data_sz = 0;
+ for (i = 0; i < pkt->op.iov_count; i++) {
+ rma_iov.addr = tx_entry->read_req.src_iov[i].addr;
+ rma_iov.len = tx_entry->read_req.src_iov[i].len;
+ rma_iov.key = tx_entry->read_req.src_iov[i].key;
+ memcpy(pkt->data + curr_offset, &rma_iov, sizeof(struct ofi_rma_iov));
+ curr_offset += sizeof(struct ofi_rma_iov);
+ }
+ *data_sz += curr_offset;
+ assert(*data_sz <= RXD_MAX_STRT_DATA_PKT_SZ(ep));
+ break;
+
+ case ofi_op_read_rsp:
+ *msg_sz = rxd_get_msg_len(tx_entry->read_rsp.src_iov,
+ tx_entry->read_rsp.iov_count);
+ *data_sz = MIN(RXD_MAX_STRT_DATA_PKT_SZ(ep), *msg_sz);
+
+ rxd_init_op_hdr(&pkt->op, 0, *msg_sz, 0, op, 0, flags);
+ pkt->op.remote_idx = tx_entry->read_rsp.peer_msg_id;
+
+ tx_entry->done = rxd_ep_copy_iov_buf(tx_entry->read_rsp.src_iov,
+ tx_entry->read_rsp.iov_count,
+ pkt->data, *data_sz, 0,
+ RXD_COPY_IOV_TO_BUF);
+ break;
+
+ default:
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid op-type\n", op);
+ }
+}
+
+ssize_t rxd_ep_post_start_msg(struct rxd_ep *ep, struct rxd_peer *peer,
+ uint8_t op, struct rxd_tx_entry *tx_entry)
+{
+ ssize_t ret;
+ uint32_t flags;
+ uint64_t msg_sz;
+ uint64_t data_sz;
+ struct rxd_pkt_meta *pkt_meta;
+ struct rxd_pkt_data_start *pkt;
+
+ pkt_meta = rxd_tx_pkt_acquire(ep);
+ if (!pkt_meta)
+ return -FI_ENOMEM;
+
+ pkt = (struct rxd_pkt_data_start *)pkt_meta->pkt_data;
+ flags = rxd_prepare_tx_flags(tx_entry->flags);
+ tx_entry->msg_id = RXD_TX_ID(peer->nxt_msg_id, RXD_TX_ENTRY_ID(ep, tx_entry));
+
+ rxd_ep_init_start_hdr(ep, peer, op, tx_entry, pkt, flags, &msg_sz, &data_sz);
+ rxd_init_ctrl_hdr(&pkt->ctrl, ofi_ctrl_start_data, data_sz, 0,
+ tx_entry->msg_id, peer->conn_data, peer->conn_data);
+ tx_entry->nxt_seg_no = 1;
+ tx_entry->op_hdr = pkt->op;
+ tx_entry->win_sz = 0;
+
+ pkt_meta->tx_entry = tx_entry;
+ pkt_meta->type = (tx_entry->op_hdr.size == tx_entry->done) ?
+ RXD_PKT_LAST : RXD_PKT_DATA;
+
+ pkt_meta->us_stamp = fi_gettime_us();
+ ret = fi_send(ep->dg_ep, pkt, data_sz + RXD_START_DATA_PKT_SZ,
+ rxd_mr_desc(pkt_meta->mr, ep),
+ tx_entry->peer, &pkt_meta->context);
+ if (ret)
+ goto err;
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "sent start %p, %d\n",
+ pkt->ctrl.msg_id, pkt->ctrl.seg_no);
+ dlist_insert_tail(&pkt_meta->entry, &tx_entry->pkt_list);
+ peer->nxt_msg_id++;
+ ep->num_out++;
+ tx_entry->num_unacked++;
+ return 0;
+err:
+ util_buf_release(ep->tx_pkt_pool, pkt_meta);
+ return ret;
+}
+
+uint64_t rxd_find_av_index(struct rxd_av *av, uint64_t start_idx,
+ const void *addr, size_t addrlen, int *p_found)
+{
+ void *tmp_addr;
+ uint64_t idx = 0, count;
+ size_t tmp_addrlen;
+
+ if (p_found)
+ *p_found = 0;
+
+ tmp_addr = calloc(1, addrlen);
+ if (!tmp_addr)
+ return 0;
+
+ for (idx = start_idx, count = 0; count < av->dg_av_used;
+ count++, idx = (idx+1) % av->dg_av_used) {
+ tmp_addrlen = addrlen;
+ if (fi_av_lookup(av->dg_av, idx, tmp_addr, &tmp_addrlen)) {
+ idx = 0;
+ goto out;
+ }
+
+ if (addrlen == tmp_addrlen &&
+ memcmp(tmp_addr, addr, addrlen) == 0) {
+ if (p_found)
+ *p_found = 1;
+ goto out;
+ }
+ }
+out:
+ free(tmp_addr);
+ return idx;
+}
+
+ssize_t rxd_ep_post_conn_msg(struct rxd_ep *ep, struct rxd_peer *peer,
+ fi_addr_t addr)
+{
+ ssize_t ret;
+ size_t addrlen;
+ uint16_t data_sz;
+ struct rxd_pkt_data *pkt;
+ struct rxd_pkt_meta *pkt_meta;
+ struct rxd_tx_entry *tx_entry;
+
+ if (peer->conn_initiated)
+ return 0;
+
+ tx_entry = rxd_tx_entry_acquire(ep, peer);
+ if (!tx_entry)
+ return -FI_EAGAIN;
+
+ dlist_init(&tx_entry->pkt_list);
+ tx_entry->op_type = RXD_TX_CONN;
+ tx_entry->peer = addr;
+ tx_entry->win_sz = 0;
+
+ pkt_meta = rxd_tx_pkt_acquire(ep);
+ if (!pkt_meta) {
+ rxd_tx_entry_release(ep, tx_entry);
+ return -FI_ENOMEM;
+ }
+
+ pkt = (struct rxd_pkt_data *)pkt_meta->pkt_data;
+ addrlen = RXD_MAX_DATA_PKT_SZ(ep);
+ ret = fi_getname(&ep->dg_ep->fid, pkt->data, &addrlen);
+ assert(ret == 0);
+ data_sz = (uint16_t) addrlen;
+ tx_entry->msg_id = RXD_TX_ID(peer->nxt_msg_id, RXD_TX_ENTRY_ID(ep, tx_entry));
+
+ rxd_init_ctrl_hdr(&pkt->ctrl, ofi_ctrl_connreq, data_sz, 0,
+ tx_entry->msg_id, ep->conn_data, addr);
+
+ pkt_meta->us_stamp = fi_gettime_us();
+ ret = fi_send(ep->dg_ep, pkt, data_sz + RXD_DATA_PKT_SZ,
+ rxd_mr_desc(pkt_meta->mr, ep),
+ addr, &pkt_meta->context);
+ if (ret)
+ goto err;
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "sent conn %p\n", pkt->ctrl.msg_id);
+ dlist_insert_tail(&pkt_meta->entry, &tx_entry->pkt_list);
+ dlist_insert_tail(&tx_entry->entry, &ep->tx_entry_list);
+ peer->nxt_msg_id++;
+ ep->num_out++;
+ peer->conn_initiated = 1;
+ return 0;
+err:
+ rxd_tx_entry_release(ep, tx_entry);
+ util_buf_release(ep->tx_pkt_pool, pkt_meta);
+ return ret;
+}
+
+struct rxd_tx_entry *rxd_tx_entry_acquire_fast(struct rxd_ep *ep, struct rxd_peer *peer)
+{
+ struct rxd_tx_entry *tx_entry;
+ if (freestack_isempty(ep->tx_entry_fs)) {
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "no-more tx entries\n");
+ return NULL;
+ }
+
+ peer->num_msg_out++;
+ tx_entry = freestack_pop(ep->tx_entry_fs);
+ tx_entry->num_unacked = 0;
+ tx_entry->is_waiting = 0;
+ dlist_init(&tx_entry->entry);
+ return tx_entry;
+}
+
+struct rxd_tx_entry *rxd_tx_entry_acquire(struct rxd_ep *ep, struct rxd_peer *peer)
+{
+ return (peer->num_msg_out == RXD_MAX_OUT_TX_MSG) ? NULL :
+ rxd_tx_entry_acquire_fast(ep, peer);
+}
+
+void rxd_tx_entry_release(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry)
+{
+ struct rxd_peer *peer;
+
+ peer = rxd_ep_getpeer_info(ep, tx_entry->peer);
+ peer->num_msg_out--;
+ dlist_remove(&tx_entry->entry);
+ freestack_push(ep->tx_entry_fs, tx_entry);
+}
+
+void rxd_ep_copy_msg_iov(const struct iovec *src_iov,
+ struct iovec *dst_iov, size_t iov_count)
+{
+ size_t i;
+ for (i = 0; i < iov_count; i++)
+ dst_iov[i] = src_iov[i];
+}
+
+void rxd_ep_copy_rma_iov(const struct fi_rma_iov *src_iov,
+ struct fi_rma_iov *dst_iov, size_t iov_count)
+{
+ size_t i;
+ for (i = 0; i < iov_count; i++)
+ dst_iov[i] = src_iov[i];
+}
+
+static ssize_t rxd_ep_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
+ uint64_t flags)
+{
+ ssize_t ret;
+ uint64_t peer_addr;
+ struct rxd_ep *rxd_ep;
+ struct rxd_peer *peer;
+ struct rxd_tx_entry *tx_entry;
+ rxd_ep = container_of(ep, struct rxd_ep, ep);
+
+ peer_addr = rxd_av_get_dg_addr(rxd_ep->av, msg->addr);
+ peer = rxd_ep_getpeer_info(rxd_ep, peer_addr);
+
+ rxd_ep_lock_if_required(rxd_ep);
+ if (!peer->addr_published) {
+ ret = rxd_ep_post_conn_msg(rxd_ep, peer, peer_addr);
+ ret = (ret) ? ret : -FI_EAGAIN;
+ goto out;
+ }
+
+ tx_entry = rxd_tx_entry_acquire(rxd_ep, peer);
+ if (!tx_entry) {
+ ret = -FI_EAGAIN;
+ goto out;
+ }
+
+ dlist_init(&tx_entry->pkt_list);
+ tx_entry->op_type = RXD_TX_MSG;
+ tx_entry->msg.msg = *msg;
+ tx_entry->flags = flags;
+ tx_entry->peer = peer_addr;
+ rxd_ep_copy_msg_iov(msg->msg_iov, &tx_entry->msg.msg_iov[0], msg->iov_count);
+
+ ret = rxd_ep_post_start_msg(rxd_ep, peer, ofi_op_msg, tx_entry);
+ if (ret)
+ goto err;
+
+ dlist_insert_tail(&tx_entry->entry, &rxd_ep->tx_entry_list);
+out:
+ rxd_ep_unlock_if_required(rxd_ep);
+ return ret;
+err:
+ rxd_tx_entry_release(rxd_ep, tx_entry);
+ goto out;
+}
+
+static ssize_t rxd_ep_send(struct fid_ep *ep, const void *buf, size_t len, void *desc,
+ fi_addr_t dest_addr, void *context)
+{
+ struct fi_msg msg;
+ struct iovec msg_iov;
+ memset(&msg, 0, sizeof(msg));
+ msg_iov.iov_base = (void *) buf;
+ msg_iov.iov_len = len;
+ msg.msg_iov = &msg_iov;
+ msg.desc = &desc;
+ msg.iov_count = 1;
+ msg.addr = dest_addr;
+ msg.context = context;
+
+ return rxd_ep_sendmsg(ep, &msg, RXD_USE_OP_FLAGS);
+}
+
+static ssize_t rxd_ep_sendv(struct fid_ep *ep, const struct iovec *iov, void **desc,
+ size_t count, fi_addr_t dest_addr, void *context)
+{
+ struct fi_msg msg;
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_iov = iov;
+ msg.desc = desc;
+ msg.iov_count = count;
+ msg.addr = dest_addr;
+ msg.context = context;
+ return rxd_ep_sendmsg(ep, &msg, RXD_USE_OP_FLAGS);
+}
+
+static ssize_t rxd_ep_inject(struct fid_ep *ep, const void *buf, size_t len,
+ fi_addr_t dest_addr)
+{
+ struct fi_msg msg;
+ struct iovec msg_iov;
+
+ memset(&msg, 0, sizeof(msg));
+ msg_iov.iov_base = (void *) buf;
+ msg_iov.iov_len = len;
+ msg.msg_iov = &msg_iov;
+ msg.iov_count = 1;
+ msg.addr = dest_addr;
+
+ return rxd_ep_sendmsg(ep, &msg, FI_INJECT |
+ RXD_NO_COMPLETION | RXD_USE_OP_FLAGS);
+}
+
+static ssize_t rxd_ep_senddata(struct fid_ep *ep, const void *buf, size_t len, void *desc,
+ uint64_t data, fi_addr_t dest_addr, void *context)
+{
+ struct fi_msg msg;
+ struct iovec msg_iov;
+
+ msg_iov.iov_base = (void *) buf;
+ msg_iov.iov_len = len;
+
+ msg.msg_iov = &msg_iov;
+ msg.desc = desc;
+ msg.iov_count = 1;
+ msg.addr = dest_addr;
+ msg.context = context;
+ msg.data = data;
+
+ return rxd_ep_sendmsg(ep, &msg, FI_REMOTE_CQ_DATA | RXD_USE_OP_FLAGS);
+}
+
+static ssize_t rxd_ep_injectdata(struct fid_ep *ep, const void *buf, size_t len,
+ uint64_t data, fi_addr_t dest_addr)
+{
+ struct fi_msg msg;
+ struct iovec msg_iov;
+
+ memset(&msg, 0, sizeof(msg));
+ msg_iov.iov_base = (void *) buf;
+ msg_iov.iov_len = len;
+ msg.msg_iov = &msg_iov;
+
+ msg.iov_count = 1;
+ msg.addr = dest_addr;
+ msg.data = data;
+
+ return rxd_ep_sendmsg(ep, &msg, FI_REMOTE_CQ_DATA | FI_INJECT |
+ RXD_NO_COMPLETION | RXD_USE_OP_FLAGS);
+}
+
+static struct fi_ops_msg rxd_ops_msg = {
+ .size = sizeof(struct fi_ops_msg),
+ .recv = rxd_ep_recv,
+ .recvv = rxd_ep_recvv,
+ .recvmsg = rxd_ep_recvmsg,
+ .send = rxd_ep_send,
+ .sendv = rxd_ep_sendv,
+ .sendmsg = rxd_ep_sendmsg,
+ .inject = rxd_ep_inject,
+ .senddata = rxd_ep_senddata,
+ .injectdata = rxd_ep_injectdata,
+};
+
+static int rxd_peek_trecv(struct dlist_entry *item, const void *arg)
+{
+ const struct fi_msg_tagged *msg = (const struct fi_msg_tagged *) arg;
+ struct rxd_rx_entry *rx_entry;
+
+ rx_entry = container_of(item, struct rxd_rx_entry, unexp_entry);
+ return ((rx_entry->op_hdr.tag | msg->ignore) ==
+ (msg->tag | msg->ignore) &&
+ ((rx_entry->source == FI_ADDR_UNSPEC) ||
+ (msg->addr == FI_ADDR_UNSPEC) ||
+ (rx_entry->source == msg->addr)));
+}
+
+static void rxd_trx_discard_recv(struct rxd_ep *ep,
+ struct rxd_rx_entry *rx_entry)
+{
+ struct rxd_rx_buf *rx_buf;
+ struct ofi_ctrl_hdr *ctrl;
+ struct rxd_peer *peer;
+
+ rx_buf = rx_entry->unexp_buf;
+ ctrl = (struct ofi_ctrl_hdr *) rx_buf->buf;
+ peer = rxd_ep_getpeer_info(ep, ctrl->conn_id);
+
+ dlist_remove(&rx_entry->unexp_entry);
+ ep->num_unexp_msg--;
+
+ rxd_ep_reply_discard(ep, ctrl, 0, ctrl->rx_key, peer->conn_data, ctrl->conn_id);
+ rxd_rx_entry_release(ep, rx_entry);
+ rxd_ep_repost_buff(rx_buf);
+}
+
+static ssize_t rxd_trx_peek_recv(struct rxd_ep *ep,
+ const struct fi_msg_tagged *msg, uint64_t flags)
+{
+ struct dlist_entry *match;
+ struct rxd_rx_entry *rx_entry;
+ struct fi_cq_err_entry err_entry = {0};
+ struct fi_cq_tagged_entry cq_entry = {0};
+ struct fi_context *context;
+
+ match = dlist_find_first_match(&ep->unexp_tag_list,
+ &rxd_peek_trecv, msg);
+ if (!match) {
+ err_entry.op_context = msg->context;
+ err_entry.flags = (FI_MSG | FI_RECV | FI_TAGGED);
+ err_entry.tag = msg->tag;
+ err_entry.err = FI_ENOMSG;
+ err_entry.prov_errno = -FI_ENOMSG;
+ rxd_cq_report_error(ep->rx_cq, &err_entry);
+ return 0;
+ }
+
+ rx_entry = container_of(match, struct rxd_rx_entry, unexp_entry);
+ cq_entry.flags = (FI_MSG | FI_RECV | FI_TAGGED);
+ cq_entry.op_context = msg->context;
+ cq_entry.len = rx_entry->op_hdr.size;
+ cq_entry.buf = NULL;
+ cq_entry.data = rx_entry->op_hdr.data;
+ cq_entry.tag = rx_entry->op_hdr.tag;
+
+ if (flags & FI_CLAIM) {
+ context = (struct fi_context *)msg->context;
+ context->internal[0] = rx_entry;
+ dlist_remove(match);
+ } else if (flags & FI_DISCARD) {
+ rxd_trx_discard_recv(ep, rx_entry);
+ }
+
+ ep->rx_cq->write_fn(ep->rx_cq, &cq_entry);
+ return 0;
+}
+
+ssize_t rxd_trx_claim_recv(struct rxd_ep *ep, const struct fi_msg_tagged *msg,
+ uint64_t flags)
+{
+ int ret = 0, i;
+ struct fi_context *context;
+ struct rxd_rx_entry *rx_entry;
+ struct rxd_trecv_entry *trecv_entry;
+ struct rxd_peer *peer;
+ struct rxd_rx_buf *rx_buf;
+ struct ofi_ctrl_hdr *ctrl;
+ struct rxd_pkt_data_start *pkt_start;
+
+ rxd_ep_lock_if_required(ep);
+ if (freestack_isempty(ep->trecv_fs)) {
+ ret = -FI_EAGAIN;
+ goto out;
+ }
+
+ trecv_entry = freestack_pop(ep->trecv_fs);
+ trecv_entry->msg = *msg;
+ trecv_entry->msg.addr = (ep->caps & FI_DIRECTED_RECV) ?
+ msg->addr : FI_ADDR_UNSPEC;
+ trecv_entry->flags = flags;
+ for (i = 0; i < msg->iov_count; i++) {
+ trecv_entry->iov[i].iov_base = msg->msg_iov[i].iov_base;
+ trecv_entry->iov[i].iov_len = msg->msg_iov[i].iov_len;
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "post claim trecv: %u, tag: %p\n",
+ msg->msg_iov[i].iov_len, msg->tag);
+ }
+
+ context = (struct fi_context *) msg->context;
+ rx_entry = context->internal[0];
+ rx_entry->trecv = trecv_entry;
+
+ rx_buf = rx_entry->unexp_buf;
+ peer = rx_entry->peer_info;
+ ctrl = (struct ofi_ctrl_hdr *) rx_buf->buf;
+ pkt_start = (struct rxd_pkt_data_start *) ctrl;
+
+ rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->trecv->iov,
+ rx_entry->trecv->msg.iov_count, ctrl,
+ pkt_start->data, rx_buf);
+out:
+ rxd_ep_unlock_if_required(ep);
+ return ret;
+}
+
+ssize_t rxd_ep_trecvmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg,
+ uint64_t flags)
+{
+ ssize_t ret = 0, i;
+ struct rxd_ep *rxd_ep;
+ struct rxd_trecv_entry *trecv_entry;
+
+ rxd_ep = container_of(ep, struct rxd_ep, ep);
+ rxd_ep_lock_if_required(rxd_ep);
+
+ if (flags & FI_PEEK) {
+ ret = rxd_trx_peek_recv(rxd_ep, msg, flags);
+ goto out;
+ } else if (flags & FI_CLAIM) {
+ ret = rxd_trx_claim_recv(rxd_ep, msg, flags);
+ goto out;
+ }
+
+ if (freestack_isempty(rxd_ep->trecv_fs)) {
+ ret = -FI_EAGAIN;
+ goto out;
+ }
+
+ trecv_entry = freestack_pop(rxd_ep->trecv_fs);
+ trecv_entry->msg = *msg;
+ trecv_entry->msg.addr = (rxd_ep->caps & FI_DIRECTED_RECV) ?
+ msg->addr : FI_ADDR_UNSPEC;
+ trecv_entry->flags = flags;
+ for (i = 0; i < msg->iov_count; i++) {
+ trecv_entry->iov[i].iov_base = msg->msg_iov[i].iov_base;
+ trecv_entry->iov[i].iov_len = msg->msg_iov[i].iov_len;
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "post trecv: %u, tag: %p\n",
+ msg->msg_iov[i].iov_len, msg->tag);
+ }
+ dlist_init(&trecv_entry->entry);
+ dlist_insert_tail(&trecv_entry->entry, &rxd_ep->trecv_list);
+
+ if (!dlist_empty(&rxd_ep->unexp_tag_list)) {
+ rxd_ep_check_unexp_tag_list(rxd_ep, trecv_entry);
+ }
+out:
+ rxd_ep_unlock_if_required(rxd_ep);
+ return ret;
+}
+
+static ssize_t rxd_ep_trecv(struct fid_ep *ep, void *buf, size_t len, void *desc,
+ fi_addr_t src_addr,
+ uint64_t tag, uint64_t ignore, void *context)
+{
+ struct fi_msg_tagged msg;
+ struct iovec msg_iov;
+
+ memset(&msg, 0, sizeof(msg));
+ msg_iov.iov_base = buf;
+ msg_iov.iov_len = len;
+
+ msg.msg_iov = &msg_iov;
+ msg.desc = &desc;
+ msg.iov_count = 1;
+ msg.addr = src_addr;
+ msg.context = context;
+ msg.tag = tag;
+ msg.ignore = ignore;
+ msg.data = 0;
+ return rxd_ep_trecvmsg(ep, &msg, RXD_USE_OP_FLAGS);
+}
+
+ssize_t rxd_ep_trecvv(struct fid_ep *ep, const struct iovec *iov, void **desc,
+ size_t count, fi_addr_t src_addr,
+ uint64_t tag, uint64_t ignore, void *context)
+{
+ struct fi_msg_tagged msg;
+
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_iov = iov;
+ msg.desc = desc;
+ msg.iov_count = count;
+ msg.addr = src_addr;
+ msg.context = context;
+ msg.tag = tag;
+ msg.ignore = ignore;
+ msg.data = 0;
+ return rxd_ep_trecvmsg(ep, &msg, RXD_USE_OP_FLAGS);
+}
+
+ssize_t rxd_ep_tsendmsg(struct fid_ep *ep, const struct fi_msg_tagged *msg,
+ uint64_t flags)
+{
+ ssize_t ret;
+ uint64_t peer_addr;
+ struct rxd_ep *rxd_ep;
+ struct rxd_peer *peer;
+ struct rxd_tx_entry *tx_entry;
+ rxd_ep = container_of(ep, struct rxd_ep, ep);
+
+ peer_addr = rxd_av_get_dg_addr(rxd_ep->av, msg->addr);
+ peer = rxd_ep_getpeer_info(rxd_ep, peer_addr);
+
+ rxd_ep_lock_if_required(rxd_ep);
+ if (!peer->addr_published) {
+ ret = rxd_ep_post_conn_msg(rxd_ep, peer, peer_addr);
+ ret = (ret) ? ret : -FI_EAGAIN;
+ goto out;
+ }
+
+ tx_entry = rxd_tx_entry_acquire(rxd_ep, peer);
+ if (!tx_entry) {
+ ret = -FI_EAGAIN;
+ goto out;
+ }
+
+ dlist_init(&tx_entry->pkt_list);
+ tx_entry->op_type = RXD_TX_TAG;
+ tx_entry->tmsg.tmsg = *msg;
+ tx_entry->flags = flags;
+ tx_entry->peer = peer_addr;
+ rxd_ep_copy_msg_iov(msg->msg_iov, &tx_entry->tmsg.msg_iov[0], msg->iov_count);
+
+ ret = rxd_ep_post_start_msg(rxd_ep, peer, ofi_op_tagged, tx_entry);
+ if (ret)
+ goto err;
+
+ dlist_insert_tail(&tx_entry->entry, &rxd_ep->tx_entry_list);
+out:
+ rxd_ep_unlock_if_required(rxd_ep);
+ return ret;
+err:
+ rxd_tx_entry_release(rxd_ep, tx_entry);
+ goto out;
+}
+
+ssize_t rxd_ep_tsend(struct fid_ep *ep, const void *buf, size_t len, void *desc,
+ fi_addr_t dest_addr, uint64_t tag, void *context)
+{
+ struct fi_msg_tagged msg;
+ struct iovec msg_iov;
+
+ memset(&msg, 0, sizeof(msg));
+ msg_iov.iov_base = (void *) buf;
+ msg_iov.iov_len = len;
+ msg.msg_iov = &msg_iov;
+ msg.desc = &desc;
+ msg.iov_count = 1;
+ msg.addr = dest_addr;
+ msg.context = context;
+ msg.tag = tag;
+
+ return rxd_ep_tsendmsg(ep, &msg, RXD_USE_OP_FLAGS);
+}
+
+ssize_t rxd_ep_tsendv(struct fid_ep *ep, const struct iovec *iov, void **desc,
+ size_t count, fi_addr_t dest_addr, uint64_t tag, void *context)
+{
+ struct fi_msg_tagged msg;
+
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_iov = iov;
+ msg.desc = desc;
+ msg.iov_count = count;
+ msg.addr = dest_addr;
+ msg.context = context;
+ msg.tag = tag;
+ return rxd_ep_tsendmsg(ep, &msg, RXD_USE_OP_FLAGS);
+}
+
+ssize_t rxd_ep_tinject(struct fid_ep *ep, const void *buf, size_t len,
+ fi_addr_t dest_addr, uint64_t tag)
+{
+ struct fi_msg_tagged msg;
+ struct iovec msg_iov;
+
+ memset(&msg, 0, sizeof(msg));
+ msg_iov.iov_base = (void *) buf;
+ msg_iov.iov_len = len;
+ msg.msg_iov = &msg_iov;
+ msg.iov_count = 1;
+ msg.addr = dest_addr;
+ msg.tag = tag;
+ return rxd_ep_tsendmsg(ep, &msg, FI_INJECT |
+ RXD_NO_COMPLETION | RXD_USE_OP_FLAGS);
+}
+
+ssize_t rxd_ep_tsenddata(struct fid_ep *ep, const void *buf, size_t len, void *desc,
+ uint64_t data, fi_addr_t dest_addr, uint64_t tag, void *context)
+{
+ struct fi_msg_tagged msg;
+ struct iovec msg_iov;
+
+ memset(&msg, 0, sizeof(msg));
+ msg_iov.iov_base = (void *) buf;
+ msg_iov.iov_len = len;
+ msg.msg_iov = &msg_iov;
+ msg.desc = desc;
+ msg.iov_count = 1;
+ msg.addr = dest_addr;
+ msg.context = context;
+ msg.data = data;
+ msg.tag = tag;
+
+ return rxd_ep_tsendmsg(ep, &msg, FI_REMOTE_CQ_DATA | RXD_USE_OP_FLAGS);
+}
+
+ssize_t rxd_ep_tinjectdata(struct fid_ep *ep, const void *buf, size_t len,
+ uint64_t data, fi_addr_t dest_addr, uint64_t tag)
+{
+ struct fi_msg_tagged msg;
+ struct iovec msg_iov;
+
+ memset(&msg, 0, sizeof(msg));
+ msg_iov.iov_base = (void *) buf;
+ msg_iov.iov_len = len;
+ msg.msg_iov = &msg_iov;
+
+ msg.iov_count = 1;
+ msg.addr = dest_addr;
+ msg.data = data;
+ msg.tag = tag;
+
+ return rxd_ep_tsendmsg(ep, &msg, FI_REMOTE_CQ_DATA | FI_INJECT |
+ RXD_NO_COMPLETION | RXD_USE_OP_FLAGS);
+}
+
+struct fi_ops_tagged rxd_ops_tagged = {
+ .size = sizeof(struct fi_ops_tagged),
+ .recv = rxd_ep_trecv,
+ .recvv = rxd_ep_trecvv,
+ .recvmsg = rxd_ep_trecvmsg,
+ .send = rxd_ep_tsend,
+ .sendv = rxd_ep_tsendv,
+ .sendmsg = rxd_ep_tsendmsg,
+ .inject = rxd_ep_tinject,
+ .senddata = rxd_ep_tsenddata,
+ .injectdata = rxd_ep_tinjectdata,
+};
+
+static void rxd_ep_free_buf_pools(struct rxd_ep *ep)
+{
+ util_buf_pool_destroy(ep->tx_pkt_pool);
+ util_buf_pool_destroy(ep->rx_pkt_pool);
+
+ if (ep->tx_entry_fs)
+ rxd_tx_entry_fs_free(ep->tx_entry_fs);
+
+ if (ep->rx_entry_fs)
+ rxd_rx_entry_fs_free(ep->rx_entry_fs);
+
+ if (ep->recv_fs)
+ rxd_recv_fs_free(ep->recv_fs);
+
+ if (ep->trecv_fs)
+ rxd_trecv_fs_free(ep->trecv_fs);
+}
+
+static int rxd_ep_close(struct fid *fid)
+{
+ int ret;
+ struct rxd_ep *ep;
+ struct slist_entry *entry;
+ struct rxd_rx_buf *buf;
+
+ ep = container_of(fid, struct rxd_ep, ep.fid);
+ while (ep->num_out) {
+ rxd_cq_progress(&ep->tx_cq->util_cq);
+ }
+
+ ret = fi_close(&ep->dg_ep->fid);
+ if (ret)
+ return ret;
+
+ fastlock_acquire(&ep->domain->lock);
+ dlist_remove(&ep->dom_entry);
+ fastlock_release(&ep->domain->lock);
+
+ while(!slist_empty(&ep->rx_pkt_list)) {
+ entry = slist_remove_head(&ep->rx_pkt_list);
+ buf = container_of(entry, struct rxd_rx_buf, entry);
+ util_buf_release(ep->rx_pkt_pool, buf);
+ }
+
+ if (ep->tx_cq)
+ atomic_dec(&ep->tx_cq->util_cq.ref);
+
+ if (ep->rx_cq)
+ atomic_dec(&ep->rx_cq->util_cq.ref);
+
+ atomic_dec(&ep->domain->util_domain.ref);
+ fastlock_destroy(&ep->lock);
+ rxd_ep_free_buf_pools(ep);
+ free(ep->peer_info);
+ free(ep->name);
+ free(ep);
+ return 0;
+}
+
+static int rxd_ep_bind_cq(struct rxd_ep *ep, struct rxd_cq *cq, uint64_t flags)
+{
+ int ret;
+
+ if (flags & ~(FI_TRANSMIT | FI_RECV)) {
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "unsupported flags\n");
+ return -FI_EBADFLAGS;
+ }
+
+ if (((flags & FI_TRANSMIT) && ep->tx_cq) ||
+ ((flags & FI_RECV) && ep->rx_cq)) {
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "duplicate CQ binding\n");
+ return -FI_EINVAL;
+ }
+
+ if (flags & FI_TRANSMIT) {
+ if (!ep->tx_cq && !ep->rx_cq) {
+ ret = fi_ep_bind(ep->dg_ep, &cq->dg_cq->fid, FI_TRANSMIT | FI_RECV);
+ if (ret)
+ return ret;
+ }
+
+ ep->tx_cq = cq;
+ atomic_inc(&cq->util_cq.ref);
+ }
+
+ if (flags & FI_RECV) {
+ if (!ep->tx_cq && !ep->rx_cq) {
+ ret = fi_ep_bind(ep->dg_ep, &cq->dg_cq->fid, FI_TRANSMIT | FI_RECV);
+ if (ret)
+ return ret;
+ }
+
+ ep->rx_cq = cq;
+ atomic_inc(&cq->util_cq.ref);
+ }
+ return 0;
+}
+
+static int rxd_ep_bind(struct fid *ep_fid, struct fid *bfid, uint64_t flags)
+{
+ struct rxd_ep *ep;
+ struct rxd_av *av;
+ int ret = 0;
+
+ ep = container_of(ep_fid, struct rxd_ep, ep.fid);
+ switch (bfid->fclass) {
+ case FI_CLASS_AV:
+ if (ep->av) {
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL,
+ "duplicate AV binding\n");
+ return -FI_EINVAL;
+ }
+ av = container_of(bfid, struct rxd_av, util_av.av_fid.fid);
+ ret = fi_ep_bind(ep->dg_ep, &av->dg_av->fid, flags);
+ if (ret)
+ return ret;
+
+ /* todo: handle case where AV is updated after binding */
+ ep->peer_info = calloc(av->size, sizeof(struct rxd_peer));
+ ep->max_peers = av->size;
+ if (!ep->peer_info) {
+ return -FI_ENOMEM;
+ }
+
+ ep->av = av;
+ break;
+ case FI_CLASS_CQ:
+ ret = rxd_ep_bind_cq(ep, container_of(bfid, struct rxd_cq,
+ util_cq.cq_fid.fid), flags);
+ break;
+ case FI_CLASS_EQ:
+ break;
+ default:
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL,
+ "invalid fid class\n");
+ ret = -FI_EINVAL;
+ break;
+ }
+ return ret;
+}
+
+static int rxd_ep_control(struct fid *fid, int command, void *arg)
+{
+ int ret;
+ struct rxd_ep *ep;
+
+ switch (command) {
+ case FI_ENABLE:
+ ep = container_of(fid, struct rxd_ep, ep.fid);
+ ret = rxd_ep_enable(ep);
+ break;
+ default:
+ ret = -FI_ENOSYS;
+ break;
+ }
+ return ret;
+}
+
+static struct fi_ops rxd_ep_fi_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = rxd_ep_close,
+ .bind = rxd_ep_bind,
+ .control = rxd_ep_control,
+ .ops_open = fi_no_ops_open,
+};
+
+static int rxd_ep_cm_setname(fid_t fid, void *addr, size_t addrlen)
+{
+ struct rxd_ep *ep;
+ ep = container_of(fid, struct rxd_ep, ep.fid);
+ return fi_setname(&ep->dg_ep->fid, addr, addrlen);
+}
+
+static int rxd_ep_cm_getname(fid_t fid, void *addr, size_t *addrlen)
+{
+ struct rxd_ep *ep;
+ ep = container_of(fid, struct rxd_ep, ep.fid);
+ return fi_getname(&ep->dg_ep->fid, addr, addrlen);
+}
+
+struct fi_ops_cm rxd_ep_cm = {
+ .size = sizeof(struct fi_ops_cm),
+ .setname = rxd_ep_cm_setname,
+ .getname = rxd_ep_cm_getname,
+ .getpeer = fi_no_getpeer,
+ .connect = fi_no_connect,
+ .listen = fi_no_listen,
+ .accept = fi_no_accept,
+ .reject = fi_no_reject,
+ .shutdown = fi_no_shutdown,
+};
+
+static int rxd_buf_region_alloc_hndlr(void *pool_ctx, void *addr, size_t len,
+ void **context)
+{
+ int ret;
+ struct fid_mr *mr;
+ struct rxd_domain *domain = (struct rxd_domain *)pool_ctx;
+
+ ret = fi_mr_reg(domain->dg_domain, addr, len,
+ FI_SEND | FI_RECV, 0, 0, 0, &mr, NULL);
+ *context = mr;
+ return ret;
+}
+
+static void rxd_buf_region_free_hndlr(void *pool_ctx, void *context)
+{
+ fi_close((struct fid *) context);
+}
+
+int rxd_ep_create_buf_pools(struct rxd_ep *ep, struct fi_info *fi_info)
+{
+ ep->tx_pkt_pool = util_buf_pool_create_ex(
+ ep->domain->max_mtu_sz + sizeof(struct rxd_pkt_meta),
+ RXD_BUF_POOL_ALIGNMENT, 0, RXD_TX_POOL_CHUNK_CNT,
+ (fi_info->mode & FI_LOCAL_MR) ? rxd_buf_region_alloc_hndlr : NULL,
+ (fi_info->mode & FI_LOCAL_MR) ? rxd_buf_region_free_hndlr : NULL,
+ ep->domain);
+ if (!ep->tx_pkt_pool)
+ return -FI_ENOMEM;
+
+ ep->rx_pkt_pool = util_buf_pool_create_ex(
+ ep->domain->max_mtu_sz + sizeof (struct rxd_rx_buf),
+ RXD_BUF_POOL_ALIGNMENT, 0, RXD_RX_POOL_CHUNK_CNT,
+ (fi_info->mode & FI_LOCAL_MR) ? rxd_buf_region_alloc_hndlr : NULL,
+ (fi_info->mode & FI_LOCAL_MR) ? rxd_buf_region_free_hndlr : NULL,
+ ep->domain);
+ if (!ep->rx_pkt_pool)
+ goto err;
+
+ ep->tx_entry_fs = rxd_tx_entry_fs_create(1ULL << RXD_MAX_TX_BITS);
+ if (!ep->tx_entry_fs)
+ goto err;
+
+ ep->rx_entry_fs = rxd_rx_entry_fs_create(1ULL << RXD_MAX_RX_BITS);
+ if (!ep->rx_entry_fs)
+ goto err;
+
+ if (ep->caps & FI_MSG) {
+ ep->recv_fs = rxd_recv_fs_create(ep->rx_size);
+ dlist_init(&ep->recv_list);
+ if (!ep->recv_fs)
+ goto err;
+ }
+
+ if (ep->caps & FI_TAGGED) {
+ ep->trecv_fs = rxd_trecv_fs_create(ep->rx_size);
+ dlist_init(&ep->trecv_list);
+ if (!ep->trecv_fs)
+ goto err;
+ }
+
+ return 0;
+err:
+ if (ep->tx_pkt_pool)
+ util_buf_pool_destroy(ep->tx_pkt_pool);
+
+ if (ep->rx_pkt_pool)
+ util_buf_pool_destroy(ep->rx_pkt_pool);
+
+ if (ep->tx_entry_fs)
+ rxd_tx_entry_fs_free(ep->tx_entry_fs);
+
+ if (ep->rx_entry_fs)
+ rxd_rx_entry_fs_free(ep->rx_entry_fs);
+
+ if (ep->recv_fs)
+ rxd_recv_fs_free(ep->recv_fs);
+
+ if (ep->trecv_fs)
+ rxd_trecv_fs_free(ep->trecv_fs);
+
+ return -FI_ENOMEM;
+}
+
+int rxd_endpoint(struct fid_domain *domain, struct fi_info *info,
+ struct fid_ep **ep, void *context)
+{
+ int ret;
+ struct fi_info *dg_info;
+ struct rxd_ep *rxd_ep;
+ struct rxd_domain *rxd_domain;
+
+ ret = fi_check_info(&rxd_util_prov, info, FI_MATCH_PREFIX);
+ if (ret)
+ return ret;
+
+ ret = ofix_getinfo(rxd_prov.version, NULL, NULL, 0, &rxd_util_prov,
+ info, rxd_alter_layer_info,
+ rxd_alter_base_info, 1, &dg_info);
+ if (ret)
+ return ret;
+
+ rxd_domain = container_of(domain, struct rxd_domain, util_domain.domain_fid);
+ rxd_ep = calloc(1, sizeof(*rxd_ep));
+ if (!rxd_ep) {
+ ret = -FI_ENOMEM;
+ goto err1;
+ }
+
+ rxd_ep->addrlen = (info->src_addr) ? info->src_addrlen : info->dest_addrlen;
+ rxd_ep->do_local_mr = (rxd_domain->dg_mode & FI_LOCAL_MR) ? 1 : 0;
+ ret = fi_endpoint(rxd_domain->dg_domain, dg_info, ep, context);
+ if (ret)
+ goto err2;
+
+ rxd_ep->caps = info->caps;
+ rxd_ep->domain = rxd_domain;
+ rxd_ep->rx_size = info->rx_attr->size;
+ ret = rxd_ep_create_buf_pools(rxd_ep, info);
+ if (ret)
+ goto err3;
+
+ rxd_ep->dg_ep = *ep;
+ rxd_ep->ep.fid.ops = &rxd_ep_fi_ops;
+ rxd_ep->ep.cm = &rxd_ep_cm;
+ rxd_ep->ep.ops = &rxd_ops_ep;
+ rxd_ep->ep.msg = &rxd_ops_msg;
+ rxd_ep->ep.tagged = &rxd_ops_tagged;
+ rxd_ep->ep.rma = &rxd_ops_rma;
+
+ dlist_init(&rxd_ep->tx_entry_list);
+ dlist_init(&rxd_ep->rx_entry_list);
+ dlist_init(&rxd_ep->wait_rx_list);
+ dlist_init(&rxd_ep->unexp_msg_list);
+ dlist_init(&rxd_ep->unexp_tag_list);
+ slist_init(&rxd_ep->rx_pkt_list);
+ fastlock_init(&rxd_ep->lock);
+
+ dlist_init(&rxd_ep->dom_entry);
+ atomic_inc(&rxd_ep->domain->util_domain.ref);
+
+ *ep = &rxd_ep->ep;
+ fi_freeinfo(dg_info);
+ return 0;
+
+err3:
+ fi_close(&(*ep)->fid);
+err2:
+ free(rxd_ep);
+err1:
+ fi_freeinfo(dg_info);
+ return ret;
+}
+
+int rxd_ep_retry_pkt(struct rxd_ep *ep, struct rxd_tx_entry *tx_entry,
+ struct rxd_pkt_meta *pkt)
+{
+ int ret;
+ struct ofi_ctrl_hdr *ctrl;
+
+ ctrl = (struct ofi_ctrl_hdr *)pkt->pkt_data;
+ if (pkt->retries > RXD_MAX_PKT_RETRY) {
+ /* todo: report error */
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "Pkt delivery failed\n", ctrl->seg_no);
+ return -FI_EIO;
+ }
+
+ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "retry packet : %2d, size: %d, tx_id :%p\n",
+ ctrl->seg_no, ctrl->type == ofi_ctrl_start_data ?
+ ctrl->seg_size + RXD_START_DATA_PKT_SZ :
+ ctrl->seg_size + RXD_DATA_PKT_SZ,
+ ctrl->msg_id);
+
+ ret = fi_send(ep->dg_ep, ctrl,
+ ctrl->type == ofi_ctrl_start_data ?
+ ctrl->seg_size + RXD_START_DATA_PKT_SZ :
+ ctrl->seg_size + RXD_DATA_PKT_SZ,
+ rxd_mr_desc(pkt->mr, ep),
+ tx_entry->peer, &pkt->context);
+
+ if (ret != -FI_EAGAIN)
+ pkt->retries++;
+
+ if (ret && ret != -FI_EAGAIN) {
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "Pkt sent failed seg: %d, ret: %d\n",
+ ctrl->seg_no, ret);
+ }
+
+ return ret;
+}
+
+void rxd_ep_progress(struct rxd_ep *ep)
+{
+ struct dlist_entry *tx_item, *pkt_item;
+ struct rxd_tx_entry *tx_entry;
+ struct rxd_pkt_meta *pkt;
+ uint64_t curr_stamp;
+
+ rxd_ep_lock_if_required(ep);
+ curr_stamp = fi_gettime_us();
+ dlist_foreach(&ep->tx_entry_list, tx_item) {
+
+ tx_entry = container_of(tx_item, struct rxd_tx_entry, entry);
+
+ if (tx_entry->win_sz)
+ rxd_tx_entry_progress(ep, tx_entry, NULL);
+
+ else if (tx_entry->is_waiting &&
+ (curr_stamp - tx_entry->retry_stamp > RXD_WAIT_TIMEOUT) &&
+ dlist_empty(&tx_entry->pkt_list)) {
+ tx_entry->win_sz = 1;
+ tx_entry->is_waiting = 0;
+
+ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "Progressing waiting entry [%p]\n",
+ tx_entry->msg_id);
+
+ rxd_tx_entry_progress(ep, tx_entry, NULL);
+ tx_entry->retry_stamp = fi_gettime_us();
+ }
+
+ dlist_foreach(&tx_entry->pkt_list, pkt_item) {
+ pkt = container_of(pkt_item, struct rxd_pkt_meta, entry);
+ /* TODO: This if check is repeated. Create a function
+ * to perform this check, and figure out what the check
+ * is actually doing with the bit-shift, multiply operation.
+ */
+ if (curr_stamp > pkt->us_stamp &&
+ curr_stamp - pkt->us_stamp >
+ (((uint64_t) 1) << ((uint64_t) pkt->retries + 1)) *
+ RXD_RETRY_TIMEOUT) {
+ pkt->us_stamp = curr_stamp;
+ rxd_ep_retry_pkt(ep, tx_entry, pkt);
+ }
+ }
+ }
+ rxd_ep_unlock_if_required(ep);
+}
diff --git a/prov/udp/src/udpx_fabric.c b/prov/rxd/src/rxd_fabric.c
similarity index 52%
copy from prov/udp/src/udpx_fabric.c
copy to prov/rxd/src/rxd_fabric.c
index 7da684b..0393ca6 100644
--- a/prov/udp/src/udpx_fabric.c
+++ b/prov/rxd/src/rxd_fabric.c
@@ -33,55 +33,93 @@
#include <stdlib.h>
#include <string.h>
-#include "udpx.h"
+#include "rxd.h"
-
-static struct fi_ops_fabric udpx_fabric_ops = {
+static struct fi_ops_fabric rxd_fabric_ops = {
.size = sizeof(struct fi_ops_fabric),
- .domain = udpx_domain_open,
+ .domain = &rxd_domain_open,
.passive_ep = fi_no_passive_ep,
- .eq_open = fi_eq_create,
- .wait_open = fi_wait_fd_open,
- .trywait = util_trywait
+ .eq_open = ofi_eq_create,
+ .wait_open = ofi_wait_fd_open,
+ .trywait = ofi_trywait
};
-static int udpx_fabric_close(fid_t fid)
+static int rxd_fabric_close(fid_t fid)
{
int ret;
- struct util_fabric *fabric;
- fabric = container_of(fid, struct util_fabric, fabric_fid.fid);
- ret = util_fabric_close(fabric);
+ struct rxd_fabric *rxd_fabric;
+
+ rxd_fabric = container_of(fid, struct rxd_fabric, util_fabric.fabric_fid.fid);
+ ret = fi_close(&rxd_fabric->dg_fabric->fid);
+ if (ret)
+ return ret;
+
+ ret = ofi_fabric_close(&rxd_fabric->util_fabric);
if (ret)
return ret;
- free(fabric);
+
+ free(rxd_fabric);
return 0;
}
-static struct fi_ops udpx_fabric_fi_ops = {
+static struct fi_ops rxd_fabric_fi_ops = {
.size = sizeof(struct fi_ops),
- .close = udpx_fabric_close,
+ .close = rxd_fabric_close,
.bind = fi_no_bind,
.control = fi_no_control,
.ops_open = fi_no_ops_open,
};
-int udpx_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric,
+int rxd_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric,
void *context)
{
+ struct rxd_fabric *rxd_fabric;
+ struct fi_info hints, *dg_info;
int ret;
- struct util_fabric *util_fabric;
- util_fabric = calloc(1, sizeof(*util_fabric));
- if (!util_fabric)
+ rxd_fabric = calloc(1, sizeof(*rxd_fabric));
+ if (!rxd_fabric)
return -FI_ENOMEM;
- ret = fi_fabric_init(&udpx_prov, udpx_info.fabric_attr, attr,
- util_fabric, context);
+ ret = ofi_fabric_init(&rxd_prov, &rxd_fabric_attr, attr,
+ &rxd_fabric->util_fabric, context, FI_MATCH_PREFIX);
if (ret)
- return ret;
+ goto err1;
+
+ memset(&hints, 0, sizeof hints);
+ if (!(hints.fabric_attr = calloc(1, sizeof(*hints.fabric_attr)))) {
+ ret = -FI_ENOMEM;
+ goto err2;
+ }
+ hints.fabric_attr->name = attr->name;
+
+ ret = ofix_getinfo(rxd_prov.version, NULL, NULL, 0, &rxd_util_prov,
+ &hints, rxd_alter_layer_info,
+ rxd_alter_base_info, 1, &dg_info);
+ if (ret) {
+ ret = -FI_EINVAL;
+ goto err3;
+ }
+
+ ret = fi_fabric(dg_info->fabric_attr, &rxd_fabric->dg_fabric, context);
+ if (ret) {
+ goto err4;
+ }
+
+ *fabric = &rxd_fabric->util_fabric.fabric_fid;
+ (*fabric)->fid.ops = &rxd_fabric_fi_ops;
+ (*fabric)->ops = &rxd_fabric_ops;
- *fabric = &util_fabric->fabric_fid;
- (*fabric)->fid.ops = &udpx_fabric_fi_ops;
- (*fabric)->ops = &udpx_fabric_ops;
+ free(hints.fabric_attr);
+ fi_freeinfo(dg_info);
return 0;
+err4:
+ fi_freeinfo(dg_info);
+err3:
+ free(hints.fabric_attr);
+err2:
+ ofi_fabric_close(&rxd_fabric->util_fabric);
+err1:
+ free(rxd_fabric);
+ return ret;
}
diff --git a/prov/udp/src/udpx_init.c b/prov/rxd/src/rxd_init.c
similarity index 60%
copy from prov/udp/src/udpx_init.c
copy to prov/rxd/src/rxd_init.c
index 8ee0e46..4526b8b 100644
--- a/prov/udp/src/udpx_init.c
+++ b/prov/rxd/src/rxd_init.c
@@ -33,36 +33,50 @@
#include <rdma/fi_errno.h>
#include <prov.h>
-#include "udpx.h"
+#include "rxd.h"
+int rxd_alter_layer_info(struct fi_info *layer_info, struct fi_info *base_info)
+{
+ base_info->caps = FI_MSG;
+ base_info->mode = FI_LOCAL_MR;
+ base_info->ep_attr->type = FI_EP_DGRAM;
+ return 0;
+}
-int udpx_check_info(struct fi_info *info)
+int rxd_alter_base_info(struct fi_info *base_info, struct fi_info *layer_info)
{
- return fi_check_info(&udpx_prov, &udpx_info, info);
+ layer_info->caps = rxd_info.caps;
+ layer_info->mode = rxd_info.mode;
+
+ *layer_info->tx_attr = *rxd_info.tx_attr;
+ *layer_info->rx_attr = *rxd_info.rx_attr;
+ *layer_info->ep_attr = *rxd_info.ep_attr;
+ *layer_info->domain_attr = *rxd_info.domain_attr;
+ return 0;
}
-static int udpx_getinfo(uint32_t version, const char *node, const char *service,
+static int rxd_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, struct fi_info *hints, struct fi_info **info)
{
- return util_getinfo(&udpx_prov, version, node, service, flags,
- &udpx_info, hints, info);
+ return ofix_getinfo(version, node, service, flags, &rxd_util_prov,
+ hints, rxd_alter_layer_info, rxd_alter_base_info, 0, info);
}
-static void udpx_fini(void)
+static void rxd_fini(void)
{
/* yawn */
}
-struct fi_provider udpx_prov = {
- .name = "UDP",
- .version = FI_VERSION(UDPX_MAJOR_VERSION, UDPX_MINOR_VERSION),
- .fi_version = FI_VERSION(1, 3),
- .getinfo = udpx_getinfo,
- .fabric = udpx_fabric,
- .cleanup = udpx_fini
+struct fi_provider rxd_prov = {
+ .name = "rxd",
+ .version = FI_VERSION(RXD_MAJOR_VERSION, RXD_MINOR_VERSION),
+ .fi_version = RXD_FI_VERSION,
+ .getinfo = rxd_getinfo,
+ .fabric = rxd_fabric,
+ .cleanup = rxd_fini
};
-UDP_INI
+RXD_INI
{
- return &udpx_prov;
+ return &rxd_prov;
}
diff --git a/prov/rxd/src/rxd_rma.c b/prov/rxd/src/rxd_rma.c
new file mode 100644
index 0000000..6e1421e
--- /dev/null
+++ b/prov/rxd/src/rxd_rma.c
@@ -0,0 +1,365 @@
+/*
+ * Copyright (c) 2013-2016 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <fi_mem.h>
+#include "rxd.h"
+
+ssize_t rxd_ep_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
+ uint64_t flags)
+{
+ ssize_t ret;
+ uint64_t peer_addr;
+ struct rxd_ep *rxd_ep;
+ struct rxd_peer *peer;
+ struct rxd_tx_entry *tx_entry;
+ rxd_ep = container_of(ep, struct rxd_ep, ep);
+
+ peer_addr = rxd_av_get_dg_addr(rxd_ep->av, msg->addr);
+ peer = rxd_ep_getpeer_info(rxd_ep, peer_addr);
+
+#if ENABLE_DEBUG
+ if (msg->iov_count > RXD_IOV_LIMIT ||
+ msg->rma_iov_count > RXD_IOV_LIMIT)
+ return -FI_EINVAL;
+#endif
+
+ rxd_ep_lock_if_required(rxd_ep);
+ if (!peer->addr_published) {
+ ret = rxd_ep_post_conn_msg(rxd_ep, peer, peer_addr);
+ ret = (ret) ? ret : -FI_EAGAIN;
+ goto out;
+ }
+
+ tx_entry = rxd_tx_entry_acquire(rxd_ep, peer);
+ if (!tx_entry) {
+ ret = -FI_EAGAIN;
+ goto out;
+ }
+
+ dlist_init(&tx_entry->pkt_list);
+ tx_entry->op_type = RXD_TX_READ_REQ;
+ tx_entry->read_req.msg = *msg;
+ tx_entry->flags = flags;
+ tx_entry->peer = peer_addr;
+ rxd_ep_copy_msg_iov(msg->msg_iov,
+ &tx_entry->read_req.dst_iov[0], msg->iov_count);
+ rxd_ep_copy_rma_iov(msg->rma_iov,
+ &tx_entry->read_req.src_iov[0], msg->rma_iov_count);
+ ret = rxd_ep_post_start_msg(rxd_ep, peer, ofi_op_read_req, tx_entry);
+ if (ret)
+ goto err;
+
+ dlist_insert_tail(&tx_entry->entry, &rxd_ep->tx_entry_list);
+out:
+ rxd_ep_unlock_if_required(rxd_ep);
+ return ret;
+err:
+ rxd_tx_entry_release(rxd_ep, tx_entry);
+ goto out;
+}
+
+static ssize_t rxd_ep_read(struct fid_ep *ep, void *buf, size_t len,
+ void *desc, fi_addr_t src_addr, uint64_t addr,
+ uint64_t key, void *context)
+{
+ struct fi_msg_rma msg;
+ struct iovec msg_iov;
+ struct fi_rma_iov rma_iov;
+
+ memset(&msg, 0, sizeof(msg));
+ msg_iov.iov_base = (void *) buf;
+ msg_iov.iov_len = len;
+ msg.msg_iov = &msg_iov;
+ msg.desc = &desc;
+ msg.iov_count = 1;
+
+ rma_iov.addr = addr;
+ rma_iov.key = key;
+ rma_iov.len = len;
+ msg.rma_iov_count = 1;
+ msg.rma_iov = &rma_iov;
+
+ msg.addr = src_addr;
+ msg.context = context;
+
+ return rxd_ep_readmsg(ep, &msg, RXD_USE_OP_FLAGS);
+}
+
+static ssize_t rxd_ep_readv(struct fid_ep *ep, const struct iovec *iov,
+ void **desc, size_t count,
+ fi_addr_t src_addr, uint64_t addr, uint64_t key,
+ void *context)
+{
+ size_t len, i;
+ struct fi_msg_rma msg;
+ struct fi_rma_iov rma_iov;
+
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_iov = iov;
+ msg.desc = desc;
+ msg.iov_count = count;
+ msg.rma_iov_count = 1;
+
+ rma_iov.addr = addr;
+ rma_iov.key = key;
+
+#if ENABLE_DEBUG
+ if (count > RXD_IOV_LIMIT)
+ return -FI_EINVAL;
+#endif
+
+ for (i = 0, len = 0; i < count; i++)
+ len += iov[i].iov_len;
+ rma_iov.len = len;
+
+ msg.rma_iov = &rma_iov;
+ msg.addr = src_addr;
+ msg.context = context;
+
+ return rxd_ep_readmsg(ep, &msg, RXD_USE_OP_FLAGS);
+}
+
+ssize_t rxd_ep_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
+ uint64_t flags)
+{
+ ssize_t ret;
+ uint64_t peer_addr;
+ struct rxd_ep *rxd_ep;
+ struct rxd_peer *peer;
+ struct rxd_tx_entry *tx_entry;
+ rxd_ep = container_of(ep, struct rxd_ep, ep);
+
+ peer_addr = rxd_av_get_dg_addr(rxd_ep->av, msg->addr);
+ peer = rxd_ep_getpeer_info(rxd_ep, peer_addr);
+
+#if ENABLE_DEBUG
+ if (msg->iov_count > RXD_IOV_LIMIT ||
+ msg->rma_iov_count > RXD_IOV_LIMIT)
+ return -FI_EINVAL;
+#endif
+
+ rxd_ep_lock_if_required(rxd_ep);
+ if (!peer->addr_published) {
+ ret = rxd_ep_post_conn_msg(rxd_ep, peer, peer_addr);
+ ret = (ret) ? ret : -FI_EAGAIN;
+ goto out;
+ }
+
+ tx_entry = rxd_tx_entry_acquire(rxd_ep, peer);
+ if (!tx_entry) {
+ ret = -FI_EAGAIN;
+ goto out;
+ }
+
+ dlist_init(&tx_entry->pkt_list);
+ tx_entry->op_type = RXD_TX_WRITE;
+ tx_entry->write.msg = *msg;
+ tx_entry->flags = flags;
+ tx_entry->peer = peer_addr;
+ rxd_ep_copy_msg_iov(msg->msg_iov, &tx_entry->write.src_iov[0], msg->iov_count);
+ rxd_ep_copy_rma_iov(msg->rma_iov, &tx_entry->write.dst_iov[0], msg->rma_iov_count);
+
+ ret = rxd_ep_post_start_msg(rxd_ep, peer, ofi_op_write, tx_entry);
+ if (ret)
+ goto err;
+
+ dlist_insert_tail(&tx_entry->entry, &rxd_ep->tx_entry_list);
+out:
+ rxd_ep_unlock_if_required(rxd_ep);
+ return ret;
+err:
+ rxd_tx_entry_release(rxd_ep, tx_entry);
+ goto out;
+}
+
+static ssize_t rxd_ep_write(struct fid_ep *ep, const void *buf,
+ size_t len, void *desc, fi_addr_t dest_addr,
+ uint64_t addr, uint64_t key, void *context)
+{
+ struct fi_msg_rma msg;
+ struct iovec msg_iov;
+ struct fi_rma_iov rma_iov;
+
+ memset(&msg, 0, sizeof(msg));
+ msg_iov.iov_base = (void *) buf;
+ msg_iov.iov_len = len;
+
+ msg.msg_iov = &msg_iov;
+ msg.desc = &desc;
+ msg.iov_count = 1;
+
+ rma_iov.addr = addr;
+ rma_iov.key = key;
+ rma_iov.len = len;
+
+ msg.rma_iov_count = 1;
+ msg.rma_iov = &rma_iov;
+
+ msg.addr = dest_addr;
+ msg.context = context;
+
+ return rxd_ep_writemsg(ep, &msg, RXD_USE_OP_FLAGS);
+}
+
+static ssize_t rxd_ep_writev(struct fid_ep *ep, const struct iovec *iov,
+ void **desc, size_t count, fi_addr_t dest_addr,
+ uint64_t addr, uint64_t key, void *context)
+{
+ int i;
+ size_t len;
+ struct fi_msg_rma msg;
+ struct fi_rma_iov rma_iov;
+
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_iov = iov;
+ msg.desc = desc;
+ msg.iov_count = count;
+ msg.rma_iov_count = 1;
+
+#if ENABLE_DEBUG
+ if (count > RXD_IOV_LIMIT)
+ return -FI_EINVAL;
+#endif
+
+ for (i = 0, len = 0; i < count; i++)
+ len += iov[i].iov_len;
+
+ rma_iov.addr = addr;
+ rma_iov.key = key;
+ rma_iov.len = len;
+
+ msg.rma_iov = &rma_iov;
+ msg.context = context;
+ msg.addr = dest_addr;
+
+ return rxd_ep_writemsg(ep, &msg, RXD_USE_OP_FLAGS);
+}
+
+static ssize_t rxd_ep_writedata(struct fid_ep *ep, const void *buf,
+ size_t len, void *desc, uint64_t data,
+ fi_addr_t dest_addr, uint64_t addr,
+ uint64_t key, void *context)
+{
+ struct fi_msg_rma msg;
+ struct iovec msg_iov;
+ struct fi_rma_iov rma_iov;
+
+ msg_iov.iov_base = (void *) buf;
+ msg_iov.iov_len = len;
+ msg.desc = &desc;
+ msg.iov_count = 1;
+ msg.rma_iov_count = 1;
+
+ rma_iov.addr = addr;
+ rma_iov.key = key;
+ rma_iov.len = len;
+
+ msg.rma_iov = &rma_iov;
+ msg.msg_iov = &msg_iov;
+
+ msg.addr = dest_addr;
+ msg.context = context;
+ msg.data = data;
+
+ return rxd_ep_writemsg(ep, &msg, FI_REMOTE_CQ_DATA |
+ RXD_USE_OP_FLAGS);
+}
+
+static ssize_t rxd_ep_inject(struct fid_ep *ep, const void *buf,
+ size_t len, fi_addr_t dest_addr, uint64_t addr,
+ uint64_t key)
+{
+ struct fi_msg_rma msg;
+ struct iovec msg_iov;
+ struct fi_rma_iov rma_iov;
+
+ memset(&msg, 0, sizeof(msg));
+ msg_iov.iov_base = (void *) buf;
+ msg_iov.iov_len = len;
+ msg.msg_iov = &msg_iov;
+ msg.iov_count = 1;
+ msg.rma_iov_count = 1;
+
+ rma_iov.addr = addr;
+ rma_iov.key = key;
+ rma_iov.len = len;
+
+ msg.rma_iov = &rma_iov;
+ msg.msg_iov = &msg_iov;
+ msg.addr = dest_addr;
+
+ return rxd_ep_writemsg(ep, &msg, FI_INJECT |
+ RXD_NO_COMPLETION | RXD_USE_OP_FLAGS);
+}
+
+static ssize_t rxd_ep_injectdata(struct fid_ep *ep, const void *buf,
+ size_t len, uint64_t data,
+ fi_addr_t dest_addr, uint64_t addr,
+ uint64_t key)
+{
+ struct fi_msg_rma msg;
+ struct iovec msg_iov;
+ struct fi_rma_iov rma_iov;
+
+ memset(&msg, 0, sizeof(msg));
+ msg_iov.iov_base = (void *) buf;
+ msg_iov.iov_len = len;
+ msg.msg_iov = &msg_iov;
+ msg.iov_count = 1;
+ msg.rma_iov_count = 1;
+
+ rma_iov.addr = addr;
+ rma_iov.key = key;
+ rma_iov.len = len;
+
+ msg.rma_iov = &rma_iov;
+ msg.msg_iov = &msg_iov;
+ msg.addr = dest_addr;
+ msg.data = data;
+ return rxd_ep_writemsg(ep, &msg, FI_INJECT | FI_REMOTE_CQ_DATA |
+ RXD_NO_COMPLETION | RXD_USE_OP_FLAGS);
+}
+
+struct fi_ops_rma rxd_ops_rma = {
+ .size = sizeof (struct fi_ops_rma),
+ .read = rxd_ep_read,
+ .readv = rxd_ep_readv,
+ .readmsg = rxd_ep_readmsg,
+ .write = rxd_ep_write,
+ .writev = rxd_ep_writev,
+ .writemsg = rxd_ep_writemsg,
+ .inject = rxd_ep_inject,
+ .writedata = rxd_ep_writedata,
+ .injectdata = rxd_ep_injectdata,
+};
diff --git a/prov/rxm/Makefile.include b/prov/rxm/Makefile.include
new file mode 100644
index 0000000..c829682
--- /dev/null
+++ b/prov/rxm/Makefile.include
@@ -0,0 +1,26 @@
+if HAVE_RXM
+_rxm_files = \
+ prov/rxm/src/rxm_attr.c \
+ prov/rxm/src/rxm_init.c \
+ prov/rxm/src/rxm_fabric.c \
+ prov/rxm/src/rxm_domain.c \
+ prov/rxm/src/rxm_conn.c \
+ prov/rxm/src/rxm_ep.c \
+ prov/rxm/src/rxm_cq.c \
+ prov/rxm/src/rxm.h
+
+if HAVE_RXM_DL
+pkglib_LTLIBRARIES += librxm-fi.la
+librxm_fi_la_SOURCES = $(_rxm_files) $(common_srcs)
+librxm_fi_la_LIBADD = $(linkback) $(rxm_shm_LIBS)
+librxm_fi_la_LDFLAGS = -module -avoid-version -shared -export-dynamic
+librxm_fi_la_DEPENDENCIES = $(linkback)
+else !HAVE_RXM_DL
+src_libfabric_la_SOURCES += $(_rxm_files)
+src_libfabric_la_LIBADD += $(rxm_shm_LIBS)
+endif !HAVE_RXM_DL
+
+
+endif HAVE_RXM
+
+
diff --git a/prov/rxm/configure.m4 b/prov/rxm/configure.m4
new file mode 100644
index 0000000..391e7a6
--- /dev/null
+++ b/prov/rxm/configure.m4
@@ -0,0 +1,16 @@
+dnl Configury specific to the libfabric rxm provider
+
+dnl Called to configure this provider
+dnl
+dnl Arguments:
+dnl
+dnl $1: action if configured successfully
+dnl $2: action if not configured successfully
+dnl
+AC_DEFUN([FI_RXM_CONFIGURE],[
+ # Determine if we can support the rxm provider
+ rxm_h_happy=0
+ AS_IF([test x"$enable_rxm" != x"no"], [rxm_h_happy=1])
+ AS_IF([test $rxm_h_happy -eq 1], [$1], [$2])
+])
+
diff --git a/prov/rxm/src/rxm.h b/prov/rxm/src/rxm.h
new file mode 100644
index 0000000..86498e6
--- /dev/null
+++ b/prov/rxm/src/rxm.h
@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <pthread.h>
+#include <stdio.h>
+
+#include <rdma/fabric.h>
+#include <rdma/fi_errno.h>
+#include <rdma/fi_cm.h>
+#include <rdma/fi_domain.h>
+#include <rdma/fi_endpoint.h>
+#include <rdma/fi_eq.h>
+
+#include <fi.h>
+#include <fi_enosys.h>
+#include <fi_util.h>
+#include <fi_list.h>
+
+#ifndef _RXM_H_
+#define _RXM_H_
+
+#endif
+
+#define RXM_MAJOR_VERSION 1
+#define RXM_MINOR_VERSION 0
+
+extern struct fi_provider rxm_prov;
+extern struct util_prov rxm_util_prov;
+
+struct rxm_fabric {
+ struct util_fabric util_fabric;
+ struct fid_fabric *msg_fabric;
+ struct fid_eq *msg_eq;
+ pthread_t msg_listener_thread;
+};
+
+struct rxm_conn {
+ struct fid_ep *msg_ep;
+ struct util_cmap_handle handle;
+};
+
+struct rxm_domain {
+ struct util_domain util_domain;
+ struct fid_domain *msg_domain;
+};
+
+struct rxm_mr {
+ struct fid_mr mr_fid;
+ struct fid_mr *msg_mr;
+};
+
+struct rxm_cq {
+ struct util_cq util_cq;
+ struct fid_cq *msg_cq;
+};
+
+struct rxm_ep {
+ struct util_ep util_ep;
+ struct fi_info *msg_info;
+ struct fid_pep *msg_pep;
+ struct rxm_cq *tx_cq;
+ struct rxm_cq *rx_cq;
+ struct fid_ep *srx_ctx;
+ struct util_cmap *cmap;
+};
+
+extern struct fi_provider rxm_prov;
+extern struct fi_info rxm_info;
+extern struct fi_fabric_attr rxm_fabric_attr;
+extern struct fi_domain_attr rxm_domain_attr;
+
+int rxm_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric,
+ void *context);
+int rxm_alter_layer_info(struct fi_info *layer_info, struct fi_info *base_info);
+int rxm_alter_base_info(struct fi_info *base_info, struct fi_info *layer_info);
+int rxm_domain_open(struct fid_fabric *fabric, struct fi_info *info,
+ struct fid_domain **dom, void *context);
+int rxm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
+ struct fid_cq **cq_fid, void *context);
+int rxm_endpoint(struct fid_domain *domain, struct fi_info *info,
+ struct fid_ep **ep, void *context);
+
+void *rxm_msg_listener(void *arg);
+int rxm_msg_connect(struct rxm_ep *rxm_ep, fi_addr_t fi_addr,
+ struct fi_info *msg_info);
+int rxm_msg_process_connreq(struct rxm_ep *rxm_ep, struct fi_info *msg_info,
+ void *data, ssize_t datalen);
+int rxm_get_msg_ep(struct rxm_ep *rxm_ep, fi_addr_t fi_addr,
+ struct fid_ep **msg_ep);
+void rxm_conn_close(void *arg);
diff --git a/prov/udp/src/udpx_attr.c b/prov/rxm/src/rxm_attr.c
similarity index 66%
copy from prov/udp/src/udpx_attr.c
copy to prov/rxm/src/rxm_attr.c
index b320423..5f9ad7e 100644
--- a/prov/udp/src/udpx_attr.c
+++ b/prov/rxm/src/rxm_attr.c
@@ -30,61 +30,65 @@
* SOFTWARE.
*/
-#include "udpx.h"
+#include "rxm.h"
-
-struct fi_tx_attr udpx_tx_attr = {
+struct fi_tx_attr rxm_tx_attr = {
.caps = FI_MSG | FI_SEND,
.comp_order = FI_ORDER_STRICT,
- .inject_size = 1472,
+ .inject_size = 0,
.size = 1024,
- .iov_limit = UDPX_IOV_LIMIT
};
-struct fi_rx_attr udpx_rx_attr = {
- .caps = FI_MSG | FI_RECV | FI_SOURCE,
+struct fi_rx_attr rxm_rx_attr = {
+ .caps = FI_MSG | FI_RECV,
.comp_order = FI_ORDER_STRICT,
- .total_buffered_recv = (1 << 16),
.size = 1024,
- .iov_limit = UDPX_IOV_LIMIT
};
-struct fi_ep_attr udpx_ep_attr = {
- .type = FI_EP_DGRAM,
- .protocol = FI_PROTO_UDP,
- .protocol_version = 0,
- .max_msg_size = 1472,
+struct fi_ep_attr rxm_ep_attr = {
+ .type = FI_EP_RDM,
+ .protocol = FI_PROTO_RXM,
+ .protocol_version = 1,
+ .max_msg_size = SIZE_MAX,
.tx_ctx_cnt = 1,
.rx_ctx_cnt = 1
};
-struct fi_domain_attr udpx_domain_attr = {
- .name = "udp",
+struct fi_domain_attr rxm_domain_attr = {
+ .name = "rxm",
.threading = FI_THREAD_SAFE,
.control_progress = FI_PROGRESS_AUTO,
.data_progress = FI_PROGRESS_AUTO,
.resource_mgmt = FI_RM_ENABLED,
.av_type = FI_AV_UNSPEC,
- .mr_mode = FI_MR_SCALABLE,
+ .mr_mode = FI_MR_BASIC,
.cq_cnt = (1 << 16),
.ep_cnt = (1 << 15),
- .tx_ctx_cnt = (1 << 15),
- .rx_ctx_cnt = (1 << 15),
+ .tx_ctx_cnt = 1,
+ .rx_ctx_cnt = 1,
.max_ep_tx_ctx = 1,
.max_ep_rx_ctx = 1
};
-struct fi_fabric_attr udpx_fabric_attr = {
- .name = "UDP-IP",
- .prov_version = FI_VERSION(UDPX_MAJOR_VERSION, UDPX_MINOR_VERSION)
+struct fi_fabric_attr rxm_fabric_attr = {
+ .name = "",
+ .prov_version = FI_VERSION(RXM_MAJOR_VERSION, RXM_MINOR_VERSION),
+ .prov_name = "rxm",
+};
+
+struct fi_info rxm_info = {
+ .caps = FI_MSG | FI_SEND | FI_RECV | FI_SOURCE,
+ .mode = FI_LOCAL_MR,
+ .addr_format = FI_SOCKADDR,
+ .tx_attr = &rxm_tx_attr,
+ .rx_attr = &rxm_rx_attr,
+ .ep_attr = &rxm_ep_attr,
+ .domain_attr = &rxm_domain_attr,
+ .fabric_attr = &rxm_fabric_attr
};
-struct fi_info udpx_info = {
- .caps = FI_MSG | FI_SEND | FI_RECV | FI_SOURCE, /* | FI_MULTI_RECV, */
- .addr_format = FI_SOCKADDR_IN,
- .tx_attr = &udpx_tx_attr,
- .rx_attr = &udpx_rx_attr,
- .ep_attr = &udpx_ep_attr,
- .domain_attr = &udpx_domain_attr,
- .fabric_attr = &udpx_fabric_attr
+struct util_prov rxm_util_prov = {
+ .prov = &rxm_prov,
+ .info = &rxm_info,
+ .flags = 0,
};
diff --git a/prov/rxm/src/rxm_conn.c b/prov/rxm/src/rxm_conn.c
new file mode 100644
index 0000000..a44a7fc
--- /dev/null
+++ b/prov/rxm/src/rxm_conn.c
@@ -0,0 +1,355 @@
+/*
+ * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <fi_util.h>
+#include "rxm.h"
+
+int rxm_msg_ep_open(struct rxm_ep *rxm_ep, struct fi_info *msg_info,
+ struct rxm_conn *rxm_conn)
+{
+ struct rxm_domain *rxm_domain;
+ struct rxm_fabric *rxm_fabric;
+ struct rxm_cq *rxm_cq;
+ struct fid_ep *msg_ep;
+ int ret;
+
+ rxm_domain = container_of(rxm_ep->util_ep.domain, struct rxm_domain,
+ util_domain);
+ rxm_fabric = container_of(rxm_domain->util_domain.fabric, struct rxm_fabric,
+ util_fabric);
+ ret = fi_endpoint(rxm_domain->msg_domain, msg_info, &msg_ep, rxm_conn);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to create msg_ep\n");
+ return ret;
+ }
+
+ ret = fi_ep_bind(msg_ep, &rxm_fabric->msg_eq->fid, 0);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to bind msg EP to EQ\n");
+ goto err;
+ }
+
+ ret = fi_ep_bind(msg_ep, &rxm_ep->srx_ctx->fid, 0);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to bind msg EP to shared RX ctx\n");
+ goto err;
+ }
+
+ // TODO add other completion flags
+ if (rxm_ep->util_ep.tx_cq) {
+ rxm_cq = container_of(rxm_ep->util_ep.tx_cq, struct rxm_cq, util_cq);
+ ret = fi_ep_bind(msg_ep, &rxm_cq->msg_cq->fid, FI_TRANSMIT);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL,
+ "Unable to bind msg_ep to tx_cq\n");
+ goto err;
+ }
+ }
+
+ if (rxm_ep->util_ep.rx_cq) {
+ rxm_cq = container_of(rxm_ep->util_ep.rx_cq, struct rxm_cq, util_cq);
+ ret = fi_ep_bind(msg_ep, &rxm_cq->msg_cq->fid, FI_RECV);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL,
+ "Unable to bind msg_ep to rx_cq\n");
+ goto err;
+ }
+ }
+
+ ret = fi_enable(msg_ep);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to enable msg_ep\n");
+ goto err;
+ }
+
+ rxm_conn->msg_ep = msg_ep;
+ return 0;
+err:
+ fi_close(&msg_ep->fid);
+ return ret;
+}
+
+void rxm_conn_close(void *arg)
+{
+ struct util_cmap_handle *handle = (struct util_cmap_handle *)arg;
+ struct rxm_conn *rxm_conn = container_of(handle, struct rxm_conn, handle);
+ int ret;
+
+ if ((rxm_conn->handle.state == CMAP_UNSPEC) || !rxm_conn->msg_ep)
+ goto out;
+
+ if (rxm_conn->handle.state == CMAP_CONNECTED) {
+ ret = fi_shutdown(rxm_conn->msg_ep, 0);
+ if (ret)
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL,
+ "Unable to close connection\n");
+ }
+ ret = fi_close(&rxm_conn->msg_ep->fid);
+ if (ret)
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL,
+ "Unable to close msg_ep\n");
+out:
+ free(rxm_conn);
+}
+
+int rxm_msg_process_connreq(struct rxm_ep *rxm_ep, struct fi_info *msg_info,
+ void *data, ssize_t datalen)
+{
+ struct rxm_conn *rxm_conn;
+ int ret;
+
+ if (!datalen) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "No addr info received in cm_data\n");
+ ret = -FI_EINVAL;
+ goto err1;
+ }
+
+ if (!(rxm_conn = calloc(1, sizeof(*rxm_conn)))) {
+ ret = -FI_ENOMEM;
+ goto err1;
+ }
+
+ ret = ofi_cmap_add_handle(rxm_ep->cmap, &rxm_conn->handle, CMAP_CONNECTING,
+ FI_ADDR_UNSPEC, data, datalen);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to add handle/peer\n");
+ goto err2;
+ }
+
+ ret = rxm_msg_ep_open(rxm_ep, msg_info, rxm_conn);
+ if (ret)
+ goto err2;
+
+ ret = fi_accept(rxm_conn->msg_ep, NULL, 0);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC,
+ "Unable to accept incoming connection\n");
+ goto err2;
+ }
+ return ret;
+err2:
+ ofi_cmap_del_handle(&rxm_conn->handle);
+err1:
+ if (fi_reject(rxm_ep->msg_pep, msg_info->handle, NULL, 0))
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL,
+ "Unable to reject incoming connection\n");
+ return ret;
+}
+
+static void rxm_msg_process_connect_event(fid_t fid)
+{
+ struct rxm_conn *rxm_conn = (struct rxm_conn *)fid->context;
+ ofi_cmap_update_state(&rxm_conn->handle, CMAP_CONNECTED);
+}
+static void rxm_msg_process_shutdown_event(fid_t fid)
+{
+ // TODO process shutdown - need to diff local and remote shutdown
+ return;
+}
+
+void *rxm_msg_listener(void *arg)
+{
+ struct fi_eq_cm_entry *entry;
+ struct fi_eq_err_entry err_entry;
+ size_t datalen = sizeof(struct sockaddr);
+ size_t len = sizeof(*entry) + datalen;
+ struct rxm_fabric *rxm_fabric = (struct rxm_fabric *)arg;
+ uint32_t event;
+ ssize_t rd;
+ int ret;
+
+ entry = calloc(1, len);
+ if (!entry) {
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to allocate memory!\n");
+ return NULL;
+ }
+
+ FI_DBG(&rxm_prov, FI_LOG_FABRIC, "Starting MSG listener thread\n");
+ while (1) {
+ rd = fi_eq_sread(rxm_fabric->msg_eq, &event, entry, len, -1, 0);
+ /* We would receive more bytes than sizeof *entry during CONNREQ */
+ if (rd < 0) {
+ if (rd == -FI_EAVAIL)
+ OFI_EQ_READERR(&rxm_prov, FI_LOG_FABRIC,
+ rxm_fabric->msg_eq, rd, err_entry);
+ else
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC,
+ "msg: unable to fi_eq_sread\n");
+ continue;
+ }
+
+ switch(event) {
+ case FI_NOTIFY:
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Closing rxm msg listener\n");
+ return NULL;
+ case FI_CONNREQ:
+ if (rd != len)
+ goto err;
+ FI_DBG(&rxm_prov, FI_LOG_FABRIC, "Got new connection\n");
+ ret = rxm_msg_process_connreq(entry->fid->context, entry->info,
+ entry->data, rd - sizeof(*entry));
+ if (ret)
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC,
+ "Unable to process connection request\n");
+ break;
+ case FI_CONNECTED:
+ FI_DBG(&rxm_prov, FI_LOG_FABRIC, "Connected\n");
+ rxm_msg_process_connect_event(entry->fid);
+ break;
+ case FI_SHUTDOWN:
+ FI_DBG(&rxm_prov, FI_LOG_FABRIC, "Received connection shutdown\n");
+ rxm_msg_process_shutdown_event(entry->fid);
+ break;
+ default:
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unknown event: %u\n", event);
+ }
+ continue;
+err:
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC,
+ "Received size (%d) not matching expected (%d)\n", rd, len);
+ }
+}
+
+static int rxm_prepare_cm_data(struct fid_pep *pep, void *cm_data, size_t *cm_data_size)
+{
+ size_t opt_size = sizeof(*cm_data_size);
+ int ret;
+
+ ret = fi_getopt(&pep->fid, FI_OPT_ENDPOINT, FI_OPT_CM_DATA_SIZE,
+ cm_data_size, &opt_size);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "fi_getopt failed\n");
+ return ret;
+ }
+
+ ret = fi_getname(&pep->fid, cm_data, cm_data_size);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to get msg pep name\n");
+ return ret;
+ }
+
+ return 0;
+}
+
+int rxm_msg_connect(struct rxm_ep *rxm_ep, fi_addr_t fi_addr,
+ struct fi_info *msg_hints)
+{
+ struct rxm_conn *rxm_conn;
+ struct fi_info *msg_info;
+ struct sockaddr name;
+ size_t name_len = 0;
+ int ret;
+
+ assert(!msg_hints->dest_addr);
+
+ msg_hints->dest_addrlen = rxm_ep->util_ep.av->addrlen;
+ msg_hints->dest_addr = mem_dup(ofi_av_get_addr(rxm_ep->util_ep.av,
+ fi_addr), msg_hints->dest_addrlen);
+
+ ret = fi_getinfo(rxm_prov.version, NULL, NULL, 0, msg_hints, &msg_info);
+ if (ret)
+ return ret;
+
+ if (!(rxm_conn = calloc(1, sizeof(*rxm_conn)))) {
+ ret = -FI_ENOMEM;
+ goto err1;
+ }
+
+ ret = ofi_cmap_add_handle(rxm_ep->cmap, &rxm_conn->handle,
+ CMAP_CONNECTING, fi_addr, NULL, 0);
+ if (ret)
+ goto err2;
+
+ ret = rxm_msg_ep_open(rxm_ep, msg_info, rxm_conn);
+ if (ret)
+ goto err2;
+
+ /* We have to send passive endpoint's address to the server since the
+ * address from which connection request would be sent would have a
+ * different port. */
+ ret = rxm_prepare_cm_data(rxm_ep->msg_pep, &name, &name_len);
+ if (ret)
+ goto err2;
+
+ ret = fi_connect(rxm_conn->msg_ep, msg_info->dest_addr, &name, name_len);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to connect msg_ep\n");
+ goto err2;
+ }
+ fi_freeinfo(msg_info);
+ return 0;
+err2:
+ ofi_cmap_del_handle(&rxm_conn->handle);
+err1:
+ fi_freeinfo(msg_info);
+ return ret;
+}
+
+int rxm_get_msg_ep(struct rxm_ep *rxm_ep, fi_addr_t fi_addr,
+ struct fid_ep **msg_ep)
+{
+ struct util_cmap_handle *handle;
+ struct rxm_conn *rxm_conn;
+
+ if (fi_addr > rxm_ep->util_ep.av->count) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Invalid fi_addr\n");
+ return -FI_EINVAL;
+ }
+
+ handle = ofi_cmap_get_handle(rxm_ep->cmap, fi_addr);
+ if (!handle)
+ goto connect;
+
+ switch (handle->state) {
+ case CMAP_CONNECTING:
+ return -FI_EAGAIN;
+ case CMAP_CONNECTED:
+ rxm_conn = container_of(handle, struct rxm_conn, handle);
+ *msg_ep = rxm_conn->msg_ep;
+ return 0;
+ default:
+ /* We shouldn't be here */
+ assert(0);
+ }
+
+connect:
+ if (rxm_msg_connect(rxm_ep, fi_addr, rxm_ep->msg_info)) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "Unable to connect\n");
+ return -FI_EOTHER;
+ }
+ return -FI_EAGAIN;
+}
+
+
diff --git a/prov/rxm/src/rxm_cq.c b/prov/rxm/src/rxm_cq.c
new file mode 100644
index 0000000..8a77334
--- /dev/null
+++ b/prov/rxm/src/rxm_cq.c
@@ -0,0 +1,148 @@
+/*
+ * Copyright (c) 2013-2016 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "rxm.h"
+
+static int rxm_msg_cq_read(struct util_cq *util_cq, struct fid_cq *cq,
+ struct fi_cq_tagged_entry *comp)
+{
+ struct util_cq_err_entry *entry;
+ int ret;
+
+ ret = fi_cq_read(cq, comp, 1);
+ if (ret == -FI_EAVAIL) {
+ entry = calloc(1, sizeof(*entry));
+ if (!entry) {
+ FI_WARN(&rxm_prov, FI_LOG_CQ,
+ "Unable to allocate util_cq_err_entry\n");
+ return -FI_ENOMEM;
+ }
+ OFI_CQ_READERR(&rxm_prov, FI_LOG_CQ, cq, ret, entry->err_entry);
+ if (ret < 0) {
+ free(entry);
+ return ret;
+ }
+ slist_insert_tail(&entry->list_entry, &util_cq->err_list);
+ comp->flags = UTIL_FLAG_ERROR;
+ }
+
+ return ret;
+}
+
+void rxm_cq_progress(struct util_cq *util_cq)
+{
+ ssize_t ret = 0;
+ struct rxm_cq *rxm_cq;
+ struct fi_cq_tagged_entry *comp;
+
+ rxm_cq = container_of(util_cq, struct rxm_cq, util_cq);
+
+ fastlock_acquire(&util_cq->cq_lock);
+ do {
+ if (cirque_isfull(util_cq->cirq))
+ goto out;
+
+ comp = cirque_tail(util_cq->cirq);
+ ret = rxm_msg_cq_read(util_cq, rxm_cq->msg_cq, comp);
+ if (ret < 0)
+ goto out;
+ cirque_commit(util_cq->cirq);
+ } while (ret > 0);
+out:
+ fastlock_release(&util_cq->cq_lock);
+}
+
+static int rxm_cq_close(struct fid *fid)
+{
+ struct rxm_cq *rxm_cq;
+ int ret, retv = 0;
+
+ rxm_cq = container_of(fid, struct rxm_cq, util_cq.cq_fid.fid);
+
+ ret = ofi_cq_cleanup(&rxm_cq->util_cq);
+ if (ret)
+ retv = ret;
+
+ ret = fi_close(&rxm_cq->msg_cq->fid);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_CQ, "Unable to close MSG CQ\n");
+ retv = ret;
+ }
+ free(rxm_cq);
+ return retv;
+}
+
+static struct fi_ops rxm_cq_fi_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = rxm_cq_close,
+ .bind = fi_no_bind,
+ .control = fi_no_control,
+ .ops_open = fi_no_ops_open,
+};
+
+int rxm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
+ struct fid_cq **cq_fid, void *context)
+{
+ struct rxm_domain *rxm_domain;
+ struct rxm_cq *rxm_cq;
+ int ret;
+
+ rxm_cq = calloc(1, sizeof(*rxm_cq));
+ if (!rxm_cq)
+ return -FI_ENOMEM;
+
+ rxm_domain = container_of(domain, struct rxm_domain, util_domain.domain_fid);
+
+ ret = fi_cq_open(rxm_domain->msg_domain, attr, &rxm_cq->msg_cq, context);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_CQ, "Unable to open MSG CQ\n");
+ goto err1;
+ }
+
+ ret = ofi_cq_init(&rxm_prov, domain, attr, &rxm_cq->util_cq,
+ &rxm_cq_progress, context);
+ if (ret)
+ goto err2;
+
+ *cq_fid = &rxm_cq->util_cq.cq_fid;
+ /* Override util_cq_fi_ops */
+ (*cq_fid)->fid.ops = &rxm_cq_fi_ops;
+ return 0;
+err2:
+ fi_close(&rxm_cq->msg_cq->fid);
+err1:
+ free(rxm_cq);
+ return ret;
+}
diff --git a/prov/rxm/src/rxm_domain.c b/prov/rxm/src/rxm_domain.c
new file mode 100644
index 0000000..eea6f32
--- /dev/null
+++ b/prov/rxm/src/rxm_domain.c
@@ -0,0 +1,185 @@
+/*
+ * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <fi_util.h>
+#include "rxm.h"
+
+static struct fi_ops_domain rxm_domain_ops = {
+ .size = sizeof(struct fi_ops_domain),
+ .av_open = ip_av_create,
+ .cq_open = rxm_cq_open,
+ .endpoint = rxm_endpoint,
+ .scalable_ep = fi_no_scalable_ep,
+ .cntr_open = fi_no_cntr_open,
+ .poll_open = fi_poll_create,
+ .stx_ctx = fi_no_stx_context,
+ .srx_ctx = fi_no_srx_context,
+};
+
+static int rxm_domain_close(fid_t fid)
+{
+ struct rxm_domain *rxm_domain;
+ int ret;
+
+ rxm_domain = container_of(fid, struct rxm_domain, util_domain.domain_fid.fid);
+
+ ret = fi_close(&rxm_domain->msg_domain->fid);
+ if (ret)
+ return ret;
+
+ ret = ofi_domain_close(&rxm_domain->util_domain);
+ if (ret)
+ return ret;
+
+ free(rxm_domain);
+ return 0;
+}
+
+static struct fi_ops rxm_domain_fi_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = rxm_domain_close,
+ .bind = fi_no_bind,
+ .control = fi_no_control,
+ .ops_open = fi_no_ops_open,
+};
+
+static int rxm_mr_close(fid_t fid)
+{
+ struct rxm_mr *rxm_mr;
+ int ret;
+
+ rxm_mr = container_of(fid, struct rxm_mr, mr_fid.fid);
+ ret = fi_close(&rxm_mr->msg_mr->fid);
+ if (ret)
+ FI_WARN(&rxm_prov, FI_LOG_DOMAIN, "Unable to close MSG MR\n");
+ free(rxm_mr);
+ return ret;
+}
+
+static struct fi_ops rxm_mr_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = rxm_mr_close,
+ .bind = fi_no_bind,
+ .control = fi_no_control,
+ .ops_open = fi_no_ops_open,
+};
+
+static int rxm_mr_reg(struct fid *domain_fid, const void *buf, size_t len,
+ uint64_t access, uint64_t offset, uint64_t requested_key,
+ uint64_t flags, struct fid_mr **mr, void *context)
+{
+ struct rxm_domain *rxm_domain;
+ struct rxm_mr *rxm_mr;
+ int ret;
+
+ rxm_domain = container_of(domain_fid, struct rxm_domain,
+ util_domain.domain_fid.fid);
+
+ if (!(rxm_mr = calloc(1, sizeof(*rxm_mr))))
+ return -FI_ENOMEM;
+
+ ret = fi_mr_reg(rxm_domain->msg_domain, buf, len, access, offset, requested_key,
+ flags, &rxm_mr->msg_mr, context);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_DOMAIN, "Unable to register MSG MR\n");
+ goto err;
+ }
+
+ rxm_mr->mr_fid.fid.fclass = FI_CLASS_MR;
+ rxm_mr->mr_fid.fid.context = context;
+ rxm_mr->mr_fid.fid.ops = &rxm_mr_ops;
+ rxm_mr->mr_fid.mem_desc = fi_mr_desc(rxm_mr->msg_mr);
+ rxm_mr->mr_fid.key = fi_mr_key(rxm_mr->msg_mr);
+ *mr = &rxm_mr->mr_fid;
+
+ return 0;
+err:
+ free(rxm_mr);
+ return ret;
+}
+
+static struct fi_ops_mr rxm_domain_mr_ops = {
+ .size = sizeof(struct fi_ops_mr),
+ .reg = rxm_mr_reg,
+ .regv = fi_no_mr_regv,
+ .regattr = fi_no_mr_regattr,
+};
+
+int rxm_domain_open(struct fid_fabric *fabric, struct fi_info *info,
+ struct fid_domain **domain, void *context)
+{
+ int ret;
+ struct rxm_domain *rxm_domain;
+ struct rxm_fabric *rxm_fabric;
+ struct fi_info *msg_info;
+
+ rxm_domain = calloc(1, sizeof(*rxm_domain));
+ if (!rxm_domain)
+ return -FI_ENOMEM;
+
+ rxm_fabric = container_of(fabric, struct rxm_fabric, util_fabric.fabric_fid);
+
+ ret = ofix_getinfo(rxm_prov.version, NULL, NULL, 0, &rxm_util_prov,
+ info, rxm_alter_layer_info, rxm_alter_base_info,
+ 1, &msg_info);
+ if (ret)
+ goto err1;
+
+ ret = fi_domain(rxm_fabric->msg_fabric, msg_info,
+ &rxm_domain->msg_domain, context);
+ if (ret)
+ goto err2;
+
+ ret = ofi_domain_init(fabric, info, &rxm_domain->util_domain, context);
+ if (ret) {
+ goto err3;
+ }
+
+ *domain = &rxm_domain->util_domain.domain_fid;
+ (*domain)->fid.ops = &rxm_domain_fi_ops;
+ /* Replace MR ops set by ofi_domain_init() */
+ (*domain)->mr = &rxm_domain_mr_ops;
+ (*domain)->ops = &rxm_domain_ops;
+
+ return 0;
+err3:
+ fi_close(&rxm_domain->msg_domain->fid);
+err2:
+ fi_freeinfo(msg_info);
+err1:
+ free(rxm_domain);
+ return ret;
+}
diff --git a/prov/rxm/src/rxm_ep.c b/prov/rxm/src/rxm_ep.c
new file mode 100644
index 0000000..bf96dcc
--- /dev/null
+++ b/prov/rxm/src/rxm_ep.c
@@ -0,0 +1,446 @@
+/*
+ * Copyright (c) 2013-2016 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "rxm.h"
+
+
+int rxm_setname(fid_t fid, void *addr, size_t addrlen)
+{
+ struct rxm_ep *rxm_ep;
+
+ rxm_ep = container_of(fid, struct rxm_ep, util_ep.ep_fid.fid);
+ return fi_setname(&rxm_ep->msg_pep->fid, addr, addrlen);
+}
+
+int rxm_getname(fid_t fid, void *addr, size_t *addrlen)
+{
+ struct rxm_ep *rxm_ep;
+
+ rxm_ep = container_of(fid, struct rxm_ep, util_ep.ep_fid.fid);
+ return fi_getname(&rxm_ep->msg_pep->fid, addr, addrlen);
+}
+
+static struct fi_ops_cm rxm_cm_ops = {
+ .size = sizeof(struct fi_ops_cm),
+ .setname = rxm_setname,
+ .getname = rxm_getname,
+ .getpeer = fi_no_getpeer,
+ .connect = fi_no_connect,
+ .listen = fi_no_listen,
+ .accept = fi_no_accept,
+ .reject = fi_no_reject,
+ .shutdown = fi_no_shutdown,
+};
+
+int rxm_getopt(fid_t fid, int level, int optname,
+ void *optval, size_t *optlen)
+{
+ return -FI_ENOPROTOOPT;
+}
+
+int rxm_setopt(fid_t fid, int level, int optname,
+ const void *optval, size_t optlen)
+{
+ return -FI_ENOPROTOOPT;
+}
+
+static struct fi_ops_ep rxm_ep_ops = {
+ .size = sizeof(struct fi_ops_ep),
+ .cancel = fi_no_cancel,
+ .getopt = rxm_getopt,
+ .setopt = rxm_setopt,
+ .tx_ctx = fi_no_tx_ctx,
+ .rx_ctx = fi_no_rx_ctx,
+ .rx_size_left = fi_no_rx_size_left,
+ .tx_size_left = fi_no_tx_size_left,
+};
+
+ssize_t rxm_recvmsg(struct fid_ep *ep_fid, const struct fi_msg *msg,
+ uint64_t flags)
+{
+ struct rxm_ep *rxm_ep;
+
+ rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
+ return fi_recvmsg(rxm_ep->srx_ctx, msg, flags);
+}
+
+ssize_t rxm_recvv(struct fid_ep *ep_fid, const struct iovec *iov, void **desc,
+ size_t count, fi_addr_t src_addr, void *context)
+{
+ struct rxm_ep *rxm_ep;
+
+ rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
+ return fi_recvv(rxm_ep->srx_ctx, iov, desc, count, 0, context);
+}
+
+ssize_t rxm_recv(struct fid_ep *ep_fid, void *buf, size_t len, void *desc,
+ fi_addr_t src_addr, void *context)
+{
+ struct rxm_ep *rxm_ep;
+
+ /* TODO Handle recv from particular src */
+ if (src_addr) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_DATA,
+ "Unable to post recv for a particular source\n");
+ return -FI_EINVAL;
+ }
+
+ rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
+ return fi_recv(rxm_ep->srx_ctx, buf, len, desc, 0, context);
+}
+
+ssize_t rxm_send(struct fid_ep *ep_fid, const void *buf, size_t len, void *desc,
+ fi_addr_t dest_addr, void *context)
+{
+ struct rxm_ep *rxm_ep;
+ struct fid_ep *msg_ep;
+ ssize_t ret;
+
+ rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
+ fastlock_acquire(&rxm_ep->cmap->lock);
+ ret = rxm_get_msg_ep(rxm_ep, dest_addr, &msg_ep);
+ if (ret)
+ goto unlock;
+
+ // TODO handle the case when send fails due to connection shutdown
+ ret = fi_send(msg_ep, buf, len, desc, 0, context);
+unlock:
+ fastlock_release(&rxm_ep->cmap->lock);
+ return ret;
+}
+
+ssize_t rxm_sendmsg(struct fid_ep *ep_fid, const struct fi_msg *msg,
+ uint64_t flags)
+{
+ struct rxm_ep *rxm_ep;
+ struct fid_ep *msg_ep;
+ ssize_t ret;
+
+ rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
+ fastlock_acquire(&rxm_ep->cmap->lock);
+ ret = rxm_get_msg_ep(rxm_ep, msg->addr, &msg_ep);
+ if (ret)
+ goto unlock;
+
+ ret = fi_sendmsg(msg_ep, msg, flags);
+unlock:
+ fastlock_release(&rxm_ep->cmap->lock);
+ return ret;
+}
+
+ssize_t rxm_sendv(struct fid_ep *ep_fid, const struct iovec *iov, void **desc,
+ size_t count, fi_addr_t dest_addr, void *context)
+{
+ struct rxm_ep *rxm_ep;
+ struct fid_ep *msg_ep;
+ ssize_t ret;
+
+ rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
+ fastlock_acquire(&rxm_ep->cmap->lock);
+ ret = rxm_get_msg_ep(rxm_ep, dest_addr, &msg_ep);
+ if (ret)
+ goto unlock;
+
+ ret = fi_sendv(msg_ep, iov, desc, count, 0, context);
+unlock:
+ fastlock_release(&rxm_ep->cmap->lock);
+ return ret;
+}
+
+ssize_t rxm_inject(struct fid_ep *ep_fid, const void *buf, size_t len,
+ fi_addr_t dest_addr)
+{
+ struct rxm_ep *rxm_ep;
+ struct fid_ep *msg_ep;
+ ssize_t ret;
+
+ rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
+ fastlock_acquire(&rxm_ep->cmap->lock);
+ ret = rxm_get_msg_ep(rxm_ep, dest_addr, &msg_ep);
+ if (ret)
+ goto unlock;
+
+ ret = fi_inject(msg_ep, buf, len, 0);
+unlock:
+ fastlock_release(&rxm_ep->cmap->lock);
+ return ret;
+}
+
+static struct fi_ops_msg rxm_msg_ops = {
+ .size = sizeof(struct fi_ops_msg),
+ .recv = rxm_recv,
+ .recvv = rxm_recvv,
+ .recvmsg = rxm_recvmsg,
+ .send = rxm_send,
+ .sendv = rxm_sendv,
+ .sendmsg = rxm_sendmsg,
+ .inject = rxm_inject,
+ .senddata = fi_no_msg_senddata,
+ .injectdata = fi_no_msg_injectdata,
+};
+
+static int rxm_ep_close(struct fid *fid)
+{
+ struct rxm_ep *rxm_ep;
+ int ret, retv = 0;
+
+ rxm_ep = container_of(fid, struct rxm_ep, util_ep.ep_fid.fid);
+
+ if (rxm_ep->util_ep.av) {
+ ofi_cmap_free(rxm_ep->cmap);
+ atomic_dec(&rxm_ep->util_ep.av->ref);
+ }
+
+ ret = fi_close(&rxm_ep->srx_ctx->fid);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to close msg shared ctx\n");
+ retv = ret;
+ }
+
+ ret = fi_close(&rxm_ep->msg_pep->fid);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to close msg passive EP\n");
+ retv = ret;
+ }
+
+ fi_freeinfo(rxm_ep->msg_info);
+
+ if (rxm_ep->util_ep.rx_cq)
+ atomic_dec(&rxm_ep->util_ep.rx_cq->ref);
+
+ if (rxm_ep->util_ep.tx_cq)
+ atomic_dec(&rxm_ep->util_ep.tx_cq->ref);
+
+ atomic_dec(&rxm_ep->util_ep.domain->ref);
+ free(rxm_ep);
+ return retv;
+}
+
+static int rxm_ep_bind_cq(struct rxm_ep *rxm_ep, struct util_cq *util_cq, uint64_t flags)
+{
+ struct rxm_cq *rxm_cq;
+
+ rxm_cq = container_of(util_cq, struct rxm_cq, util_cq);
+
+ if (flags & ~(FI_TRANSMIT | FI_RECV)) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "unsupported flags\n");
+ return -FI_EBADFLAGS;
+ }
+
+ if (((flags & FI_TRANSMIT) && rxm_ep->tx_cq) ||
+ ((flags & FI_RECV) && rxm_ep->rx_cq)) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "duplicate CQ binding\n");
+ return -FI_EINVAL;
+ }
+
+ if (flags & FI_TRANSMIT) {
+ rxm_ep->util_ep.tx_cq = &rxm_cq->util_cq;
+ atomic_inc(&rxm_cq->util_cq.ref);
+ }
+
+ if (flags & FI_RECV) {
+ rxm_ep->util_ep.rx_cq = &rxm_cq->util_cq;
+ atomic_inc(&rxm_cq->util_cq.ref);
+ }
+ return 0;
+}
+
+int rxm_ep_bind_av(struct rxm_ep *rxm_ep, struct util_av *av)
+{
+ if (rxm_ep->util_ep.av) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL,
+ "duplicate AV binding\n");
+ return -FI_EINVAL;
+ }
+ rxm_ep->cmap = ofi_cmap_alloc(av, rxm_conn_close);
+ if (!rxm_ep->cmap)
+ return -FI_ENOMEM;
+
+ atomic_inc(&av->ref);
+ rxm_ep->util_ep.av = av;
+ return 0;
+}
+
+static int rxm_ep_bind(struct fid *ep_fid, struct fid *bfid, uint64_t flags)
+{
+ struct rxm_ep *rxm_ep;
+ int ret = 0;
+
+ rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
+ switch (bfid->fclass) {
+ case FI_CLASS_AV:
+ ret = rxm_ep_bind_av(rxm_ep, container_of(bfid, struct util_av,
+ av_fid.fid));
+ break;
+ case FI_CLASS_CQ:
+ ret = rxm_ep_bind_cq(rxm_ep, container_of(bfid, struct util_cq,
+ cq_fid.fid), flags);
+ break;
+ case FI_CLASS_EQ:
+ break;
+ default:
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL,
+ "invalid fid class\n");
+ ret = -FI_EINVAL;
+ break;
+ }
+ return ret;
+}
+
+static int rxm_ep_ctrl(struct fid *fid, int command, void *arg)
+{
+ struct rxm_ep *rxm_ep;
+ struct rxm_fabric *rxm_fabric;
+ int ret;
+
+ rxm_ep = container_of(fid, struct rxm_ep, util_ep.ep_fid.fid);
+ rxm_fabric = container_of(rxm_ep->util_ep.domain->fabric,
+ struct rxm_fabric, util_fabric);
+ switch (command) {
+ case FI_ENABLE:
+ if (!rxm_ep->util_ep.rx_cq || !rxm_ep->util_ep.tx_cq)
+ return -FI_ENOCQ;
+ if (!rxm_ep->util_ep.av)
+ return -FI_EOPBADSTATE; /* TODO: Add FI_ENOAV */
+ ret = fi_pep_bind(rxm_ep->msg_pep, &rxm_fabric->msg_eq->fid, 0);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL,
+ "Unable to bind msg PEP to msg EQ\n");
+ return ret;
+ }
+ ret = fi_listen(rxm_ep->msg_pep);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_EP_CTRL,
+ "Unable to set msg PEP to listen state\n");
+ return ret;
+ }
+ break;
+ default:
+ return -FI_ENOSYS;
+ }
+ return 0;
+}
+
+static struct fi_ops rxm_ep_fi_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = rxm_ep_close,
+ .bind = rxm_ep_bind,
+ .control = rxm_ep_ctrl,
+ .ops_open = fi_no_ops_open,
+};
+
+static int rxm_ep_msg_res_open(struct fi_info *rxm_info,
+ struct util_domain *util_domain, struct rxm_ep *rxm_ep)
+{
+ struct rxm_fabric *rxm_fabric;
+ struct rxm_domain *rxm_domain;
+ int ret;
+
+ ret = ofix_getinfo(rxm_prov.version, NULL, NULL, 0, &rxm_util_prov,
+ rxm_info, rxm_alter_layer_info, rxm_alter_base_info,
+ 1, &rxm_ep->msg_info);
+ if (ret)
+ return ret;
+
+ rxm_domain = container_of(util_domain, struct rxm_domain, util_domain);
+ rxm_fabric = container_of(util_domain->fabric, struct rxm_fabric, util_fabric);
+
+ ret = fi_passive_ep(rxm_fabric->msg_fabric, rxm_ep->msg_info, &rxm_ep->msg_pep, rxm_ep);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to open msg PEP\n");
+ goto err1;
+ }
+
+ ret = fi_srx_context(rxm_domain->msg_domain, rxm_ep->msg_info->rx_attr,
+ &rxm_ep->srx_ctx, NULL);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to open shared receive context\n");
+ goto err2;
+ }
+
+ /* We don't care what's in the dest_addr at this point. We go by AV. */
+ if (rxm_ep->msg_info->dest_addr) {
+ free(rxm_ep->msg_info->dest_addr);
+ rxm_ep->msg_info->dest_addr = NULL;
+ rxm_ep->msg_info->dest_addrlen = 0;
+ }
+
+ /* Zero out the port as we would be creating multiple MSG EPs for a single
+ * RXM EP and we don't want address conflicts. */
+ // TODO handle other address types?
+ if (rxm_ep->msg_info->src_addr)
+ ((struct sockaddr_in *)(rxm_ep->msg_info->src_addr))->sin_port = 0;
+
+ return 0;
+err2:
+ fi_close(&rxm_ep->msg_pep->fid);
+err1:
+ fi_freeinfo(rxm_ep->msg_info);
+ return ret;
+}
+
+int rxm_endpoint(struct fid_domain *domain, struct fi_info *info,
+ struct fid_ep **ep_fid, void *context)
+{
+ struct util_domain *util_domain;
+ struct rxm_ep *rxm_ep;
+ int ret;
+
+ rxm_ep = calloc(1, sizeof(*rxm_ep));
+ if (!rxm_ep)
+ return -FI_ENOMEM;
+
+ ret = ofi_endpoint_init(domain, &rxm_util_prov, info, &rxm_ep->util_ep,
+ context, FI_MATCH_PREFIX);
+ if (ret)
+ goto err;
+
+ util_domain = container_of(domain, struct util_domain, domain_fid);
+
+ ret = rxm_ep_msg_res_open(info, util_domain, rxm_ep);
+ if (ret)
+ goto err;
+
+ *ep_fid = &rxm_ep->util_ep.ep_fid;
+ (*ep_fid)->fid.ops = &rxm_ep_fi_ops;
+ (*ep_fid)->ops = &rxm_ep_ops;
+ (*ep_fid)->cm = &rxm_cm_ops;
+ (*ep_fid)->msg = &rxm_msg_ops;
+
+ return 0;
+err:
+ free(rxm_ep);
+ return ret;
+}
diff --git a/prov/rxm/src/rxm_fabric.c b/prov/rxm/src/rxm_fabric.c
new file mode 100644
index 0000000..f22c12f
--- /dev/null
+++ b/prov/rxm/src/rxm_fabric.c
@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <fi_util.h>
+
+#include "rxm.h"
+
+static struct fi_ops_fabric rxm_fabric_ops = {
+ .size = sizeof(struct fi_ops_fabric),
+ .domain = rxm_domain_open,
+ .passive_ep = fi_no_passive_ep,
+ .eq_open = ofi_eq_create,
+ .wait_open = ofi_wait_fd_open,
+ .trywait = ofi_trywait
+};
+
+static int rxm_fabric_close(fid_t fid)
+{
+ struct rxm_fabric *rxm_fabric;
+ struct fi_eq_entry entry = {0};
+ ssize_t rd;
+ int ret;
+
+ rxm_fabric = container_of(fid, struct rxm_fabric, util_fabric.fabric_fid.fid);
+
+ rd = fi_eq_write(rxm_fabric->msg_eq, FI_NOTIFY, &entry, sizeof(entry), 0);
+ if (rd != sizeof(entry)) {
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to notify listener thread\n");
+ return rd;
+ }
+
+ pthread_join(rxm_fabric->msg_listener_thread, NULL);
+
+ ret = fi_close(&rxm_fabric->msg_eq->fid);
+ if (ret)
+ return ret;
+
+ ret = fi_close(&rxm_fabric->msg_fabric->fid);
+ if (ret)
+ return ret;
+
+ ret = ofi_fabric_close(&rxm_fabric->util_fabric);
+ if (ret)
+ return ret;
+
+ free(rxm_fabric);
+ return 0;
+}
+
+static struct fi_ops rxm_fabric_fi_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = rxm_fabric_close,
+ .bind = fi_no_bind,
+ .control = fi_no_control,
+ .ops_open = fi_no_ops_open,
+};
+
+int rxm_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric,
+ void *context)
+{
+ struct rxm_fabric *rxm_fabric;
+ struct fi_info hints, *msg_info;
+ struct fi_eq_attr eq_attr;
+ int ret;
+
+ rxm_fabric = calloc(1, sizeof(*rxm_fabric));
+ if (!rxm_fabric)
+ return -FI_ENOMEM;
+
+ ret = ofi_fabric_init(&rxm_prov, &rxm_fabric_attr, attr,
+ &rxm_fabric->util_fabric, context, FI_MATCH_PREFIX);
+ if (ret)
+ goto err1;
+
+ memset(&hints, 0, sizeof hints);
+ if (!(hints.fabric_attr = calloc(1, sizeof(*hints.fabric_attr)))) {
+ ret = -FI_ENOMEM;
+ goto err2;
+ }
+ hints.fabric_attr->name = attr->name;
+ hints.mode = rxm_info.mode;
+
+ ret = ofix_getinfo(rxm_prov.version, NULL, NULL, 0, &rxm_util_prov,
+ &hints, rxm_alter_layer_info,
+ rxm_alter_base_info, 1, &msg_info);
+ if (ret) {
+ ret = -FI_EINVAL;
+ goto err3;
+ }
+
+ ret = fi_fabric(msg_info->fabric_attr, &rxm_fabric->msg_fabric, context);
+ if (ret) {
+ goto err4;
+ }
+
+ eq_attr.wait_obj = FI_WAIT_UNSPEC;
+ eq_attr.flags = FI_WRITE;
+
+ ret = fi_eq_open(rxm_fabric->msg_fabric, &eq_attr, &rxm_fabric->msg_eq, NULL);
+ if (ret) {
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to open msg EQ\n");
+ goto err5;
+ }
+
+ if (pthread_create(&rxm_fabric->msg_listener_thread, 0,
+ rxm_msg_listener, rxm_fabric)) {
+ ret = -errno;
+ FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to create msg_cm_listener_thread\n");
+ goto err6;
+ }
+
+ *fabric = &rxm_fabric->util_fabric.fabric_fid;
+ (*fabric)->fid.ops = &rxm_fabric_fi_ops;
+ (*fabric)->ops = &rxm_fabric_ops;
+
+ free(hints.fabric_attr);
+ fi_freeinfo(msg_info);
+ return 0;
+err6:
+ fi_close(&rxm_fabric->msg_eq->fid);
+err5:
+ fi_close(&rxm_fabric->msg_fabric->fid);
+err4:
+ fi_freeinfo(msg_info);
+err3:
+ free(hints.fabric_attr);
+err2:
+ ofi_fabric_close(&rxm_fabric->util_fabric);
+err1:
+ free(rxm_fabric);
+ return ret;
+}
diff --git a/prov/sockets/include/rdma/fi_direct.h b/prov/rxm/src/rxm_init.c
similarity index 50%
rename from prov/sockets/include/rdma/fi_direct.h
rename to prov/rxm/src/rxm_init.c
index 6d42ac4..91d6dcd 100644
--- a/prov/sockets/include/rdma/fi_direct.h
+++ b/prov/rxm/src/rxm_init.c
@@ -1,7 +1,5 @@
/*
* Copyright (c) 2016 Intel Corporation. All rights reserved.
- * Copyright (c) 2015-2016 Los Alamos National Security, LLC.
- * All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -32,6 +30,59 @@
* SOFTWARE.
*/
-/* Do not remove this file. All the include/rdma/fi_direct*.h files are created
- * build the provider with FABRIC_DIRECT option. For details see man/fi_direct.7.md
- */
+#include <rdma/fi_errno.h>
+
+#include <prov.h>
+#include "rxm.h"
+
+int rxm_alter_layer_info(struct fi_info *layer_info, struct fi_info *base_info)
+{
+ /* TODO choose base_info attr based on layer_info attr */
+ base_info->caps = FI_MSG;
+ base_info->mode = FI_LOCAL_MR;
+ base_info->ep_attr->rx_ctx_cnt = FI_SHARED_CONTEXT;
+ base_info->ep_attr->type = FI_EP_MSG;
+
+ return 0;
+}
+
+int rxm_alter_base_info(struct fi_info *base_info, struct fi_info *layer_info)
+{
+ // TODO choose caps based on base_info caps
+ layer_info->caps = rxm_info.caps;
+ layer_info->mode = rxm_info.mode;
+
+ *layer_info->tx_attr = *rxm_info.tx_attr;
+ *layer_info->rx_attr = *rxm_info.rx_attr;
+ *layer_info->ep_attr = *rxm_info.ep_attr;
+ layer_info->ep_attr->max_msg_size = base_info->ep_attr->max_msg_size;
+ *layer_info->domain_attr = *rxm_info.domain_attr;
+
+ return 0;
+}
+
+static int rxm_getinfo(uint32_t version, const char *node, const char *service,
+ uint64_t flags, struct fi_info *hints, struct fi_info **info)
+{
+ return ofix_getinfo(version, node, service, flags, &rxm_util_prov,
+ hints, rxm_alter_layer_info, rxm_alter_base_info, 0, info);
+}
+
+static void rxm_fini(void)
+{
+ /* yawn */
+}
+
+struct fi_provider rxm_prov = {
+ .name = "rxm",
+ .version = FI_VERSION(RXM_MAJOR_VERSION, RXM_MINOR_VERSION),
+ .fi_version = FI_VERSION(1, 3),
+ .getinfo = rxm_getinfo,
+ .fabric = rxm_fabric,
+ .cleanup = rxm_fini
+};
+
+RXM_INI
+{
+ return &rxm_prov;
+}
diff --git a/prov/sockets/Makefile.include b/prov/sockets/Makefile.include
index 84f87f9..ebf6541 100644
--- a/prov/sockets/Makefile.include
+++ b/prov/sockets/Makefile.include
@@ -1,6 +1,9 @@
-AM_CFLAGS += -I$(top_srcdir)/prov/sockets/include -I$(top_srcdir)/prov/sockets
+# Makefile.include for sockets provider
if HAVE_SOCKETS
+
+AM_CPPFLAGS += -I$(top_srcdir)/prov/sockets/include -I$(top_srcdir)/prov/sockets
+
_sockets_files = \
prov/sockets/src/sock_av.c \
prov/sockets/src/sock_dom.c \
diff --git a/prov/sockets/include/rdma/fi_direct_atomic.h b/prov/sockets/include/rdma/fi_direct_atomic.h
deleted file mode 100644
index 971eba0..0000000
--- a/prov/sockets/include/rdma/fi_direct_atomic.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-/* Do not remove this file. See fi_direct.h */
diff --git a/prov/sockets/include/rdma/fi_direct_atomic_def.h b/prov/sockets/include/rdma/fi_direct_atomic_def.h
deleted file mode 100644
index 971eba0..0000000
--- a/prov/sockets/include/rdma/fi_direct_atomic_def.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-/* Do not remove this file. See fi_direct.h */
diff --git a/prov/sockets/include/rdma/fi_direct_domain.h b/prov/sockets/include/rdma/fi_direct_domain.h
deleted file mode 100644
index 971eba0..0000000
--- a/prov/sockets/include/rdma/fi_direct_domain.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-/* Do not remove this file. See fi_direct.h */
diff --git a/prov/sockets/include/rdma/fi_direct_endpoint.h b/prov/sockets/include/rdma/fi_direct_endpoint.h
deleted file mode 100644
index 971eba0..0000000
--- a/prov/sockets/include/rdma/fi_direct_endpoint.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-/* Do not remove this file. See fi_direct.h */
diff --git a/prov/sockets/include/rdma/fi_direct_eq.h b/prov/sockets/include/rdma/fi_direct_eq.h
deleted file mode 100644
index 971eba0..0000000
--- a/prov/sockets/include/rdma/fi_direct_eq.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-/* Do not remove this file. See fi_direct.h */
diff --git a/prov/sockets/include/rdma/fi_direct_rma.h b/prov/sockets/include/rdma/fi_direct_rma.h
deleted file mode 100644
index 971eba0..0000000
--- a/prov/sockets/include/rdma/fi_direct_rma.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-/* Do not remove this file. See fi_direct.h */
diff --git a/prov/sockets/include/rdma/fi_direct_tagged.h b/prov/sockets/include/rdma/fi_direct_tagged.h
deleted file mode 100644
index 971eba0..0000000
--- a/prov/sockets/include/rdma/fi_direct_tagged.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-/* Do not remove this file. See fi_direct.h */
diff --git a/prov/sockets/include/rdma/fi_direct_trigger.h b/prov/sockets/include/rdma/fi_direct_trigger.h
deleted file mode 100644
index 971eba0..0000000
--- a/prov/sockets/include/rdma/fi_direct_trigger.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-/* Do not remove this file. See fi_direct.h */
diff --git a/prov/sockets/include/sock.h b/prov/sockets/include/sock.h
index 9985750..cda7794 100644
--- a/prov/sockets/include/sock.h
+++ b/prov/sockets/include/sock.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2014 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -41,7 +42,6 @@
#include <rdma/fi_endpoint.h>
#include <rdma/fi_eq.h>
#include <rdma/fi_errno.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_tagged.h>
#include <rdma/fi_trigger.h>
@@ -53,7 +53,9 @@
#include <fi_indexer.h>
#include <fi_rbuf.h>
#include <fi_list.h>
-#include <rbtree.h>
+#include <fi_file.h>
+#include <fi_osd.h>
+#include "fi_util.h"
#ifndef _SOCK_H_
#define _SOCK_H_
@@ -88,6 +90,7 @@
#define SOCK_CQ_DATA_SIZE (sizeof(uint64_t))
#define SOCK_TAG_SIZE (sizeof(uint64_t))
+#define SOCK_MAX_NETWORK_ADDR_SZ (35)
#define SOCK_PEP_LISTENER_TIMEOUT (10000)
#define SOCK_CM_COMM_TIMEOUT (2000)
@@ -154,7 +157,7 @@ enum {
SOCK_SIGNAL_WR_FD
};
-#define SOCK_MAJOR_VERSION 1
+#define SOCK_MAJOR_VERSION 2
#define SOCK_MINOR_VERSION 0
#define SOCK_WIRE_PROTO_VERSION (1)
@@ -192,12 +195,12 @@ struct sock_fabric {
struct sock_conn {
int sock_fd;
- int disconnected;
+ int connected;
int address_published;
struct sockaddr_in addr;
struct sock_pe_entry *rx_pe_entry;
struct sock_pe_entry *tx_pe_entry;
- struct sock_ep *ep;
+ struct sock_ep_attr *ep_attr;
fi_addr_t av_index;
struct dlist_entry ep_entry;
};
@@ -221,7 +224,7 @@ struct sock_domain {
struct sock_eq *mr_eq;
enum fi_progress progress_mode;
- RbtHandle mr_heap;
+ struct ofi_util_mr *mr_heap;
struct sock_pe *pe;
struct dlist_entry dom_list_entry;
struct fi_domain_attr attr;
@@ -268,9 +271,9 @@ struct sock_cntr {
struct fid_cntr cntr_fid;
struct sock_domain *domain;
atomic_t value;
- atomic_t threshold;
atomic_t ref;
atomic_t err_cnt;
+ atomic_t last_read_val;
pthread_cond_t cond;
pthread_mutex_t mut;
struct fi_cntr_attr attr;
@@ -284,21 +287,17 @@ struct sock_cntr {
struct fid_wait *waitset;
int signal;
- int is_waiting;
+ atomic_t num_waiting;
int err_flag;
};
struct sock_mr {
struct fid_mr mr_fid;
struct sock_domain *domain;
- uint64_t access;
- uint64_t offset;
uint64_t key;
uint64_t flags;
- size_t iov_count;
struct sock_cntr *cntr;
struct sock_cq *cq;
- struct iovec mr_iov[1];
};
struct sock_av_addr {
@@ -310,7 +309,6 @@ struct sock_av_addr {
struct sock_av_table_hdr {
uint64_t size;
uint64_t stored;
- uint64_t req_sz;
};
struct sock_av {
@@ -325,8 +323,10 @@ struct sock_av {
struct sock_av_table_hdr *table_hdr;
struct sock_av_addr *table;
uint64_t *idx_arr;
- char *name;
- int shared_fd;
+ struct util_shm shm;
+ int shared;
+ struct dlist_entry ep_list;
+ fastlock_t list_lock;
};
struct sock_fid_list {
@@ -481,7 +481,7 @@ struct sock_cm_entry {
int signal_fds[2];
uint64_t next_msg_id;
fastlock_t lock;
- int shutdown_received;
+ int is_connected;
pthread_t listener_thread;
struct dlist_entry msg_list;
};
@@ -495,10 +495,8 @@ struct sock_conn_listener {
char service[NI_MAXSERV];
};
-struct sock_ep {
- struct fid_ep ep;
+struct sock_ep_attr {
size_t fclass;
- uint64_t op_flags;
int tx_shared;
int rx_shared;
@@ -506,8 +504,6 @@ struct sock_ep {
size_t min_multi_recv;
atomic_t ref;
- struct sock_comp comp;
-
struct sock_eq *eq;
struct sock_av *av;
struct sock_domain *domain;
@@ -525,20 +521,18 @@ struct sock_ep {
struct fi_info info;
struct fi_ep_attr ep_attr;
- struct fi_tx_attr tx_attr;
- struct fi_rx_attr rx_attr;
enum fi_ep_type ep_type;
struct sockaddr_in *src_addr;
struct sockaddr_in *dest_addr;
+ uint16_t msg_src_port;
+ uint16_t msg_dest_port;
- struct sockaddr_in cm_addr;
uint64_t peer_fid;
uint16_t key;
- int is_disabled;
+ int is_enabled;
struct sock_cm_entry cm;
struct sock_conn_listener listener;
- struct dlist_entry conn_list;
fastlock_t lock;
struct index_map conn_idm;
@@ -546,6 +540,14 @@ struct sock_ep {
struct sock_conn_map cmap;
};
+struct sock_ep {
+ struct fid_ep ep;
+ struct fi_tx_attr tx_attr;
+ struct fi_rx_attr rx_attr;
+ struct sock_ep_attr *attr;
+ int is_alias;
+};
+
struct sock_pep {
struct fid_pep pep;
struct sock_fabric *sock_fab;
@@ -554,6 +556,7 @@ struct sock_pep {
struct sockaddr_in src_addr;
struct fi_info info;
struct sock_eq *eq;
+ int name_set;
};
struct sock_rx_entry {
@@ -591,21 +594,22 @@ struct sock_rx_ctx {
int progress;
int is_ctrl_ctx;
int recv_cq_event;
+ int use_shared;
size_t num_left;
size_t buffered_len;
size_t min_multi_recv;
uint64_t addr;
struct sock_comp comp;
+ struct sock_rx_ctx *srx_ctx;
- struct sock_ep *ep;
+ struct sock_ep_attr *ep_attr;
struct sock_av *av;
struct sock_eq *eq;
struct sock_domain *domain;
struct dlist_entry pe_entry;
struct dlist_entry cq_entry;
- struct dlist_entry cntr_entry;
struct dlist_entry pe_entry_list;
struct dlist_entry rx_entry_list;
@@ -633,18 +637,19 @@ struct sock_tx_ctx {
uint8_t enabled;
uint8_t progress;
+ int use_shared;
uint64_t addr;
struct sock_comp comp;
struct sock_rx_ctx *rx_ctrl_ctx;
+ struct sock_tx_ctx *stx_ctx;
- struct sock_ep *ep;
+ struct sock_ep_attr *ep_attr;
struct sock_av *av;
struct sock_eq *eq;
struct sock_domain *domain;
struct dlist_entry pe_entry;
struct dlist_entry cq_entry;
- struct dlist_entry cntr_entry;
struct dlist_entry pe_entry_list;
struct dlist_entry ep_list;
@@ -787,7 +792,7 @@ struct sock_pe_entry {
uint64_t data_len;
uint64_t rem;
void *comm_addr;
- struct sock_ep *ep;
+ struct sock_ep_attr *ep_attr;
struct sock_conn *conn;
struct sock_comp *comp;
@@ -812,6 +817,7 @@ struct sock_pe {
struct util_buf_pool *atomic_rx_pool;
struct dlist_entry free_list;
struct dlist_entry busy_list;
+ struct dlist_entry pool_list;
struct dlist_entry tx_list;
struct dlist_entry rx_list;
@@ -856,42 +862,19 @@ struct sock_cq {
sock_cq_report_fn report_completion;
};
-struct sock_cm_msg_list_entry {
- uint64_t msg_len;
- uint8_t retry;
- uint8_t reserved[7];
- uint64_t timestamp_ms;
- struct sockaddr_in addr;
- struct dlist_entry entry;
- fid_t fid;
- struct sock_eq *eq;
- char msg[0];
-};
-
struct sock_conn_hdr {
uint8_t type;
uint8_t reserved[3];
- int32_t s_port;
- uint64_t msg_id;
+ uint16_t port;
+ uint16_t cm_data_sz;
+ char cm_data[0];
};
struct sock_conn_req {
struct sock_conn_hdr hdr;
- struct fi_info info;
struct sockaddr_in src_addr;
- struct sockaddr_in dest_addr;
- struct fi_tx_attr tx_attr;
- struct fi_rx_attr rx_attr;
- struct fi_ep_attr ep_attr;
- struct fi_domain_attr domain_attr;
- struct fi_fabric_attr fabric_attr;
- struct sockaddr_in from_addr;
- char user_data[0];
-};
-
-struct sock_conn_response {
- struct sock_conn_hdr hdr;
- char user_data[0];
+ uint64_t caps;
+ char cm_data[0];
};
enum {
@@ -899,12 +882,25 @@ enum {
SOCK_CONN_ACCEPT,
SOCK_CONN_REJECT,
SOCK_CONN_SHUTDOWN,
- SOCK_CONN_ACK
};
struct sock_conn_req_handle {
struct fid handle;
struct sock_conn_req *req;
+ int sock_fd;
+ int is_accepted;
+ struct sock_pep *pep;
+ struct sock_ep *ep;
+ size_t paramlen;
+ pthread_t req_handler;
+ struct sockaddr_in dest_addr;
+ struct dlist_entry entry;
+ char cm_data[SOCK_EP_MAX_CM_DATA_SZ];
+};
+
+struct sock_host_list_entry {
+ char hostname[HOST_NAME_MAX];
+ struct slist_entry entry;
};
union sock_tx_op {
@@ -948,6 +944,7 @@ int sock_verify_info(struct fi_info *hints);
int sock_verify_fabric_attr(struct fi_fabric_attr *attr);
int sock_verify_domain_attr(struct fi_domain_attr *attr);
+size_t sock_get_tx_size(size_t size);
int sock_rdm_verify_ep_attr(struct fi_ep_attr *ep_attr, struct fi_tx_attr *tx_attr,
struct fi_rx_attr *rx_attr);
int sock_dgram_verify_ep_attr(struct fi_ep_attr *ep_attr, struct fi_tx_attr *tx_attr,
@@ -957,7 +954,6 @@ int sock_msg_verify_ep_attr(struct fi_ep_attr *ep_attr, struct fi_tx_attr *tx_at
int sock_get_src_addr(struct sockaddr_in *dest_addr,
struct sockaddr_in *src_addr);
int sock_get_src_addr_from_hostname(struct sockaddr_in *src_addr, const char *service);
-void sock_getnodename(char *buf, int buflen);
struct fi_info *sock_fi_info(enum fi_ep_type ep_type,
struct fi_info *hints, void *src_addr, void *dest_addr);
@@ -974,10 +970,6 @@ int sock_msg_getinfo(uint32_t version, const char *node, const char *service,
int sock_domain(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **dom, void *context);
-void sock_fabric_add_service(struct sock_fabric *fab, int service);
-void sock_fabric_remove_service(struct sock_fabric *fab, int service);
-int sock_fabric_check_service(struct sock_fabric *fab, int service);
-
void sock_dom_add_to_list(struct sock_domain *domain);
int sock_dom_check_list(struct sock_domain *domain);
void sock_dom_remove_from_list(struct sock_domain *domain);
@@ -1021,6 +1013,10 @@ int sock_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
int sock_cq_report_error(struct sock_cq *cq, struct sock_pe_entry *entry,
size_t olen, int err, int prov_errno, void *err_data);
int sock_cq_progress(struct sock_cq *cq);
+void sock_cq_add_tx_ctx(struct sock_cq *cq, struct sock_tx_ctx *tx_ctx);
+void sock_cq_remove_tx_ctx(struct sock_cq *cq, struct sock_tx_ctx *tx_ctx);
+void sock_cq_add_rx_ctx(struct sock_cq *cq, struct sock_rx_ctx *rx_ctx);
+void sock_cq_remove_rx_ctx(struct sock_cq *cq, struct sock_rx_ctx *rx_ctx);
int sock_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr,
@@ -1037,19 +1033,23 @@ int sock_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
void sock_cntr_inc(struct sock_cntr *cntr);
void sock_cntr_err_inc(struct sock_cntr *cntr);
int sock_cntr_progress(struct sock_cntr *cntr);
+void sock_cntr_add_tx_ctx(struct sock_cntr *cntr, struct sock_tx_ctx *tx_ctx);
+void sock_cntr_remove_tx_ctx(struct sock_cntr *cntr, struct sock_tx_ctx *tx_ctx);
+void sock_cntr_add_rx_ctx(struct sock_cntr *cntr, struct sock_rx_ctx *rx_ctx);
+void sock_cntr_remove_rx_ctx(struct sock_cntr *cntr, struct sock_rx_ctx *rx_ctx);
-struct sock_mr *sock_mr_verify_key(struct sock_domain *domain, uint64_t key,
- void *buf, size_t len, uint64_t access);
+struct sock_mr *sock_mr_verify_key(struct sock_domain *domain, uint64_t key,
+ uintptr_t *buf, size_t len, uint64_t access);
struct sock_mr *sock_mr_verify_desc(struct sock_domain *domain, void *desc,
void *buf, size_t len, uint64_t access);
-struct sock_mr * sock_mr_get_entry(struct sock_domain *domain, uint64_t key);
-
-struct sock_rx_ctx *sock_rx_ctx_alloc(const struct fi_rx_attr *attr, void *context);
+struct sock_rx_ctx *sock_rx_ctx_alloc(const struct fi_rx_attr *attr,
+ void *context, int use_shared);
void sock_rx_ctx_free(struct sock_rx_ctx *rx_ctx);
-struct sock_tx_ctx *sock_tx_ctx_alloc(const struct fi_tx_attr *attr, void *context);
+struct sock_tx_ctx *sock_tx_ctx_alloc(const struct fi_tx_attr *attr,
+ void *context, int use_shared);
struct sock_tx_ctx *sock_stx_ctx_alloc(const struct fi_tx_attr *attr, void *context);
void sock_tx_ctx_free(struct sock_tx_ctx *tx_ctx);
void sock_tx_ctx_start(struct sock_tx_ctx *tx_ctx);
@@ -1058,15 +1058,15 @@ void sock_tx_ctx_commit(struct sock_tx_ctx *tx_ctx);
void sock_tx_ctx_abort(struct sock_tx_ctx *tx_ctx);
void sock_tx_ctx_write_op_send(struct sock_tx_ctx *tx_ctx,
struct sock_op *op, uint64_t flags, uint64_t context,
- uint64_t dest_addr, uint64_t buf, struct sock_ep *ep,
+ uint64_t dest_addr, uint64_t buf, struct sock_ep_attr *ep_attr,
struct sock_conn *conn);
void sock_tx_ctx_write_op_tsend(struct sock_tx_ctx *tx_ctx,
struct sock_op *op, uint64_t flags, uint64_t context,
- uint64_t dest_addr, uint64_t buf, struct sock_ep *ep,
+ uint64_t dest_addr, uint64_t buf, struct sock_ep_attr *ep_attr,
struct sock_conn *conn, uint64_t tag);
void sock_tx_ctx_read_op_send(struct sock_tx_ctx *tx_ctx,
struct sock_op *op, uint64_t *flags, uint64_t *context,
- uint64_t *dest_addr, uint64_t *buf, struct sock_ep **ep,
+ uint64_t *dest_addr, uint64_t *buf, struct sock_ep_attr **ep_attr,
struct sock_conn **conn);
int sock_poll_open(struct fid_domain *domain, struct fi_poll_attr *attr,
@@ -1081,27 +1081,31 @@ int sock_wait_close(fid_t fid);
int sock_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
struct fid_av **av, void *context);
int sock_av_compare_addr(struct sock_av *av, fi_addr_t addr1, fi_addr_t addr2);
-int sock_compare_addr(struct sockaddr_in *addr1, struct sockaddr_in *addr2);
int sock_av_get_addr_index(struct sock_av *av, struct sockaddr_in *addr);
-struct sock_conn *sock_ep_lookup_conn(struct sock_ep *ep, fi_addr_t index,
+struct sock_conn *sock_ep_lookup_conn(struct sock_ep_attr *attr, fi_addr_t index,
struct sockaddr_in *addr);
-int sock_ep_get_conn(struct sock_ep *ep, struct sock_tx_ctx *tx_ctx,
+int sock_ep_get_conn(struct sock_ep_attr *ep_attr, struct sock_tx_ctx *tx_ctx,
fi_addr_t index, struct sock_conn **pconn);
-struct sock_conn *sock_ep_connect(struct sock_ep *ep, fi_addr_t index);
-ssize_t sock_conn_send_src_addr(struct sock_ep *sock_ep, struct sock_tx_ctx *tx_ctx,
+void sock_ep_remove_conn(struct sock_ep_attr *ep_attr, struct sock_conn *conn);
+struct sock_conn *sock_ep_connect(struct sock_ep_attr *attr, fi_addr_t index);
+ssize_t sock_conn_send_src_addr(struct sock_ep_attr *ep_attr, struct sock_tx_ctx *tx_ctx,
struct sock_conn *conn);
-int sock_conn_listen(struct sock_ep *ep);
-void sock_conn_map_destroy(struct sock_conn_map *cmap);
+int sock_conn_listen(struct sock_ep_attr *ep_attr);
+void sock_conn_map_destroy(struct sock_ep_attr *ep_attr);
+void sock_conn_release_entry(struct sock_conn_map *map, struct sock_conn *conn);
void sock_set_sockopts(int sock);
int fd_set_nonblock(int fd);
+void sock_set_sockopt_reuseaddr(int sock);
int sock_conn_map_init(struct sock_ep *ep, int init_size);
+void sock_set_sockopts_conn(int sock);
struct sock_pe *sock_pe_init(struct sock_domain *domain);
void sock_pe_add_tx_ctx(struct sock_pe *pe, struct sock_tx_ctx *ctx);
void sock_pe_add_rx_ctx(struct sock_pe *pe, struct sock_rx_ctx *ctx);
void sock_pe_signal(struct sock_pe *pe);
void sock_pe_poll_add(struct sock_pe *pe, int fd);
+void sock_pe_poll_del(struct sock_pe *pe, int fd);
int sock_pe_progress_rx_ctx(struct sock_pe *pe, struct sock_rx_ctx *rx_ctx);
int sock_pe_progress_tx_ctx(struct sock_pe *pe, struct sock_tx_ctx *tx_ctx);
void sock_pe_remove_tx_ctx(struct sock_tx_ctx *tx_ctx);
@@ -1125,7 +1129,6 @@ ssize_t sock_rx_claim_recv(struct sock_rx_ctx *rx_ctx, void *context,
uint64_t flags, uint64_t tag, uint64_t ignore,
uint8_t is_tagged, const struct iovec *msg_iov,
size_t iov_count);
-size_t sock_rx_avail_len(struct sock_rx_entry *rx_entry);
void sock_rx_release_entry(struct sock_rx_entry *rx_entry);
ssize_t sock_comm_send(struct sock_pe_entry *pe_entry, const void *buf, size_t len);
@@ -1134,6 +1137,7 @@ ssize_t sock_comm_peek(struct sock_conn *conn, void *buf, size_t len);
ssize_t sock_comm_discard(struct sock_pe_entry *pe_entry, size_t len);
int sock_comm_tx_done(struct sock_pe_entry *pe_entry);
ssize_t sock_comm_flush(struct sock_pe_entry *pe_entry);
+int sock_comm_is_disconnected(struct sock_pe_entry *pe_entry);
ssize_t sock_ep_recvmsg(struct fid_ep *ep, const struct fi_msg *msg,
uint64_t flags);
@@ -1164,6 +1168,7 @@ ssize_t sock_queue_tmsg_op(struct fid_ep *ep, const struct fi_msg_tagged *msg,
uint64_t flags, uint8_t op_type);
ssize_t sock_queue_msg_op(struct fid_ep *ep, const struct fi_msg *msg,
uint64_t flags, uint8_t op_type);
+void sock_cntr_check_trigger_list(struct sock_cntr *cntr);
int sock_epoll_create(struct sock_epoll_set *set, int size);
int sock_epoll_add(struct sock_epoll_set *set, int fd);
@@ -1172,4 +1177,9 @@ int sock_epoll_wait(struct sock_epoll_set *set, int timeout);
int sock_epoll_get_fd_at_index(struct sock_epoll_set *set, int index);
void sock_epoll_close(struct sock_epoll_set *set);
+static inline size_t sock_rx_avail_len(struct sock_rx_entry *rx_entry)
+{
+ return rx_entry->total_len - rx_entry->used;
+}
+
#endif
diff --git a/prov/sockets/include/sock_util.h b/prov/sockets/include/sock_util.h
index fe60fbf..b093dc5 100644
--- a/prov/sockets/include/sock_util.h
+++ b/prov/sockets/include/sock_util.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2014 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -34,7 +35,7 @@
#define _SOCK_UTIL_H_
#include <sys/mman.h>
-#include <rdma/fi_log.h>
+#include <rdma/providers/fi_log.h>
#include "sock.h"
extern const char sock_fab_name[];
@@ -55,12 +56,12 @@ extern int sock_dgram_drop_rate;
#define _SOCK_LOG_DBG(subsys, ...) FI_DBG(&sock_prov, subsys, __VA_ARGS__)
#define _SOCK_LOG_ERROR(subsys, ...) FI_WARN(&sock_prov, subsys, __VA_ARGS__)
-static inline int sock_drop_packet(struct sock_ep *sock_ep)
+static inline int sock_drop_packet(struct sock_ep_attr *ep_attr)
{
#if ENABLE_DEBUG
- if (sock_ep->ep_type == FI_EP_DGRAM && sock_dgram_drop_rate > 0) {
- sock_ep->domain->fab->num_send_msg++;
- if (!(sock_ep->domain->fab->num_send_msg % sock_dgram_drop_rate))
+ if (ep_attr->ep_type == FI_EP_DGRAM && sock_dgram_drop_rate > 0) {
+ ep_attr->domain->fab->num_send_msg++;
+ if (!(ep_attr->domain->fab->num_send_msg % sock_dgram_drop_rate))
return 1;
}
#endif
diff --git a/prov/sockets/provider_FABRIC_1.0.map b/prov/sockets/provider_FABRIC_1.0.map
deleted file mode 100644
index b1029d5..0000000
--- a/prov/sockets/provider_FABRIC_1.0.map
+++ /dev/null
@@ -1 +0,0 @@
-/* Do not remove this file. This is needed for FABRIC_DIRECT option. See man/fi_direct.7.md for details. */
diff --git a/prov/sockets/src/sock_atomic.c b/prov/sockets/src/sock_atomic.c
index cf2b16f..b3b4140 100644
--- a/prov/sockets/src/sock_atomic.c
+++ b/prov/sockets/src/sock_atomic.c
@@ -69,17 +69,22 @@ ssize_t sock_ep_tx_atomic(struct fid_ep *ep,
union sock_iov tx_iov;
struct sock_conn *conn;
struct sock_tx_ctx *tx_ctx;
- uint64_t total_len, src_len, dst_len;
+ uint64_t total_len, src_len, dst_len, cmp_len, op_flags;
struct sock_ep *sock_ep;
+ struct sock_ep_attr *ep_attr;
switch (ep->fid.fclass) {
case FI_CLASS_EP:
sock_ep = container_of(ep, struct sock_ep, ep);
- tx_ctx = sock_ep->tx_ctx;
+ tx_ctx = sock_ep->attr->tx_ctx->use_shared ?
+ sock_ep->attr->tx_ctx->stx_ctx : sock_ep->attr->tx_ctx;
+ ep_attr = sock_ep->attr;
+ op_flags = sock_ep->tx_attr.op_flags;
break;
case FI_CLASS_TX_CTX:
tx_ctx = container_of(ep, struct sock_tx_ctx, fid.ctx);
- sock_ep = tx_ctx->ep;
+ ep_attr = tx_ctx->ep_attr;
+ op_flags = tx_ctx->attr.op_flags;
break;
default:
SOCK_LOG_ERROR("Invalid EP type\n");
@@ -93,13 +98,13 @@ ssize_t sock_ep_tx_atomic(struct fid_ep *ep,
if (!tx_ctx->enabled)
return -FI_EOPBADSTATE;
- ret = sock_ep_get_conn(sock_ep, tx_ctx, msg->addr, &conn);
+ ret = sock_ep_get_conn(ep_attr, tx_ctx, msg->addr, &conn);
if (ret)
return ret;
SOCK_EP_SET_TX_OP_FLAGS(flags);
if (flags & SOCK_USE_OP_FLAGS)
- flags |= tx_ctx->attr.op_flags;
+ flags |= op_flags;
if (msg->op == FI_ATOMIC_READ) {
flags &= ~FI_INJECT;
@@ -113,16 +118,18 @@ ssize_t sock_ep_tx_atomic(struct fid_ep *ep,
return ret;
}
- src_len = 0;
+ src_len = cmp_len = 0;
datatype_sz = fi_datatype_size(msg->datatype);
+ for (i = 0; i < compare_count; i++)
+ cmp_len += (comparev[i].count * datatype_sz);
if (flags & FI_INJECT) {
for (i = 0; i < msg->iov_count; i++)
src_len += (msg->msg_iov[i].count * datatype_sz);
- if (src_len > SOCK_EP_MAX_INJECT_SZ)
+ if ((src_len + cmp_len) > SOCK_EP_MAX_INJECT_SZ)
return -FI_EINVAL;
- total_len = src_len;
+ total_len = src_len + cmp_len;
} else {
total_len = msg->iov_count * sizeof(union sock_iov);
}
@@ -145,25 +152,32 @@ ssize_t sock_ep_tx_atomic(struct fid_ep *ep,
tx_op.atomic.res_iov_len = result_count;
tx_op.atomic.cmp_iov_len = compare_count;
- if (flags & FI_INJECT)
+ if (flags & FI_INJECT) {
tx_op.src_iov_len = src_len;
- else
+ tx_op.atomic.cmp_iov_len = cmp_len;
+ } else {
tx_op.src_iov_len = msg->iov_count;
+ }
sock_tx_ctx_write_op_send(tx_ctx, &tx_op, flags,
(uintptr_t) msg->context, msg->addr,
- (uintptr_t) msg->msg_iov[0].addr, sock_ep, conn);
+ (uintptr_t) msg->msg_iov[0].addr, ep_attr, conn);
if (flags & FI_REMOTE_CQ_DATA)
sock_tx_ctx_write(tx_ctx, &msg->data, sizeof(uint64_t));
- src_len = 0;
+ src_len = dst_len = 0;
if (flags & FI_INJECT) {
for (i = 0; i < msg->iov_count; i++) {
sock_tx_ctx_write(tx_ctx, msg->msg_iov[i].addr,
msg->msg_iov[i].count * datatype_sz);
src_len += (msg->msg_iov[i].count * datatype_sz);
}
+ for (i = 0; i < compare_count; i++) {
+ sock_tx_ctx_write(tx_ctx, comparev[i].addr,
+ comparev[i].count * datatype_sz);
+ dst_len += comparev[i].count * datatype_sz;
+ }
} else {
for (i = 0; i < msg->iov_count; i++) {
tx_iov.ioc.addr = (uintptr_t) msg->msg_iov[i].addr;
@@ -171,10 +185,22 @@ ssize_t sock_ep_tx_atomic(struct fid_ep *ep,
sock_tx_ctx_write(tx_ctx, &tx_iov, sizeof(tx_iov));
src_len += (tx_iov.ioc.count * datatype_sz);
}
+ for (i = 0; i < compare_count; i++) {
+ tx_iov.ioc.addr = (uintptr_t) comparev[i].addr;
+ tx_iov.ioc.count = comparev[i].count;
+ sock_tx_ctx_write(tx_ctx, &tx_iov, sizeof(tx_iov));
+ dst_len += (tx_iov.ioc.count * datatype_sz);
+ }
}
#if ENABLE_DEBUG
- if (src_len > SOCK_EP_MAX_ATOMIC_SZ) {
+ if ((src_len > SOCK_EP_MAX_ATOMIC_SZ) ||
+ (dst_len > SOCK_EP_MAX_ATOMIC_SZ)) {
+ SOCK_LOG_ERROR("Max atomic operation size exceeded!\n");
+ ret = -FI_EINVAL;
+ goto err;
+ } else if (compare_count && (dst_len != src_len)) {
+ SOCK_LOG_ERROR("Buffer length mismatch\n");
ret = -FI_EINVAL;
goto err;
}
@@ -189,7 +215,7 @@ ssize_t sock_ep_tx_atomic(struct fid_ep *ep,
dst_len += (tx_iov.ioc.count * datatype_sz);
}
- if (msg->iov_count && dst_len != src_len) {
+ if (msg->iov_count && (dst_len != src_len)) {
SOCK_LOG_ERROR("Buffer length mismatch\n");
ret = -FI_EINVAL;
goto err;
@@ -213,22 +239,6 @@ ssize_t sock_ep_tx_atomic(struct fid_ep *ep,
}
#endif
- dst_len = 0;
- for (i = 0; i < compare_count; i++) {
- tx_iov.ioc.addr = (uintptr_t) comparev[i].addr;
- tx_iov.ioc.count = comparev[i].count;
- sock_tx_ctx_write(tx_ctx, &tx_iov, sizeof(tx_iov));
- dst_len += (tx_iov.ioc.count * datatype_sz);
- }
-
-#if ENABLE_DEBUG
- if (compare_count && (dst_len != src_len)) {
- SOCK_LOG_ERROR("Buffer length mismatch\n");
- ret = -FI_EINVAL;
- goto err;
- }
-#endif
-
sock_tx_ctx_commit(tx_ctx);
return 0;
@@ -237,7 +247,6 @@ err:
return ret;
}
-
static ssize_t sock_ep_atomic_writemsg(struct fid_ep *ep,
const struct fi_msg_atomic *msg, uint64_t flags)
{
diff --git a/prov/sockets/src/sock_av.c b/prov/sockets/src/sock_av.c
index 0b2d048..9fdbdf8 100644
--- a/prov/sockets/src/sock_av.c
+++ b/prov/sockets/src/sock_av.c
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2014 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016, Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -49,6 +50,9 @@
#include "sock.h"
#include "sock_util.h"
+#include "fi_osd.h"
+#include "fi_util.h"
+
#define SOCK_LOG_DBG(...) _SOCK_LOG_DBG(FI_LOG_AV, __VA_ARGS__)
#define SOCK_LOG_ERROR(...) _SOCK_LOG_ERROR(FI_LOG_AV, __VA_ARGS__)
@@ -57,21 +61,17 @@
count * sizeof(struct sock_av_addr))
#define SOCK_IS_SHARED_AV(av_name) ((av_name) ? 1 : 0)
-int sock_compare_addr(struct sockaddr_in *addr1,
- struct sockaddr_in *addr2)
-{
- return ((addr1->sin_addr.s_addr == addr2->sin_addr.s_addr) &&
- (addr1->sin_port == addr2->sin_port));
-}
-
int sock_av_get_addr_index(struct sock_av *av, struct sockaddr_in *addr)
{
int i;
struct sock_av_addr *av_addr;
- for (i = 0; i < av->table_hdr->stored; i++) {
+ for (i = 0; i < av->table_hdr->size; i++) {
av_addr = &av->table[i];
- if (sock_compare_addr(addr, (struct sockaddr_in *)&av_addr->addr))
+ if (!av_addr->valid)
+ continue;
+
+ if (ofi_equals_sockaddr(addr, (struct sockaddr_in *)&av_addr->addr))
return i;
}
SOCK_LOG_DBG("failed to get index in AV\n");
@@ -87,8 +87,8 @@ int sock_av_compare_addr(struct sock_av *av,
index1 = ((uint64_t)addr1 & av->mask);
index2 = ((uint64_t)addr2 & av->mask);
- if (index1 >= av->table_hdr->stored || index1 < 0 ||
- index2 >= av->table_hdr->stored || index2 < 0) {
+ if (index1 >= av->table_hdr->size || index1 < 0 ||
+ index2 >= av->table_hdr->size || index2 < 0) {
SOCK_LOG_ERROR("requested rank is larger than av table\n");
return -1;
}
@@ -138,24 +138,63 @@ static void sock_update_av_table(struct sock_av *_av, size_t count)
sizeof(struct sock_av_table_hdr));
}
+static int sock_resize_av_table(struct sock_av *av)
+{
+ void *new_addr;
+ size_t new_count, table_sz, old_sz;
+
+ new_count = av->table_hdr->size * 2;
+ table_sz = SOCK_AV_TABLE_SZ(new_count, av->attr.name);
+ old_sz = SOCK_AV_TABLE_SZ(av->table_hdr->size, av->attr.name);
+
+ if (av->attr.name) {
+ new_addr = sock_mremap(av->table_hdr, old_sz, table_sz);
+ if (new_addr == MAP_FAILED)
+ return -1;
+
+ av->idx_arr[av->table_hdr->stored] = av->table_hdr->stored;
+ } else {
+ new_addr = realloc(av->table_hdr, table_sz);
+ if (!new_addr)
+ return -1;
+ }
+
+ av->table_hdr = new_addr;
+ av->table_hdr->size = new_count;
+ sock_update_av_table(av, new_count);
+
+ return 0;
+}
+
+static int sock_av_get_next_index(struct sock_av *av)
+{
+ int i;
+
+ for (i = 0; i < av->table_hdr->size; i++) {
+ if (!av->table[i].valid)
+ return i;
+ }
+
+ return -1;
+}
+
static int sock_check_table_in(struct sock_av *_av, struct sockaddr_in *addr,
fi_addr_t *fi_addr, int count, uint64_t flags,
- void *context, int index)
+ void *context)
{
- void *new_addr;
int i, j, ret = 0;
char sa_ip[INET_ADDRSTRLEN];
struct sock_av_addr *av_addr;
- size_t new_count, table_sz, old_sz;
+ int index;
if ((_av->attr.flags & FI_EVENT) && !_av->eq)
return -FI_ENOEQ;
if (_av->attr.flags & FI_READ) {
for (i = 0; i < count; i++) {
- for (j = 0; j < _av->table_hdr->stored; j++) {
-
- if (!sock_av_is_valid_address(&addr[i])) {
+ for (j = 0; j < _av->table_hdr->size; j++) {
+ if (_av->table[j].valid &&
+ !sock_av_is_valid_address(&addr[i])) {
if (fi_addr)
fi_addr[i] = FI_ADDR_NOTAVAIL;
sock_av_report_error(_av, context, i,
@@ -178,65 +217,38 @@ static int sock_check_table_in(struct sock_av *_av, struct sockaddr_in *addr,
}
for (i = 0, ret = 0; i < count; i++) {
- if (_av->table_hdr->stored == _av->table_hdr->size) {
- if (_av->table_hdr->req_sz) {
- if (fi_addr)
- fi_addr[i] = FI_ADDR_NOTAVAIL;
- sock_av_report_error(_av, context, i, FI_ENOSPC);
- SOCK_LOG_ERROR("Cannot insert to AV table\n");
- continue;
- } else {
- new_count = _av->table_hdr->size * 2;
- table_sz = SOCK_AV_TABLE_SZ(new_count, _av->attr.name);
- old_sz = SOCK_AV_TABLE_SZ(_av->table_hdr->size, _av->attr.name);
-
- if (_av->attr.name) {
- new_addr = sock_mremap(_av->table_hdr,
- old_sz, table_sz);
- if (new_addr == ((void *) -1)) {
- if (fi_addr)
- fi_addr[i] = FI_ADDR_NOTAVAIL;
- sock_av_report_error(_av,
- context, i, FI_ENOMEM);
- continue;
- }
- _av->idx_arr[_av->table_hdr->stored] = _av->table_hdr->stored;
- } else {
- new_addr = realloc(_av->table_hdr,
- table_sz);
- if (!new_addr) {
- if (fi_addr)
- fi_addr[i] = FI_ADDR_NOTAVAIL;
- sock_av_report_error(_av,
- context, i, FI_ENOMEM);
- continue;
- }
- }
- _av->table_hdr = new_addr;
- _av->table_hdr->size = new_count;
- sock_update_av_table(_av, new_count);
- }
- }
-
if (!sock_av_is_valid_address(&addr[i])) {
if (fi_addr)
fi_addr[i] = FI_ADDR_NOTAVAIL;
sock_av_report_error(_av, context, i, FI_EINVAL);
continue;
}
+ if (_av->table_hdr->stored == _av->table_hdr->size) {
+ index = sock_av_get_next_index(_av);
+ if (index < 0) {
+ if (sock_resize_av_table(_av)) {
+ if (fi_addr)
+ fi_addr[i] = FI_ADDR_NOTAVAIL;
+ sock_av_report_error(_av, context, i, FI_ENOMEM);
+ continue;
+ }
+ index = _av->table_hdr->stored++;
+ }
+ } else {
+ index = _av->table_hdr->stored++;
+ }
- av_addr = &_av->table[_av->table_hdr->stored];
+ av_addr = &_av->table[index];
memcpy(sa_ip, inet_ntoa((&addr[i])->sin_addr), INET_ADDRSTRLEN);
- SOCK_LOG_DBG("AV-INSERT:dst_addr: family: %d, IP is %s, port: %d\n",
+ SOCK_LOG_DBG("AV-INSERT: dst_addr family: %d, IP %s, port: %d\n",
((struct sockaddr_in *)&addr[i])->sin_family,
sa_ip, ntohs(((struct sockaddr_in *)&addr[i])->sin_port));
memcpy(&av_addr->addr, &addr[i], sizeof(struct sockaddr_in));
if (fi_addr)
- fi_addr[i] = (fi_addr_t)_av->table_hdr->stored;
+ fi_addr[i] = (fi_addr_t)index;
av_addr->valid = 1;
- _av->table_hdr->stored++;
ret++;
}
sock_av_report_success(_av, context, ret, flags);
@@ -249,7 +261,7 @@ static int sock_av_insert(struct fid_av *av, const void *addr, size_t count,
struct sock_av *_av;
_av = container_of(av, struct sock_av, av_fid);
return sock_check_table_in(_av, (struct sockaddr_in *)addr,
- fi_addr, count, flags, context, 0);
+ fi_addr, count, flags, context);
}
static int sock_av_lookup(struct fid_av *av, fi_addr_t fi_addr, void *addr,
@@ -261,7 +273,7 @@ static int sock_av_lookup(struct fid_av *av, fi_addr_t fi_addr, void *addr,
_av = container_of(av, struct sock_av, av_fid);
index = ((uint64_t)fi_addr & _av->mask);
- if (index >= _av->table_hdr->stored || index < 0) {
+ if (index >= _av->table_hdr->size || index < 0) {
SOCK_LOG_ERROR("requested address not inserted\n");
return -EINVAL;
}
@@ -274,7 +286,7 @@ static int sock_av_lookup(struct fid_av *av, fi_addr_t fi_addr, void *addr,
static int _sock_av_insertsvc(struct fid_av *av, const char *node,
const char *service, fi_addr_t *fi_addr,
- uint64_t flags, void *context, int index)
+ uint64_t flags, void *context)
{
int ret;
struct addrinfo sock_hints;
@@ -296,7 +308,7 @@ static int _sock_av_insertsvc(struct fid_av *av, const char *node,
}
ret = sock_check_table_in(_av, (struct sockaddr_in *)result->ai_addr,
- fi_addr, 1, flags, context, index);
+ fi_addr, 1, flags, context);
freeaddrinfo(result);
return ret;
}
@@ -310,7 +322,7 @@ static int sock_av_insertsvc(struct fid_av *av, const char *node,
return -FI_EINVAL;
}
- return _sock_av_insertsvc(av, node, service, fi_addr, flags, context, 0);
+ return _sock_av_insertsvc(av, node, service, fi_addr, flags, context);
}
static int sock_av_insertsym(struct fid_av *av, const char *node, size_t nodecnt,
@@ -350,7 +362,7 @@ static int sock_av_insertsym(struct fid_av *av, const char *node, size_t nodecnt
len2 = snprintf(tmp_port, FI_NAME_MAX, "%d",
var_port + j);
if (len1 > 0 && len1 < FI_NAME_MAX && len2 > 0 && len2 < FI_NAME_MAX) {
- ret = _sock_av_insertsvc(av, tmp_host, tmp_port, fi_addr, flags, context, i * nodecnt + j);
+ ret = _sock_av_insertsvc(av, tmp_host, tmp_port, fi_addr, flags, context);
if (ret == 1)
success++;
else
@@ -371,12 +383,35 @@ static int sock_av_remove(struct fid_av *av, fi_addr_t *fi_addr, size_t count,
int i;
struct sock_av *_av;
struct sock_av_addr *av_addr;
+ struct dlist_entry *item;
+ struct fid_list_entry *fid_entry;
+ struct sock_ep *sock_ep;
+ struct sock_conn *conn;
+ uint16_t idx;
+
_av = container_of(av, struct sock_av, av_fid);
+ fastlock_acquire(&_av->list_lock);
+ dlist_foreach(&_av->ep_list, item) {
+ fid_entry = container_of(item, struct fid_list_entry, entry);
+ sock_ep = container_of(fid_entry->fid, struct sock_ep, ep.fid);
+ fastlock_acquire(&sock_ep->attr->cmap.lock);
+ for (i = 0; i < count; i++) {
+ idx = fi_addr[i] & sock_ep->attr->av->mask;
+ conn = idm_lookup(&sock_ep->attr->av_idm, idx);
+ if (conn && conn->sock_fd != -1) {
+ sock_ep_remove_conn(sock_ep->attr, conn);
+ idm_clear(&sock_ep->attr->av_idm, idx);
+ }
+ }
+ fastlock_release(&sock_ep->attr->cmap.lock);
+ }
+ fastlock_release(&_av->list_lock);
for (i = 0; i < count; i++) {
av_addr = &_av->table[fi_addr[i]];
av_addr->valid = 0;
}
+
return 0;
}
@@ -389,7 +424,7 @@ static const char *sock_av_straddr(struct fid_av *av, const void *addr,
sin = addr;
size = snprintf(straddr, sizeof(straddr), "%s:%d",
- inet_ntoa(sin->sin_addr), sin->sin_port);
+ inet_ntoa(sin->sin_addr), ntohs(sin->sin_port));
snprintf(buf, *len, "%s", straddr);
*len = size + 1;
return buf;
@@ -417,18 +452,16 @@ static int sock_av_close(struct fid *fid)
if (atomic_get(&av->ref))
return -FI_EBUSY;
- if (!av->name)
+ if (!av->shared)
free(av->table_hdr);
else {
- shm_unlink(av->name);
- free(av->name);
- ret = munmap(av->table_hdr, SOCK_AV_TABLE_SZ(av->attr.count, av->name));
+ ret = ofi_shm_unmap(&av->shm);
if (ret)
- SOCK_LOG_ERROR("munmap failed: %s\n", strerror(errno));
- close(av->shared_fd);
+ SOCK_LOG_ERROR("unmap failed: %s\n", strerror(errno));
}
atomic_dec(&av->domain->ref);
+ fastlock_destroy(&av->list_lock);
free(av);
return 0;
}
@@ -488,9 +521,7 @@ int sock_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
int ret = 0;
struct sock_domain *dom;
struct sock_av *_av;
- size_t table_sz, i;
- int flags = O_RDWR;
- struct stat mapstat;
+ size_t table_sz;
if (!attr || sock_verify_av_attr(attr))
return -FI_EINVAL;
@@ -512,46 +543,15 @@ int sock_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
table_sz = SOCK_AV_TABLE_SZ(_av->attr.count, attr->name);
if (attr->name) {
- _av->name = calloc(1, strlen(attr->name) + 1);
- if (!_av->name) {
- ret = -FI_ENOMEM;
- goto err1;
- }
- sprintf(_av->name, "/%s", attr->name);
- if (!(attr->flags & FI_READ))
- flags |= O_CREAT;
-
- for (i = 0; i < strlen(_av->name); i++)
- if (_av->name[i] == ' ')
- _av->name[i] = '_';
+ ret = ofi_shm_map(&_av->shm, attr->name, table_sz,
+ attr->flags & FI_READ, (void**)&_av->table_hdr);
- SOCK_LOG_DBG("Creating shm segment :%s (size: %lu)\n",
- _av->name, table_sz);
-
- _av->shared_fd = shm_open(_av->name, flags, S_IRUSR | S_IWUSR);
- if (_av->shared_fd < 0) {
- SOCK_LOG_ERROR("shm_open failed\n");
+ if (ret || _av->table_hdr == MAP_FAILED) {
+ SOCK_LOG_ERROR("map failed\n");
ret = -FI_EINVAL;
- goto err2;
- }
-
- if (fstat(_av->shared_fd, &mapstat)) {
- SOCK_LOG_ERROR("failed to do fstat: %s\n", strerror(errno));
- goto err_shm;
- }
-
- if (mapstat.st_size == 0) {
- if (ftruncate(_av->shared_fd, table_sz)) {
- SOCK_LOG_ERROR("ftruncate failed: %s\n", strerror(errno));
- goto err_shm;
- }
- } else if (mapstat.st_size < table_sz) {
- SOCK_LOG_ERROR("shm file too small\n");
- goto err_shm;
+ goto err;
}
- _av->table_hdr = mmap(NULL, table_sz, PROT_READ | PROT_WRITE,
- MAP_SHARED, _av->shared_fd, 0);
_av->idx_arr = (uint64_t *)(_av->table_hdr + 1);
_av->attr.map_addr = _av->idx_arr;
attr->map_addr = _av->attr.map_addr;
@@ -566,21 +566,14 @@ int sock_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
_av->table_hdr->size = _av->attr.count;
_av->table_hdr->stored = 0;
}
-
- if (_av->table_hdr == MAP_FAILED) {
- SOCK_LOG_ERROR("mmap failed\n");
- shm_unlink(_av->name);
- ret = -FI_EINVAL;
- goto err2;
- }
+ _av->shared = 1;
} else {
_av->table_hdr = calloc(1, table_sz);
if (!_av->table_hdr) {
ret = -FI_ENOMEM;
- goto err3;
+ goto err;
}
_av->table_hdr->size = _av->attr.count;
- _av->table_hdr->req_sz = attr->count;
}
sock_update_av_table(_av, _av->attr.count);
@@ -597,7 +590,7 @@ int sock_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
break;
default:
ret = -FI_EINVAL;
- goto err3;
+ goto err2;
}
atomic_initialize(&_av->ref, 0);
@@ -610,24 +603,24 @@ int sock_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
default:
SOCK_LOG_ERROR("Invalid address format: only IPv4 supported\n");
ret = -FI_EINVAL;
- goto err3;
+ goto err2;
}
+ dlist_init(&_av->ep_list);
+ fastlock_init(&_av->list_lock);
_av->rx_ctx_bits = attr->rx_ctx_bits;
_av->mask = attr->rx_ctx_bits ?
((uint64_t)1 << (64 - attr->rx_ctx_bits)) - 1 : ~0;
*av = &_av->av_fid;
return 0;
-err_shm:
- shm_unlink(_av->name);
- ret = -FI_EINVAL;
- goto err2;
-err3:
- free(_av->table_hdr);
err2:
- free(_av->name);
-err1:
+ if(attr->name) {
+ ofi_shm_unmap(&_av->shm);
+ } else {
+ if(_av->table_hdr != MAP_FAILED)
+ free(_av->table_hdr);
+ }
+err:
free(_av);
-
return ret;
}
diff --git a/prov/sockets/src/sock_cntr.c b/prov/sockets/src/sock_cntr.c
index 4be96a3..5a32c63 100644
--- a/prov/sockets/src/sock_cntr.c
+++ b/prov/sockets/src/sock_cntr.c
@@ -41,6 +41,8 @@
#include "sock.h"
#include "sock_util.h"
+#include <fi_util.h>
+
#define SOCK_LOG_DBG(...) _SOCK_LOG_DBG(FI_LOG_EP_DATA, __VA_ARGS__)
#define SOCK_LOG_ERROR(...) _SOCK_LOG_ERROR(FI_LOG_EP_DATA, __VA_ARGS__)
@@ -51,29 +53,69 @@ const struct fi_cntr_attr sock_cntr_attr = {
.flags = 0,
};
+void sock_cntr_add_tx_ctx(struct sock_cntr *cntr, struct sock_tx_ctx *tx_ctx)
+{
+ int ret;
+ struct fid *fid = &tx_ctx->fid.ctx.fid;
+ ret = fid_list_insert(&cntr->tx_list, &cntr->list_lock, fid);
+ if (ret)
+ SOCK_LOG_ERROR("Error in adding ctx to progress list\n");
+}
+
+void sock_cntr_remove_tx_ctx(struct sock_cntr *cntr, struct sock_tx_ctx *tx_ctx)
+{
+ struct fid *fid = &tx_ctx->fid.ctx.fid;
+ fid_list_remove(&cntr->tx_list, &cntr->list_lock, fid);
+}
+
+void sock_cntr_add_rx_ctx(struct sock_cntr *cntr, struct sock_rx_ctx *rx_ctx)
+{
+ int ret;
+ struct fid *fid = &rx_ctx->ctx.fid;
+ ret = fid_list_insert(&cntr->rx_list, &cntr->list_lock, fid);
+ if (ret)
+ SOCK_LOG_ERROR("Error in adding ctx to progress list\n");
+}
+
+void sock_cntr_remove_rx_ctx(struct sock_cntr *cntr, struct sock_rx_ctx *rx_ctx)
+{
+ struct fid *fid = &rx_ctx->ctx.fid;
+ fid_list_remove(&cntr->rx_list, &cntr->list_lock, fid);
+}
+
int sock_cntr_progress(struct sock_cntr *cntr)
{
struct sock_tx_ctx *tx_ctx;
struct sock_rx_ctx *rx_ctx;
struct dlist_entry *entry;
+ struct fid_list_entry *fid_entry;
+
if (cntr->domain->progress_mode == FI_PROGRESS_AUTO)
return 0;
fastlock_acquire(&cntr->list_lock);
for (entry = cntr->tx_list.next; entry != &cntr->tx_list;
entry = entry->next) {
- tx_ctx = container_of(entry, struct sock_tx_ctx, cntr_entry);
- sock_pe_progress_tx_ctx(cntr->domain->pe, tx_ctx);
+ fid_entry = container_of(entry, struct fid_list_entry, entry);
+ tx_ctx = container_of(fid_entry->fid, struct sock_tx_ctx, fid.ctx.fid);
+ if (tx_ctx->use_shared)
+ sock_pe_progress_tx_ctx(cntr->domain->pe, tx_ctx->stx_ctx);
+ else
+ sock_pe_progress_tx_ctx(cntr->domain->pe, tx_ctx);
}
for (entry = cntr->rx_list.next; entry != &cntr->rx_list;
entry = entry->next) {
- rx_ctx = container_of(entry, struct sock_rx_ctx, cntr_entry);
- sock_pe_progress_rx_ctx(cntr->domain->pe, rx_ctx);
+ fid_entry = container_of(entry, struct fid_list_entry, entry);
+ rx_ctx = container_of(fid_entry->fid, struct sock_rx_ctx, ctx.fid);
+ if (rx_ctx->use_shared)
+ sock_pe_progress_rx_ctx(cntr->domain->pe, rx_ctx->srx_ctx);
+ else
+ sock_pe_progress_rx_ctx(cntr->domain->pe, rx_ctx);
}
- fastlock_release(&cntr->list_lock);
+ fastlock_release(&cntr->list_lock);
return 0;
}
@@ -118,14 +160,14 @@ void sock_cntr_check_trigger_list(struct sock_cntr *cntr)
case SOCK_OP_WRITE:
ret = sock_ep_rma_writemsg(trigger->ep,
- &trigger->op.rma.msg,
- trigger->flags & ~FI_TRIGGER);
+ &trigger->op.rma.msg,
+ trigger->flags & ~FI_TRIGGER);
break;
case SOCK_OP_READ:
ret = sock_ep_rma_readmsg(trigger->ep,
- &trigger->op.rma.msg,
- trigger->flags & ~FI_TRIGGER);
+ &trigger->op.rma.msg,
+ trigger->flags & ~FI_TRIGGER);
break;
case SOCK_OP_ATOMIC:
@@ -168,8 +210,8 @@ void sock_cntr_inc(struct sock_cntr *cntr)
{
pthread_mutex_lock(&cntr->mut);
atomic_inc(&cntr->value);
- if (atomic_get(&cntr->value) >= atomic_get(&cntr->threshold))
- pthread_cond_signal(&cntr->cond);
+ if(atomic_get(&cntr->num_waiting))
+ pthread_cond_broadcast(&cntr->cond);
pthread_mutex_unlock(&cntr->mut);
sock_cntr_check_trigger_list(cntr);
}
@@ -186,13 +228,15 @@ void sock_cntr_err_inc(struct sock_cntr *cntr)
static int sock_cntr_add(struct fid_cntr *cntr, uint64_t value)
{
+ uint64_t new_val;
struct sock_cntr *_cntr;
_cntr = container_of(cntr, struct sock_cntr, cntr_fid);
pthread_mutex_lock(&_cntr->mut);
- atomic_set(&_cntr->value, atomic_get(&_cntr->value) + value);
- if (atomic_get(&_cntr->value) >= atomic_get(&_cntr->threshold))
- pthread_cond_signal(&_cntr->cond);
+ new_val = atomic_add(&_cntr->value, value);
+ atomic_set(&_cntr->last_read_val, new_val);
+ if(atomic_get(&_cntr->num_waiting))
+ pthread_cond_broadcast(&_cntr->cond);
pthread_mutex_unlock(&_cntr->mut);
sock_cntr_check_trigger_list(_cntr);
return 0;
@@ -200,23 +244,25 @@ static int sock_cntr_add(struct fid_cntr *cntr, uint64_t value)
static int sock_cntr_set(struct fid_cntr *cntr, uint64_t value)
{
+ uint64_t new_val;
struct sock_cntr *_cntr;
_cntr = container_of(cntr, struct sock_cntr, cntr_fid);
pthread_mutex_lock(&_cntr->mut);
- atomic_set(&_cntr->value, value);
- if (atomic_get(&_cntr->value) >= atomic_get(&_cntr->threshold))
- pthread_cond_signal(&_cntr->cond);
+ new_val = atomic_set(&_cntr->value, value);
+ atomic_set(&_cntr->last_read_val, new_val);
+ if(atomic_get(&_cntr->num_waiting))
+ pthread_cond_broadcast(&_cntr->cond);
pthread_mutex_unlock(&_cntr->mut);
sock_cntr_check_trigger_list(_cntr);
return 0;
}
static int sock_cntr_wait(struct fid_cntr *cntr, uint64_t threshold,
- int timeout)
+ int timeout)
{
- int ret = 0;
- uint64_t start_ms = 0, end_ms = 0;
+ int last_read, ret = 0;
+ uint64_t start_ms = 0, end_ms = 0, remaining_ms = 0;
struct sock_cntr *_cntr;
_cntr = container_of(cntr, struct sock_cntr, cntr_fid);
@@ -232,38 +278,47 @@ static int sock_cntr_wait(struct fid_cntr *cntr, uint64_t threshold,
goto out;
}
- if (_cntr->is_waiting) {
- ret = -FI_EBUSY;
- goto out;
+ atomic_inc(&_cntr->num_waiting);
+
+ if (timeout >= 0) {
+ start_ms = fi_gettime_ms();
+ end_ms = start_ms + timeout;
}
- _cntr->is_waiting = 1;
- atomic_set(&_cntr->threshold, threshold);
+ last_read = atomic_get(&_cntr->value);
+ remaining_ms = timeout;
+
+ while (!ret && last_read < threshold) {
+ if(_cntr->domain->progress_mode == FI_PROGRESS_MANUAL) {
+ pthread_mutex_unlock(&_cntr->mut);
+
+ ret = sock_cntr_progress(_cntr);
+
+ pthread_mutex_lock(&_cntr->mut);
- if (_cntr->domain->progress_mode == FI_PROGRESS_MANUAL) {
- pthread_mutex_unlock(&_cntr->mut);
- if (timeout >= 0) {
- start_ms = fi_gettime_ms();
- end_ms = start_ms + timeout;
+ } else {
+ ret = fi_wait_cond(&_cntr->cond, &_cntr->mut, remaining_ms);
}
- while (atomic_get(&_cntr->value) < threshold) {
- sock_cntr_progress(_cntr);
- if (timeout >= 0 && fi_gettime_ms() >= end_ms) {
- ret = FI_ETIMEDOUT;
+ uint64_t curr_ms = fi_gettime_ms();
+ if(timeout >= 0) {
+ if(curr_ms >= end_ms) {
+ ret = -FI_ETIMEDOUT;
break;
+ } else {
+ remaining_ms = end_ms - curr_ms;
}
}
- pthread_mutex_lock(&_cntr->mut);
- } else {
- ret = fi_wait_cond(&_cntr->cond, &_cntr->mut, timeout);
+
+ last_read = atomic_get(&_cntr->value);
}
- _cntr->is_waiting = 0;
- atomic_set(&_cntr->threshold, ~0);
+ atomic_set(&_cntr->last_read_val, last_read);
+ atomic_dec(&_cntr->num_waiting);
+
pthread_mutex_unlock(&_cntr->mut);
sock_cntr_check_trigger_list(_cntr);
- return (_cntr->err_flag) ? -FI_EAVAIL : -ret;
+ return (_cntr->err_flag) ? -FI_EAVAIL : ret;
out:
pthread_mutex_unlock(&_cntr->mut);
@@ -467,7 +522,8 @@ int sock_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
atomic_initialize(&_cntr->err_cnt, 0);
atomic_initialize(&_cntr->value, 0);
- atomic_initialize(&_cntr->threshold, ~0);
+ atomic_initialize(&_cntr->last_read_val, 0);
+ atomic_initialize(&_cntr->num_waiting, 0);
dlist_init(&_cntr->tx_list);
dlist_init(&_cntr->rx_list);
diff --git a/prov/sockets/src/sock_comm.c b/prov/sockets/src/sock_comm.c
index b6430a8..b930e8e 100644
--- a/prov/sockets/src/sock_comm.c
+++ b/prov/sockets/src/sock_comm.c
@@ -49,13 +49,12 @@ static ssize_t sock_comm_send_socket(struct sock_conn *conn,
{
ssize_t ret;
- ret = write(conn->sock_fd, buf, len);
+ ret = ofi_write_socket(conn->sock_fd, buf, len);
if (ret < 0) {
- if (errno == EAGAIN || errno == EWOULDBLOCK) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK)
ret = 0;
- } else {
- SOCK_LOG_DBG("write %s\n", strerror(errno));
- }
+ else
+ SOCK_LOG_DBG("write error: %s\n", strerror(errno));
}
if (ret > 0)
SOCK_LOG_DBG("wrote to network: %lu\n", ret);
@@ -72,14 +71,14 @@ ssize_t sock_comm_flush(struct sock_pe_entry *pe_entry)
(pe_entry->comm_buf.rcnt & pe_entry->comm_buf.size_mask);
xfer_len = MIN(len, endlen);
- ret1 = sock_comm_send_socket(pe_entry->conn, pe_entry->comm_buf.buf +
+ ret1 = sock_comm_send_socket(pe_entry->conn, (char*)pe_entry->comm_buf.buf +
(pe_entry->comm_buf.rcnt & pe_entry->comm_buf.size_mask),
xfer_len);
if (ret1 > 0)
pe_entry->comm_buf.rcnt += ret1;
if (ret1 == xfer_len && xfer_len < len) {
- ret2 = sock_comm_send_socket(pe_entry->conn, pe_entry->comm_buf.buf +
+ ret2 = sock_comm_send_socket(pe_entry->conn, (char*)pe_entry->comm_buf.buf +
(pe_entry->comm_buf.rcnt & pe_entry->comm_buf.size_mask),
len - xfer_len);
if (ret2 > 0)
@@ -129,10 +128,9 @@ static ssize_t sock_comm_recv_socket(struct sock_conn *conn,
ssize_t ret;
ret = recv(conn->sock_fd, buf, len, 0);
if (ret == 0) {
- conn->disconnected = 1;
+ conn->connected = 0;
SOCK_LOG_DBG("Disconnected: %s:%d\n", inet_ntoa(conn->addr.sin_addr),
ntohs(conn->addr.sin_port));
-
return ret;
}
@@ -187,7 +185,7 @@ ssize_t sock_comm_peek(struct sock_conn *conn, void *buf, size_t len)
ssize_t ret;
ret = recv(conn->sock_fd, buf, len, MSG_PEEK);
if (ret == 0) {
- conn->disconnected = 1;
+ conn->connected = 0;
SOCK_LOG_DBG("Disconnected\n");
return ret;
}
@@ -215,3 +213,8 @@ ssize_t sock_comm_discard(struct sock_pe_entry *pe_entry, size_t len)
free(buf);
return ret;
}
+
+int sock_comm_is_disconnected(struct sock_pe_entry *pe_entry)
+{
+ return (rbempty(&pe_entry->comm_buf) && !pe_entry->conn->connected);
+}
diff --git a/prov/sockets/src/sock_conn.c b/prov/sockets/src/sock_conn.c
index 96d5b42..9ed67d2 100644
--- a/prov/sockets/src/sock_conn.c
+++ b/prov/sockets/src/sock_conn.c
@@ -52,11 +52,12 @@
#include "sock.h"
#include "sock_util.h"
+#include "fi_file.h"
#define SOCK_LOG_DBG(...) _SOCK_LOG_DBG(FI_LOG_EP_CTRL, __VA_ARGS__)
#define SOCK_LOG_ERROR(...) _SOCK_LOG_ERROR(FI_LOG_EP_CTRL, __VA_ARGS__)
-ssize_t sock_conn_send_src_addr(struct sock_ep *sock_ep, struct sock_tx_ctx *tx_ctx,
+ssize_t sock_conn_send_src_addr(struct sock_ep_attr *ep_attr, struct sock_tx_ctx *tx_ctx,
struct sock_conn *conn)
{
int ret;
@@ -77,9 +78,9 @@ ssize_t sock_conn_send_src_addr(struct sock_ep *sock_ep, struct sock_tx_ctx *tx_
goto err;
}
- sock_tx_ctx_write_op_send(tx_ctx, &tx_op, 0, (uintptr_t) NULL, 0, 0, sock_ep,
- conn);
- sock_tx_ctx_write(tx_ctx, sock_ep->src_addr, sizeof(struct sockaddr_in));
+ sock_tx_ctx_write_op_send(tx_ctx, &tx_op, 0, (uintptr_t) NULL, 0, 0,
+ ep_attr, conn);
+ sock_tx_ctx_write(tx_ctx, ep_attr->src_addr, sizeof(struct sockaddr_in));
sock_tx_ctx_commit(tx_ctx);
conn->address_published = 1;
return 0;
@@ -91,7 +92,7 @@ err:
int sock_conn_map_init(struct sock_ep *ep, int init_size)
{
- struct sock_conn_map *map = &ep->cmap;
+ struct sock_conn_map *map = &ep->attr->cmap;
map->table = calloc(init_size, sizeof(*map->table));
if (!map->table)
return -FI_ENOMEM;
@@ -124,12 +125,15 @@ static int sock_conn_map_increase(struct sock_conn_map *map, int new_size)
return 0;
}
-void sock_conn_map_destroy(struct sock_conn_map *cmap)
+void sock_conn_map_destroy(struct sock_ep_attr *ep_attr)
{
int i;
-
+ struct sock_conn_map *cmap = &ep_attr->cmap;
for (i = 0; i < cmap->used; i++) {
- close(cmap->table[i].sock_fd);
+ if (cmap->table[i].sock_fd != -1) {
+ sock_pe_poll_del(ep_attr->domain->pe, cmap->table[i].sock_fd);
+ sock_conn_release_entry(cmap, &cmap->table[i]);
+ }
}
free(cmap->table);
cmap->table = NULL;
@@ -138,56 +142,77 @@ void sock_conn_map_destroy(struct sock_conn_map *cmap)
fastlock_destroy(&cmap->lock);
}
-static struct sock_conn *sock_conn_map_insert(struct sock_ep *ep,
+void sock_conn_release_entry(struct sock_conn_map *map, struct sock_conn *conn)
+{
+ sock_epoll_del(&map->epoll_set, conn->sock_fd);
+ ofi_close_socket(conn->sock_fd);
+
+ conn->address_published = 0;
+ conn->connected = 0;
+ conn->sock_fd = -1;
+}
+
+static int sock_conn_get_next_index(struct sock_conn_map *map)
+{
+ int i;
+ for (i = 0; i < map->size; i++) {
+ if (map->table[i].sock_fd == -1)
+ return i;
+ }
+ return -1;
+}
+
+static struct sock_conn *sock_conn_map_insert(struct sock_ep_attr *ep_attr,
struct sockaddr_in *addr, int conn_fd,
int addr_published)
{
int index;
- struct sock_conn_map *map = &ep->cmap;
+ struct sock_conn_map *map = &ep_attr->cmap;
if (map->size == map->used) {
- if (sock_conn_map_increase(map, map->size * 2))
- return NULL;
+ index = sock_conn_get_next_index(map);
+ if (index < 0) {
+ if (sock_conn_map_increase(map, map->size * 2))
+ return NULL;
+ index = map->used;
+ map->used++;
+ }
+ } else {
+ index = map->used;
+ map->used++;
}
- index = map->used;
- map->used++;
-
+ map->table[index].connected = 1;
map->table[index].addr = *addr;
map->table[index].sock_fd = conn_fd;
- map->table[index].ep = ep;
+ map->table[index].ep_attr = ep_attr;
sock_set_sockopts(conn_fd);
- fastlock_acquire(&ep->lock);
- dlist_insert_tail(&map->table[index].ep_entry, &ep->conn_list);
- fastlock_release(&ep->lock);
- if (idm_set(&ep->conn_idm, conn_fd, &map->table[index]) < 0)
- SOCK_LOG_ERROR("idm_set failed\n");
+ if (idm_set(&ep_attr->conn_idm, conn_fd, &map->table[index]) < 0)
+ SOCK_LOG_ERROR("idm_set failed\n");
if (sock_epoll_add(&map->epoll_set, conn_fd))
- SOCK_LOG_ERROR("failed to add to epoll set: %d\n", conn_fd);
+ SOCK_LOG_ERROR("failed to add to epoll set: %d\n", conn_fd);
map->table[index].address_published = addr_published;
- sock_pe_poll_add(ep->domain->pe, conn_fd);
+ sock_pe_poll_add(ep_attr->domain->pe, conn_fd);
return &map->table[index];
}
int fd_set_nonblock(int fd)
{
- int flags, ret;
+ int ret;
- flags = fcntl(fd, F_GETFL, 0);
- ret = fcntl(fd, F_SETFL, flags | O_NONBLOCK);
+ ret = fi_fd_nonblock(fd);
if (ret) {
- SOCK_LOG_ERROR("fcntl failed\n");
- ret = -errno;
+ SOCK_LOG_ERROR("fi_fd_nonblock failed\n");
}
return ret;
}
-static void sock_set_sockopt_reuseaddr(int sock)
+void sock_set_sockopt_reuseaddr(int sock)
{
int optval;
optval = 1;
@@ -224,9 +249,9 @@ static void *_sock_conn_listen(void *arg)
struct sockaddr_in remote;
struct pollfd poll_fds[2];
- struct sock_ep *ep = (struct sock_ep *)arg;
- struct sock_conn_listener *listener = &ep->listener;
- struct sock_conn_map *map = &ep->cmap;
+ struct sock_ep_attr *ep_attr = (struct sock_ep_attr *)arg;
+ struct sock_conn_listener *listener = &ep_attr->listener;
+ struct sock_conn_map *map = &ep_attr->cmap;
poll_fds[0].fd = listener->sock;
poll_fds[1].fd = listener->signal_fds[1];
@@ -236,7 +261,7 @@ static void *_sock_conn_listen(void *arg)
while (listener->do_listen) {
if (poll(poll_fds, 2, -1) > 0) {
if (poll_fds[1].revents & POLLIN) {
- ret = read(listener->signal_fds[1], &tmp, 1);
+ ret = ofi_read_socket(listener->signal_fds[1], &tmp, 1);
if (ret != 1) {
SOCK_LOG_ERROR("Invalid signal\n");
goto err;
@@ -252,7 +277,7 @@ static void *_sock_conn_listen(void *arg)
&addr_size);
SOCK_LOG_DBG("CONN: accepted conn-req: %d\n", conn_fd);
if (conn_fd < 0) {
- SOCK_LOG_ERROR("failed to accept: %d\n", errno);
+ SOCK_LOG_ERROR("failed to accept: %s\n", strerror(errno));
goto err;
}
@@ -260,26 +285,25 @@ static void *_sock_conn_listen(void *arg)
ntohs(remote.sin_port));
fastlock_acquire(&map->lock);
- sock_conn_map_insert(ep, &remote, conn_fd, 1);
+ sock_conn_map_insert(ep_attr, &remote, conn_fd, 1);
fastlock_release(&map->lock);
- sock_pe_signal(ep->domain->pe);
+ sock_pe_signal(ep_attr->domain->pe);
}
err:
- close(listener->sock);
+ ofi_close_socket(listener->sock);
SOCK_LOG_DBG("Listener thread exited\n");
return NULL;
}
-int sock_conn_listen(struct sock_ep *ep)
+int sock_conn_listen(struct sock_ep_attr *ep_attr)
{
struct addrinfo *s_res = NULL, *p;
struct addrinfo hints;
int listen_fd = 0, ret;
socklen_t addr_size;
struct sockaddr_in addr;
- struct sock_conn_listener *listener = &ep->listener;
- struct sock_domain *domain = ep->domain;
+ struct sock_conn_listener *listener = &ep_attr->listener;
char service[NI_MAXSERV] = {0};
char *port;
@@ -288,22 +312,22 @@ int sock_conn_listen(struct sock_ep *ep)
hints.ai_socktype = SOCK_STREAM;
hints.ai_flags = AI_PASSIVE;
- if (getnameinfo((void *)ep->src_addr, sizeof(*ep->src_addr),
+ memcpy(&addr, ep_attr->src_addr, sizeof(addr));
+ if (getnameinfo((void *)ep_attr->src_addr, sizeof(*ep_attr->src_addr),
NULL, 0, listener->service,
sizeof(listener->service), NI_NUMERICSERV)) {
SOCK_LOG_ERROR("could not resolve src_addr\n");
return -FI_EINVAL;
}
- if (!sock_fabric_check_service(domain->fab, atoi(listener->service))) {
+ if (ep_attr->ep_type == FI_EP_MSG) {
memset(listener->service, 0, NI_MAXSERV);
- ((struct sockaddr_in *)ep->src_addr)->sin_port = 0;
port = NULL;
+ addr.sin_port = 0;
} else
port = listener->service;
- ret = getaddrinfo(inet_ntoa(((struct sockaddr_in *)ep->src_addr)->sin_addr),
- port, &hints, &s_res);
+ ret = getaddrinfo(inet_ntoa(addr.sin_addr), port, &hints, &s_res);
if (ret) {
SOCK_LOG_ERROR("no available AF_INET address, service %s, %s\n",
listener->service, gai_strerror(ret));
@@ -318,7 +342,7 @@ int sock_conn_listen(struct sock_ep *ep)
if (!bind(listen_fd, s_res->ai_addr, s_res->ai_addrlen))
break;
- close(listen_fd);
+ ofi_close_socket(listen_fd);
listen_fd = -1;
}
}
@@ -337,11 +361,12 @@ int sock_conn_listen(struct sock_ep *ep)
snprintf(listener->service, sizeof listener->service, "%d",
ntohs(addr.sin_port));
SOCK_LOG_DBG("Bound to port: %s - %d\n", listener->service, getpid());
+ ep_attr->msg_src_port = ntohs(addr.sin_port);
}
- if (ep->src_addr->sin_addr.s_addr == 0) {
+ if (ep_attr->src_addr->sin_addr.s_addr == 0) {
snprintf(service, sizeof service, "%s", listener->service);
- ret = sock_get_src_addr_from_hostname(ep->src_addr, service);
+ ret = sock_get_src_addr_from_hostname(ep_attr->src_addr, service);
if (ret)
goto err;
}
@@ -351,51 +376,51 @@ int sock_conn_listen(struct sock_ep *ep)
goto err;
}
- ((struct sockaddr_in *) (ep->src_addr))->sin_port =
- htons(atoi(listener->service));
- listener->do_listen = 1;
- listener->sock = listen_fd;
+ if (((struct sockaddr_in *) (ep_attr->src_addr))->sin_port == 0) {
+ ((struct sockaddr_in *) (ep_attr->src_addr))->sin_port =
+ htons(atoi(listener->service));
+ }
- sock_fabric_add_service(domain->fab, atoi(listener->service));
+ listener->sock = listen_fd;
if (socketpair(AF_UNIX, SOCK_STREAM, 0, listener->signal_fds) < 0)
goto err;
+ listener->do_listen = 1;
+
fd_set_nonblock(listener->signal_fds[1]);
if (pthread_create(&listener->listener_thread, 0,
- _sock_conn_listen, ep)) {
+ _sock_conn_listen, ep_attr)) {
SOCK_LOG_ERROR("failed to create conn listener thread\n");
goto err;
} while (!*((volatile int*)&listener->is_ready));
return 0;
err:
if (listen_fd >= 0)
- close(listen_fd);
+ ofi_close_socket(listen_fd);
return -FI_EINVAL;
}
-struct sock_conn *sock_ep_connect(struct sock_ep *ep, fi_addr_t index)
+struct sock_conn *sock_ep_connect(struct sock_ep_attr *ep_attr, fi_addr_t index)
{
int conn_fd = -1, ret;
int do_retry = sock_conn_retry;
struct sock_conn *conn, *new_conn;
- uint16_t idx;
- struct sockaddr_in *addr;
+ struct sockaddr_in addr;
socklen_t lon;
int valopt = 0;
struct pollfd poll_fd;
- if (ep->ep_type == FI_EP_MSG) {
- idx = 0;
- addr = ep->dest_addr;
+ if (ep_attr->ep_type == FI_EP_MSG) {
+ addr = *ep_attr->dest_addr;
+ addr.sin_port = htons(ep_attr->msg_dest_port);
} else {
- idx = index & ep->av->mask;
- addr = (struct sockaddr_in *)&ep->av->table[idx].addr;
+ addr = *((struct sockaddr_in *)&ep_attr->av->table[index].addr);
}
do_connect:
- fastlock_acquire(&ep->cmap.lock);
- conn = sock_ep_lookup_conn(ep, index, addr);
- fastlock_release(&ep->cmap.lock);
+ fastlock_acquire(&ep_attr->cmap.lock);
+ conn = sock_ep_lookup_conn(ep_attr, index, &addr);
+ fastlock_release(&ep_attr->cmap.lock);
if (conn != SOCK_CM_CONN_IN_PROGRESS)
return conn;
@@ -411,18 +436,18 @@ do_connect:
if (ret) {
SOCK_LOG_ERROR("failed to set conn_fd nonblocking, errno: %d\n", errno);
errno = FI_EOTHER;
- close(conn_fd);
+ ofi_close_socket(conn_fd);
return NULL;
}
- SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr->sin_addr),
- ntohs(addr->sin_port));
+ SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr.sin_addr),
+ ntohs(addr.sin_port));
SOCK_LOG_DBG("Connecting using address:%s\n",
- inet_ntoa(ep->src_addr->sin_addr));
+ inet_ntoa(ep_attr->src_addr->sin_addr));
- ret = connect(conn_fd, (struct sockaddr *) addr, sizeof *addr);
+ ret = connect(conn_fd, (struct sockaddr *) &addr, sizeof addr);
if (ret < 0) {
- if (errno == EINPROGRESS) {
+ if (ofi_sockerr() == EINPROGRESS) {
poll_fd.fd = conn_fd;
poll_fd.events = POLLOUT;
@@ -441,19 +466,19 @@ do_connect:
if (valopt) {
SOCK_LOG_DBG("Error in connection() %d - %s - %d\n", valopt, strerror(valopt), conn_fd);
- SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr->sin_addr),
- ntohs(addr->sin_port));
+ SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr.sin_addr),
+ ntohs(addr.sin_port));
SOCK_LOG_DBG("Connecting using address:%s\n",
- inet_ntoa(ep->src_addr->sin_addr));
+ inet_ntoa(ep_attr->src_addr->sin_addr));
goto retry;
}
goto out;
} else {
SOCK_LOG_DBG("Timeout or error() - %s: %d\n", strerror(errno), conn_fd);
- SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr->sin_addr),
- ntohs(addr->sin_port));
+ SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr.sin_addr),
+ ntohs(addr.sin_port));
SOCK_LOG_DBG("Connecting using address:%s\n",
- inet_ntoa(ep->src_addr->sin_addr));
+ inet_ntoa(ep_attr->src_addr->sin_addr));
goto retry;
}
} else {
@@ -466,34 +491,37 @@ retry:
if (!do_retry)
goto err;
- close(conn_fd);
- conn_fd = -1;
+ if (conn_fd != -1) {
+ ofi_close_socket(conn_fd);
+ conn_fd = -1;
+ }
SOCK_LOG_ERROR("Connect error, retrying - %s - %d\n", strerror(errno), conn_fd);
- SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr->sin_addr),
- ntohs(addr->sin_port));
+ SOCK_LOG_DBG("Connecting to: %s:%d\n", inet_ntoa(addr.sin_addr),
+ ntohs(addr.sin_port));
SOCK_LOG_DBG("Connecting using address:%s\n",
- inet_ntoa(ep->src_addr->sin_addr));
+ inet_ntoa(ep_attr->src_addr->sin_addr));
goto do_connect;
out:
- fastlock_acquire(&ep->cmap.lock);
- new_conn = sock_conn_map_insert(ep, addr, conn_fd, 0);
+ fastlock_acquire(&ep_attr->cmap.lock);
+ new_conn = sock_conn_map_insert(ep_attr, &addr, conn_fd, 0);
if (!new_conn) {
- fastlock_release(&ep->cmap.lock);
+ fastlock_release(&ep_attr->cmap.lock);
goto err;
}
- new_conn->av_index = (ep->ep_type == FI_EP_MSG) ? FI_ADDR_NOTAVAIL : (fi_addr_t) idx;
- conn = idm_lookup(&ep->av_idm, index);
+ new_conn->av_index = (ep_attr->ep_type == FI_EP_MSG) ? FI_ADDR_NOTAVAIL : index;
+ conn = idm_lookup(&ep_attr->av_idm, index);
if (conn == SOCK_CM_CONN_IN_PROGRESS) {
- idm_set(&ep->av_idm, index, new_conn);
+ if (idm_set(&ep_attr->av_idm, index, new_conn) < 0)
+ SOCK_LOG_ERROR("idm_set failed\n");
conn = new_conn;
}
- fastlock_release(&ep->cmap.lock);
+ fastlock_release(&ep_attr->cmap.lock);
return conn;
err:
- close(conn_fd);
+ ofi_close_socket(conn_fd);
return NULL;
}
diff --git a/prov/sockets/src/sock_cq.c b/prov/sockets/src/sock_cq.c
index 34be1ca..3c3f33c 100644
--- a/prov/sockets/src/sock_cq.c
+++ b/prov/sockets/src/sock_cq.c
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2014 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -49,6 +50,53 @@
#define SOCK_LOG_DBG(...) _SOCK_LOG_DBG(FI_LOG_CQ, __VA_ARGS__)
#define SOCK_LOG_ERROR(...) _SOCK_LOG_ERROR(FI_LOG_CQ, __VA_ARGS__)
+void sock_cq_add_tx_ctx(struct sock_cq *cq, struct sock_tx_ctx *tx_ctx)
+{
+ struct dlist_entry *entry;
+ struct sock_tx_ctx *curr_ctx;
+ fastlock_acquire(&cq->list_lock);
+ for (entry = cq->tx_list.next; entry != &cq->tx_list;
+ entry = entry->next) {
+ curr_ctx = container_of(entry, struct sock_tx_ctx, cq_entry);
+ if (tx_ctx == curr_ctx)
+ goto out;
+ }
+ dlist_insert_tail(&tx_ctx->cq_entry, &cq->tx_list);
+out:
+ fastlock_release(&cq->list_lock);
+}
+
+void sock_cq_remove_tx_ctx(struct sock_cq *cq, struct sock_tx_ctx *tx_ctx)
+{
+ fastlock_acquire(&cq->list_lock);
+ dlist_remove(&tx_ctx->cq_entry);
+ fastlock_release(&cq->list_lock);
+}
+
+void sock_cq_add_rx_ctx(struct sock_cq *cq, struct sock_rx_ctx *rx_ctx)
+{
+ struct dlist_entry *entry;
+ struct sock_rx_ctx *curr_ctx;
+ fastlock_acquire(&cq->list_lock);
+
+ for (entry = cq->rx_list.next; entry != &cq->rx_list;
+ entry = entry->next) {
+ curr_ctx = container_of(entry, struct sock_rx_ctx, cq_entry);
+ if (rx_ctx == curr_ctx)
+ goto out;
+ }
+ dlist_insert_tail(&rx_ctx->cq_entry, &cq->rx_list);
+out:
+ fastlock_release(&cq->list_lock);
+}
+
+void sock_cq_remove_rx_ctx(struct sock_cq *cq, struct sock_rx_ctx *rx_ctx)
+{
+ fastlock_acquire(&cq->list_lock);
+ dlist_remove(&rx_ctx->cq_entry);
+ fastlock_release(&cq->list_lock);
+}
+
int sock_cq_progress(struct sock_cq *cq)
{
struct sock_tx_ctx *tx_ctx;
@@ -62,13 +110,19 @@ int sock_cq_progress(struct sock_cq *cq)
for (entry = cq->tx_list.next; entry != &cq->tx_list;
entry = entry->next) {
tx_ctx = container_of(entry, struct sock_tx_ctx, cq_entry);
- sock_pe_progress_tx_ctx(cq->domain->pe, tx_ctx);
+ if (tx_ctx->use_shared)
+ sock_pe_progress_tx_ctx(cq->domain->pe, tx_ctx->stx_ctx);
+ else
+ sock_pe_progress_tx_ctx(cq->domain->pe, tx_ctx);
}
for (entry = cq->rx_list.next; entry != &cq->rx_list;
entry = entry->next) {
rx_ctx = container_of(entry, struct sock_rx_ctx, cq_entry);
- sock_pe_progress_rx_ctx(cq->domain->pe, rx_ctx);
+ if (rx_ctx->use_shared)
+ sock_pe_progress_rx_ctx(cq->domain->pe, rx_ctx->srx_ctx);
+ else
+ sock_pe_progress_rx_ctx(cq->domain->pe, rx_ctx);
}
fastlock_release(&cq->list_lock);
@@ -155,12 +209,21 @@ static int sock_cq_report_context(struct sock_cq *cq, fi_addr_t addr,
return _sock_cq_write(cq, addr, &cq_entry, sizeof(cq_entry));
}
+static uint64_t sock_cq_sanitize_flags(uint64_t flags)
+{
+ return (flags & (FI_SEND | FI_RECV | FI_RMA | FI_ATOMIC |
+ FI_MSG | FI_TAGGED |
+ FI_READ | FI_WRITE |
+ FI_REMOTE_READ | FI_REMOTE_WRITE |
+ FI_REMOTE_CQ_DATA | FI_MULTI_RECV));
+}
+
static int sock_cq_report_msg(struct sock_cq *cq, fi_addr_t addr,
struct sock_pe_entry *pe_entry)
{
struct fi_cq_msg_entry cq_entry;
cq_entry.op_context = (void *) (uintptr_t) pe_entry->context;
- cq_entry.flags = pe_entry->flags;
+ cq_entry.flags = sock_cq_sanitize_flags(pe_entry->flags);
cq_entry.len = pe_entry->data_len;
return _sock_cq_write(cq, addr, &cq_entry, sizeof(cq_entry));
}
@@ -170,7 +233,7 @@ static int sock_cq_report_data(struct sock_cq *cq, fi_addr_t addr,
{
struct fi_cq_data_entry cq_entry;
cq_entry.op_context = (void *) (uintptr_t) pe_entry->context;
- cq_entry.flags = pe_entry->flags;
+ cq_entry.flags = sock_cq_sanitize_flags(pe_entry->flags);
cq_entry.len = pe_entry->data_len;
cq_entry.buf = (void *) (uintptr_t) pe_entry->buf;
cq_entry.data = pe_entry->data;
@@ -182,7 +245,7 @@ static int sock_cq_report_tagged(struct sock_cq *cq, fi_addr_t addr,
{
struct fi_cq_tagged_entry cq_entry;
cq_entry.op_context = (void *) (uintptr_t) pe_entry->context;
- cq_entry.flags = pe_entry->flags;
+ cq_entry.flags = sock_cq_sanitize_flags(pe_entry->flags);
cq_entry.len = pe_entry->data_len;
cq_entry.buf = (void *) (uintptr_t) pe_entry->buf;
cq_entry.data = pe_entry->data;
@@ -455,6 +518,9 @@ static int sock_cq_verify_attr(struct fi_cq_attr *attr)
case FI_CQ_FORMAT_DATA:
case FI_CQ_FORMAT_TAGGED:
break;
+ case FI_CQ_FORMAT_UNSPEC:
+ attr->format = FI_CQ_FORMAT_CONTEXT;
+ break;
default:
return -FI_ENOSYS;
}
diff --git a/prov/sockets/src/sock_ctx.c b/prov/sockets/src/sock_ctx.c
index ba597da..ebd7ca1 100644
--- a/prov/sockets/src/sock_ctx.c
+++ b/prov/sockets/src/sock_ctx.c
@@ -42,7 +42,7 @@
#define SOCK_LOG_ERROR(...) _SOCK_LOG_ERROR(FI_LOG_EP_CTRL, __VA_ARGS__)
struct sock_rx_ctx *sock_rx_ctx_alloc(const struct fi_rx_attr *attr,
- void *context)
+ void *context, int use_shared)
{
struct sock_rx_ctx *rx_ctx;
rx_ctx = calloc(1, sizeof(*rx_ctx));
@@ -50,7 +50,6 @@ struct sock_rx_ctx *sock_rx_ctx_alloc(const struct fi_rx_attr *attr,
return NULL;
dlist_init(&rx_ctx->cq_entry);
- dlist_init(&rx_ctx->cntr_entry);
dlist_init(&rx_ctx->pe_entry);
dlist_init(&rx_ctx->pe_entry_list);
@@ -62,8 +61,9 @@ struct sock_rx_ctx *sock_rx_ctx_alloc(const struct fi_rx_attr *attr,
rx_ctx->ctx.fid.fclass = FI_CLASS_RX_CTX;
rx_ctx->ctx.fid.context = context;
- rx_ctx->num_left = attr->size;
+ rx_ctx->num_left = sock_get_tx_size(attr->size);
rx_ctx->attr = *attr;
+ rx_ctx->use_shared = use_shared;
return rx_ctx;
}
@@ -75,7 +75,8 @@ void sock_rx_ctx_free(struct sock_rx_ctx *rx_ctx)
}
static struct sock_tx_ctx *sock_tx_context_alloc(const struct fi_tx_attr *attr,
- void *context, size_t fclass)
+ void *context, int use_shared,
+ size_t fclass)
{
struct sock_tx_ctx *tx_ctx;
struct fi_rx_attr rx_attr = {0};
@@ -84,13 +85,12 @@ static struct sock_tx_ctx *sock_tx_context_alloc(const struct fi_tx_attr *attr,
if (!tx_ctx)
return NULL;
- if (rbinit(&tx_ctx->rb,
- (attr->size) ? attr->size * SOCK_EP_TX_ENTRY_SZ :
- SOCK_EP_TX_SZ * SOCK_EP_TX_ENTRY_SZ))
+ if (!use_shared && rbinit(&tx_ctx->rb,
+ (attr->size) ? attr->size * SOCK_EP_TX_ENTRY_SZ :
+ SOCK_EP_TX_SZ * SOCK_EP_TX_ENTRY_SZ))
goto err;
dlist_init(&tx_ctx->cq_entry);
- dlist_init(&tx_ctx->cntr_entry);
dlist_init(&tx_ctx->pe_entry);
dlist_init(&tx_ctx->pe_entry_list);
@@ -105,6 +105,7 @@ static struct sock_tx_ctx *sock_tx_context_alloc(const struct fi_tx_attr *attr,
tx_ctx->fid.ctx.fid.fclass = FI_CLASS_TX_CTX;
tx_ctx->fid.ctx.fid.context = context;
tx_ctx->fclass = FI_CLASS_TX_CTX;
+ tx_ctx->use_shared = use_shared;
break;
case FI_CLASS_STX_CTX:
tx_ctx->fid.stx.fid.fclass = FI_CLASS_STX_CTX;
@@ -117,10 +118,12 @@ static struct sock_tx_ctx *sock_tx_context_alloc(const struct fi_tx_attr *attr,
tx_ctx->attr = *attr;
tx_ctx->attr.op_flags |= FI_TRANSMIT_COMPLETE;
- tx_ctx->rx_ctrl_ctx = sock_rx_ctx_alloc(&rx_attr, NULL);
- if (!tx_ctx->rx_ctrl_ctx)
- goto err;
- tx_ctx->rx_ctrl_ctx->is_ctrl_ctx = 1;
+ if (!use_shared) {
+ tx_ctx->rx_ctrl_ctx = sock_rx_ctx_alloc(&rx_attr, NULL, 0);
+ if (!tx_ctx->rx_ctrl_ctx)
+ goto err;
+ tx_ctx->rx_ctrl_ctx->is_ctrl_ctx = 1;
+ }
return tx_ctx;
err:
@@ -130,15 +133,15 @@ err:
struct sock_tx_ctx *sock_tx_ctx_alloc(const struct fi_tx_attr *attr,
- void *context)
+ void *context, int use_shared)
{
- return sock_tx_context_alloc(attr, context, FI_CLASS_TX_CTX);
+ return sock_tx_context_alloc(attr, context, use_shared, FI_CLASS_TX_CTX);
}
struct sock_tx_ctx *sock_stx_ctx_alloc(const struct fi_tx_attr *attr,
void *context)
{
- return sock_tx_context_alloc(attr, context, FI_CLASS_STX_CTX);
+ return sock_tx_context_alloc(attr, context, 0, FI_CLASS_STX_CTX);
}
void sock_tx_ctx_free(struct sock_tx_ctx *tx_ctx)
@@ -146,8 +149,11 @@ void sock_tx_ctx_free(struct sock_tx_ctx *tx_ctx)
fastlock_destroy(&tx_ctx->rlock);
fastlock_destroy(&tx_ctx->wlock);
fastlock_destroy(&tx_ctx->lock);
- rbfree(&tx_ctx->rb);
- sock_rx_ctx_free(tx_ctx->rx_ctrl_ctx);
+
+ if (!tx_ctx->use_shared) {
+ rbfree(&tx_ctx->rb);
+ sock_rx_ctx_free(tx_ctx->rx_ctrl_ctx);
+ }
free(tx_ctx);
}
@@ -176,7 +182,7 @@ void sock_tx_ctx_abort(struct sock_tx_ctx *tx_ctx)
void sock_tx_ctx_write_op_send(struct sock_tx_ctx *tx_ctx,
struct sock_op *op, uint64_t flags, uint64_t context,
- uint64_t dest_addr, uint64_t buf, struct sock_ep *ep,
+ uint64_t dest_addr, uint64_t buf, struct sock_ep_attr *ep_attr,
struct sock_conn *conn)
{
sock_tx_ctx_write(tx_ctx, op, sizeof(*op));
@@ -184,23 +190,23 @@ void sock_tx_ctx_write_op_send(struct sock_tx_ctx *tx_ctx,
sock_tx_ctx_write(tx_ctx, &context, sizeof(context));
sock_tx_ctx_write(tx_ctx, &dest_addr, sizeof(dest_addr));
sock_tx_ctx_write(tx_ctx, &buf, sizeof(buf));
- sock_tx_ctx_write(tx_ctx, &ep, sizeof(ep));
+ sock_tx_ctx_write(tx_ctx, &ep_attr, sizeof(ep_attr));
sock_tx_ctx_write(tx_ctx, &conn, sizeof(conn));
}
void sock_tx_ctx_write_op_tsend(struct sock_tx_ctx *tx_ctx,
struct sock_op *op, uint64_t flags, uint64_t context,
- uint64_t dest_addr, uint64_t buf, struct sock_ep *ep,
+ uint64_t dest_addr, uint64_t buf, struct sock_ep_attr *ep_attr,
struct sock_conn *conn, uint64_t tag)
{
sock_tx_ctx_write_op_send(tx_ctx, op, flags, context, dest_addr,
- buf, ep, conn);
+ buf, ep_attr, conn);
sock_tx_ctx_write(tx_ctx, &tag, sizeof(tag));
}
void sock_tx_ctx_read_op_send(struct sock_tx_ctx *tx_ctx,
struct sock_op *op, uint64_t *flags, uint64_t *context,
- uint64_t *dest_addr, uint64_t *buf, struct sock_ep **ep,
+ uint64_t *dest_addr, uint64_t *buf, struct sock_ep_attr **ep_attr,
struct sock_conn **conn)
{
rbread(&tx_ctx->rb, op, sizeof(*op));
@@ -208,6 +214,6 @@ void sock_tx_ctx_read_op_send(struct sock_tx_ctx *tx_ctx,
rbread(&tx_ctx->rb, context, sizeof(*context));
rbread(&tx_ctx->rb, dest_addr, sizeof(*dest_addr));
rbread(&tx_ctx->rb, buf, sizeof(*buf));
- rbread(&tx_ctx->rb, ep, sizeof(*ep));
+ rbread(&tx_ctx->rb, ep_attr, sizeof(*ep_attr));
rbread(&tx_ctx->rb, conn, sizeof(*conn));
}
diff --git a/prov/sockets/src/sock_dom.c b/prov/sockets/src/sock_dom.c
index 70b74b8..adce1c4 100644
--- a/prov/sockets/src/sock_dom.c
+++ b/prov/sockets/src/sock_dom.c
@@ -65,11 +65,6 @@ int sock_verify_domain_attr(struct fi_domain_attr *attr)
if (!attr)
return 0;
- if (attr->name) {
- if (strcmp(attr->name, sock_dom_name))
- return -FI_ENODATA;
- }
-
switch (attr->threading) {
case FI_THREAD_UNSPEC:
case FI_THREAD_SAFE:
@@ -167,39 +162,27 @@ static int sock_dom_close(struct fid *fid)
sock_pe_finalize(dom->pe);
fastlock_destroy(&dom->lock);
- rbtDelete(dom->mr_heap);
+ ofi_mr_close(dom->mr_heap);
sock_dom_remove_from_list(dom);
free(dom);
return 0;
}
-static uint64_t sock_get_mr_key(struct sock_domain *dom)
-{
- uint64_t i;
- for (i = 0; i < UINT64_MAX; i++) {
- if (!sock_mr_get_entry(dom, i))
- return i;
- }
- SOCK_LOG_ERROR("failed to get a free key\n");
- return UINT64_MAX;
-}
-
static int sock_mr_close(struct fid *fid)
{
struct sock_domain *dom;
struct sock_mr *mr;
- RbtIterator it;
- RbtStatus res;
uint64_t mr_key;
+ int err = 0;
mr = container_of(fid, struct sock_mr, mr_fid.fid);
dom = mr->domain;
mr_key = mr->key;
fastlock_acquire(&dom->lock);
- it = rbtFind(dom->mr_heap, &mr_key);
- if (!it || ((res = rbtErase(dom->mr_heap, it)) != RBT_STATUS_OK))
- SOCK_LOG_ERROR("Invalid mr\n");
+ err = ofi_mr_erase(dom->mr_heap, mr_key);
+ if (err != 0)
+ SOCK_LOG_ERROR("MR Erase error %d \n", err);
fastlock_release(&dom->lock);
atomic_dec(&dom->ref);
@@ -247,47 +230,21 @@ static struct fi_ops sock_mr_fi_ops = {
.ops_open = fi_no_ops_open,
};
-struct sock_mr *sock_mr_get_entry(struct sock_domain *domain, uint64_t key)
-{
- RbtIterator it;
- void *value;
- void *mr_key = &key;
-
- it = rbtFind(domain->mr_heap, mr_key);
- if (!it)
- return NULL;
-
- rbtKeyValue(domain->mr_heap, it, &mr_key, &value);
- return (struct sock_mr *) value;
-}
-
struct sock_mr *sock_mr_verify_key(struct sock_domain *domain, uint64_t key,
- void *buf, size_t len, uint64_t access)
+ uintptr_t *buf, size_t len, uint64_t access)
{
- int i;
+ int err = 0;
struct sock_mr *mr;
fastlock_acquire(&domain->lock);
- mr = sock_mr_get_entry(domain, key);
- if (!mr) {
- fastlock_release(&domain->lock);
- return NULL;
- }
-
- if (domain->attr.mr_mode == FI_MR_SCALABLE)
- buf = (char *)buf + mr->offset;
- for (i = 0; i < mr->iov_count; i++) {
- if ((uintptr_t)buf >= (uintptr_t)mr->mr_iov[i].iov_base &&
- ((uintptr_t)buf + len <= (uintptr_t) mr->mr_iov[i].iov_base +
- mr->mr_iov[i].iov_len)) {
- if ((access & mr->access) == access)
- goto out;
- }
+ err = ofi_mr_retrieve_and_verify(domain->mr_heap, len, buf, key, access,
+ (void **)&mr);
+ if(err != 0) {
+ SOCK_LOG_ERROR("MR check failed\n");
+ mr = NULL;
}
- SOCK_LOG_ERROR("MR check failed\n");
- mr = NULL;
-out:
+
fastlock_release(&domain->lock);
return mr;
}
@@ -307,7 +264,6 @@ static int sock_regattr(struct fid *fid, const struct fi_mr_attr *attr,
struct sock_mr *_mr;
uint64_t key;
struct fid_domain *domain;
- RbtStatus res;
int ret = 0;
if (fid->fclass != FI_CLASS_DOMAIN || !attr || attr->iov_count <= 0) {
@@ -317,46 +273,28 @@ static int sock_regattr(struct fid *fid, const struct fi_mr_attr *attr,
domain = container_of(fid, struct fid_domain, fid);
dom = container_of(domain, struct sock_domain, dom_fid);
- _mr = calloc(1, sizeof(*_mr) +
- sizeof(_mr->mr_iov) * (attr->iov_count - 1));
+ _mr = calloc(1, sizeof(*_mr));
if (!_mr)
return -FI_ENOMEM;
fastlock_acquire(&dom->lock);
- if (dom->attr.mr_mode == FI_MR_SCALABLE &&
- sock_mr_get_entry(dom, attr->requested_key) != NULL) {
- ret = -FI_ENOKEY;
- goto err;
- }
_mr->mr_fid.fid.fclass = FI_CLASS_MR;
_mr->mr_fid.fid.context = attr->context;
_mr->mr_fid.fid.ops = &sock_mr_fi_ops;
_mr->domain = dom;
- _mr->access = attr->access;
_mr->flags = flags;
- _mr->offset = (dom->attr.mr_mode == FI_MR_SCALABLE) ?
- (uintptr_t) attr->mr_iov[0].iov_base + attr->offset :
- (uintptr_t) attr->mr_iov[0].iov_base;
- key = (dom->attr.mr_mode == FI_MR_BASIC) ?
- sock_get_mr_key(dom) : attr->requested_key;
-
- _mr->key = key;
- res = rbtInsert(dom->mr_heap, &_mr->key, _mr);
- if (res != RBT_STATUS_OK) {
- ret = -FI_ENOMEM;
+ ret = ofi_mr_insert(dom->mr_heap, attr, &key, _mr);
+ if (ret != 0)
goto err;
- }
- _mr->mr_fid.key = key;
+
+ _mr->mr_fid.key = _mr->key = key;
_mr->mr_fid.mem_desc = (void *) (uintptr_t) key;
fastlock_release(&dom->lock);
- _mr->iov_count = attr->iov_count;
- memcpy(&_mr->mr_iov, attr->mr_iov, sizeof(_mr->mr_iov) * attr->iov_count);
-
*mr = &_mr->mr_fid;
atomic_inc(&dom->ref);
@@ -478,13 +416,6 @@ static struct fi_ops_mr sock_dom_mr_ops = {
.regattr = sock_regattr,
};
-static int sock_compare_mr_keys(void *key1, void *key2)
-{
- uint64_t k1 = *((uint64_t *) key1);
- uint64_t k2 = *((uint64_t *) key2);
- return (k1 < k2) ? -1 : (k1 > k2);
-}
-
int sock_domain(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **dom, void *context)
{
@@ -531,10 +462,6 @@ int sock_domain(struct fid_fabric *fabric, struct fi_info *info,
goto err;
}
- sock_domain->mr_heap = rbtNew(&sock_compare_mr_keys);
- if (!sock_domain->mr_heap) {
- goto err;
- }
sock_domain->fab = fab;
*dom = &sock_domain->dom_fid;
@@ -543,6 +470,11 @@ int sock_domain(struct fid_fabric *fabric, struct fi_info *info,
else
sock_domain->attr = sock_domain_attr;
+ ret = ofi_mr_init(&sock_prov, sock_domain->attr.mr_mode, &sock_domain->mr_heap);
+ if (!ret) {
+ return ret;
+ }
+
sock_dom_add_to_list(sock_domain);
return 0;
diff --git a/prov/sockets/src/sock_ep.c b/prov/sockets/src/sock_ep.c
index 57f5bc2..6c4ab2e 100644
--- a/prov/sockets/src/sock_ep.c
+++ b/prov/sockets/src/sock_ep.c
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013-2014 Intel Corporation. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -32,6 +33,7 @@
#include "config.h"
+#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <limits.h>
@@ -39,6 +41,12 @@
#include <arpa/inet.h>
#include <sys/socket.h>
+#if HAVE_GETIFADDRS
+#include <net/if.h>
+#include <ifaddrs.h>
+#endif
+
+#include "fi_util.h"
#include "sock.h"
#include "sock_util.h"
@@ -80,6 +88,36 @@ const struct fi_rx_attr sock_srx_attr = {
.iov_limit = SOCK_EP_MAX_IOV_LIMIT,
};
+static void sock_tx_ctx_close(struct sock_tx_ctx *tx_ctx)
+{
+ if (tx_ctx->comp.send_cq)
+ sock_cq_remove_tx_ctx(tx_ctx->comp.send_cq, tx_ctx);
+
+ if (tx_ctx->comp.send_cntr)
+ sock_cntr_remove_tx_ctx(tx_ctx->comp.send_cntr, tx_ctx);
+
+ if (tx_ctx->comp.read_cntr)
+ sock_cntr_remove_tx_ctx(tx_ctx->comp.read_cntr, tx_ctx);
+
+ if (tx_ctx->comp.write_cntr)
+ sock_cntr_remove_tx_ctx(tx_ctx->comp.write_cntr, tx_ctx);
+}
+
+static void sock_rx_ctx_close(struct sock_rx_ctx *rx_ctx)
+{
+ if (rx_ctx->comp.recv_cq)
+ sock_cq_remove_rx_ctx(rx_ctx->comp.recv_cq, rx_ctx);
+
+ if (rx_ctx->comp.recv_cntr)
+ sock_cntr_remove_rx_ctx(rx_ctx->comp.recv_cntr, rx_ctx);
+
+ if (rx_ctx->comp.rem_read_cntr)
+ sock_cntr_remove_rx_ctx(rx_ctx->comp.rem_read_cntr, rx_ctx);
+
+ if (rx_ctx->comp.rem_write_cntr)
+ sock_cntr_remove_rx_ctx(rx_ctx->comp.rem_write_cntr, rx_ctx);
+}
+
static int sock_ctx_close(struct fid *fid)
{
struct sock_tx_ctx *tx_ctx;
@@ -89,16 +127,18 @@ static int sock_ctx_close(struct fid *fid)
case FI_CLASS_TX_CTX:
tx_ctx = container_of(fid, struct sock_tx_ctx, fid.ctx.fid);
sock_pe_remove_tx_ctx(tx_ctx);
- atomic_dec(&tx_ctx->ep->num_tx_ctx);
+ atomic_dec(&tx_ctx->ep_attr->num_tx_ctx);
atomic_dec(&tx_ctx->domain->ref);
+ sock_tx_ctx_close(tx_ctx);
sock_tx_ctx_free(tx_ctx);
break;
case FI_CLASS_RX_CTX:
rx_ctx = container_of(fid, struct sock_rx_ctx, ctx.fid);
sock_pe_remove_rx_ctx(rx_ctx);
- atomic_dec(&rx_ctx->ep->num_rx_ctx);
+ atomic_dec(&rx_ctx->ep_attr->num_rx_ctx);
atomic_dec(&rx_ctx->domain->ref);
+ sock_rx_ctx_close(rx_ctx);
sock_rx_ctx_free(rx_ctx);
break;
@@ -143,9 +183,7 @@ static int sock_ctx_bind_cq(struct fid *fid, struct fid *bfid, uint64_t flags)
tx_ctx->comp.send_cq_event = 1;
}
- fastlock_acquire(&sock_cq->list_lock);
- dlist_insert_tail(&tx_ctx->cq_entry, &sock_cq->tx_list);
- fastlock_release(&sock_cq->list_lock);
+ sock_cq_add_tx_ctx(sock_cq, tx_ctx);
break;
case FI_CLASS_RX_CTX:
@@ -156,22 +194,7 @@ static int sock_ctx_bind_cq(struct fid *fid, struct fid *bfid, uint64_t flags)
rx_ctx->comp.recv_cq_event = 1;
}
- fastlock_acquire(&sock_cq->list_lock);
- dlist_insert_tail(&rx_ctx->cq_entry, &sock_cq->rx_list);
- fastlock_release(&sock_cq->list_lock);
- break;
-
- case FI_CLASS_STX_CTX:
- tx_ctx = container_of(fid, struct sock_tx_ctx, fid.stx.fid);
- if (flags & FI_SEND) {
- tx_ctx->comp.send_cq = sock_cq;
- if (flags & FI_SELECTIVE_COMPLETION)
- tx_ctx->comp.send_cq_event = 1;
- }
-
- fastlock_acquire(&sock_cq->list_lock);
- dlist_insert_tail(&tx_ctx->cq_entry, &sock_cq->tx_list);
- fastlock_release(&sock_cq->list_lock);
+ sock_cq_add_rx_ctx(sock_cq, rx_ctx);
break;
default:
@@ -205,9 +228,7 @@ static int sock_ctx_bind_cntr(struct fid *fid, struct fid *bfid, uint64_t flags)
if (flags & FI_WRITE)
tx_ctx->comp.write_cntr = cntr;
- fastlock_acquire(&cntr->list_lock);
- dlist_insert_tail(&tx_ctx->cntr_entry, &cntr->tx_list);
- fastlock_release(&cntr->list_lock);
+ sock_cntr_add_tx_ctx(cntr, tx_ctx);
break;
case FI_CLASS_RX_CTX:
@@ -220,27 +241,7 @@ static int sock_ctx_bind_cntr(struct fid *fid, struct fid *bfid, uint64_t flags)
if (flags & FI_REMOTE_WRITE)
rx_ctx->comp.rem_write_cntr = cntr;
-
- fastlock_acquire(&cntr->list_lock);
- dlist_insert_tail(&rx_ctx->cntr_entry, &cntr->rx_list);
- fastlock_release(&cntr->list_lock);
- break;
-
- case FI_CLASS_STX_CTX:
- tx_ctx = container_of(fid, struct sock_tx_ctx, fid.ctx.fid);
- if (flags & FI_SEND)
- tx_ctx->comp.send_cntr = cntr;
-
- if (flags & FI_READ)
- tx_ctx->comp.read_cntr = cntr;
-
- if (flags & FI_WRITE)
- tx_ctx->comp.write_cntr = cntr;
-
- fastlock_acquire(&cntr->list_lock);
- dlist_insert_tail(&tx_ctx->cntr_entry, &cntr->tx_list);
- fastlock_release(&cntr->list_lock);
-
+ sock_cntr_add_rx_ctx(cntr, rx_ctx);
break;
default:
@@ -278,12 +279,10 @@ static int sock_ctx_enable(struct fid_ep *ep)
case FI_CLASS_RX_CTX:
rx_ctx = container_of(ep, struct sock_rx_ctx, ctx.fid);
rx_ctx->enabled = 1;
- if (!rx_ctx->progress) {
- sock_pe_add_rx_ctx(rx_ctx->domain->pe, rx_ctx);
- rx_ctx->progress = 1;
- }
- if (!rx_ctx->ep->listener.listener_thread &&
- sock_conn_listen(rx_ctx->ep)) {
+ sock_pe_add_rx_ctx(rx_ctx->domain->pe, rx_ctx);
+
+ if (!rx_ctx->ep_attr->listener.listener_thread &&
+ sock_conn_listen(rx_ctx->ep_attr)) {
SOCK_LOG_ERROR("failed to create listener\n");
}
return 0;
@@ -291,12 +290,10 @@ static int sock_ctx_enable(struct fid_ep *ep)
case FI_CLASS_TX_CTX:
tx_ctx = container_of(ep, struct sock_tx_ctx, fid.ctx.fid);
tx_ctx->enabled = 1;
- if (!tx_ctx->progress) {
- sock_pe_add_tx_ctx(tx_ctx->domain->pe, tx_ctx);
- tx_ctx->progress = 1;
- }
- if (!tx_ctx->ep->listener.listener_thread &&
- sock_conn_listen(tx_ctx->ep)) {
+ sock_pe_add_tx_ctx(tx_ctx->domain->pe, tx_ctx);
+
+ if (!tx_ctx->ep_attr->listener.listener_thread &&
+ sock_conn_listen(tx_ctx->ep_attr)) {
SOCK_LOG_ERROR("failed to create listener\n");
}
return 0;
@@ -308,22 +305,65 @@ static int sock_ctx_enable(struct fid_ep *ep)
return -FI_EINVAL;
}
+int sock_getopflags(struct fi_tx_attr *tx_attr, struct fi_rx_attr *rx_attr,
+ uint64_t *flags)
+{
+ if ((*flags & FI_TRANSMIT) && (*flags & FI_RECV)) {
+ SOCK_LOG_ERROR("Both Tx/Rx flags cannot be specified\n");
+ return -FI_EINVAL;
+ } else if (tx_attr && (*flags & FI_TRANSMIT)) {
+ *flags = tx_attr->op_flags;
+ } else if (rx_attr && (*flags & FI_RECV)) {
+ *flags = rx_attr->op_flags;
+ } else {
+ SOCK_LOG_ERROR("Tx/Rx flags not specified\n");
+ return -FI_EINVAL;
+ }
+ return 0;
+}
+
+int sock_setopflags(struct fi_tx_attr *tx_attr, struct fi_rx_attr *rx_attr,
+ uint64_t flags)
+{
+ if ((flags & FI_TRANSMIT) && (flags & FI_RECV)) {
+ SOCK_LOG_ERROR("Both Tx/Rx flags cannot be specified\n");
+ return -FI_EINVAL;
+ } else if (tx_attr && (flags & FI_TRANSMIT)) {
+ tx_attr->op_flags = flags;
+ tx_attr->op_flags &= ~FI_TRANSMIT;
+ if (!(flags & (FI_INJECT_COMPLETE | FI_TRANSMIT_COMPLETE |
+ FI_DELIVERY_COMPLETE)))
+ tx_attr->op_flags |= FI_TRANSMIT_COMPLETE;
+ } else if (rx_attr && (flags & FI_RECV)) {
+ rx_attr->op_flags = flags;
+ rx_attr->op_flags &= ~FI_RECV;
+ } else {
+ SOCK_LOG_ERROR("Tx/Rx flags not specified\n");
+ return -FI_EINVAL;
+ }
+ return 0;
+}
+
static int sock_ctx_control(struct fid *fid, int command, void *arg)
{
struct fid_ep *ep;
struct sock_tx_ctx *tx_ctx;
struct sock_rx_ctx *rx_ctx;
+ int ret;
switch (fid->fclass) {
case FI_CLASS_TX_CTX:
tx_ctx = container_of(fid, struct sock_tx_ctx, fid.ctx.fid);
switch (command) {
case FI_GETOPSFLAG:
- *(uint64_t *) arg = tx_ctx->attr.op_flags;
+ ret = sock_getopflags(&tx_ctx->attr, NULL, (uint64_t *) arg);
+ if (ret)
+ return -EINVAL;
break;
case FI_SETOPSFLAG:
- tx_ctx->attr.op_flags = *(uint64_t *) arg;
- tx_ctx->attr.op_flags |= FI_TRANSMIT_COMPLETE;
+ ret = sock_setopflags(&tx_ctx->attr, NULL, *(uint64_t *) arg);
+ if (ret)
+ return -EINVAL;
break;
case FI_ENABLE:
ep = container_of(fid, struct fid_ep, fid);
@@ -335,13 +375,18 @@ static int sock_ctx_control(struct fid *fid, int command, void *arg)
break;
case FI_CLASS_RX_CTX:
+ case FI_CLASS_SRX_CTX:
rx_ctx = container_of(fid, struct sock_rx_ctx, ctx.fid);
switch (command) {
case FI_GETOPSFLAG:
- *(uint64_t *) arg = rx_ctx->attr.op_flags;
+ ret = sock_getopflags(NULL, &rx_ctx->attr, (uint64_t *) arg);
+ if (ret)
+ return -EINVAL;
break;
case FI_SETOPSFLAG:
- rx_ctx->attr.op_flags = *(uint64_t *) arg;
+ ret = sock_setopflags(NULL, &rx_ctx->attr, *(uint64_t *) arg);
+ if (ret)
+ return -EINVAL;
break;
case FI_ENABLE:
ep = container_of(fid, struct fid_ep, fid);
@@ -352,21 +397,6 @@ static int sock_ctx_control(struct fid *fid, int command, void *arg)
}
break;
- case FI_CLASS_STX_CTX:
- tx_ctx = container_of(fid, struct sock_tx_ctx, fid.stx.fid);
- switch (command) {
- case FI_GETOPSFLAG:
- *(uint64_t *) arg = tx_ctx->attr.op_flags;
- break;
- case FI_SETOPSFLAG:
- tx_ctx->attr.op_flags = *(uint64_t *) arg;
- tx_ctx->attr.op_flags |= FI_TRANSMIT_COMPLETE;
- break;
- default:
- return -FI_ENOSYS;
- }
- break;
-
default:
return -FI_ENOSYS;
}
@@ -483,13 +513,12 @@ static ssize_t sock_ep_cancel(fid_t fid, void *context)
switch (fid->fclass) {
case FI_CLASS_EP:
sock_ep = container_of(fid, struct sock_ep, ep.fid);
- rx_ctx = sock_ep->rx_ctx;
+ rx_ctx = sock_ep->attr->rx_ctx;
break;
case FI_CLASS_RX_CTX:
case FI_CLASS_SRX_CTX:
rx_ctx = container_of(fid, struct sock_rx_ctx, ctx.fid);
- sock_ep = rx_ctx->ep;
break;
case FI_CLASS_TX_CTX:
@@ -504,6 +533,12 @@ static ssize_t sock_ep_cancel(fid_t fid, void *context)
return sock_rx_ctx_cancel(rx_ctx, context);
}
+size_t sock_get_tx_size(size_t size)
+{
+ return roundup_power_of_two(size * SOCK_EP_TX_ENTRY_SZ) /
+ SOCK_EP_TX_ENTRY_SZ;
+}
+
static ssize_t sock_rx_size_left(struct fid_ep *ep)
{
struct sock_rx_ctx *rx_ctx;
@@ -512,7 +547,7 @@ static ssize_t sock_rx_size_left(struct fid_ep *ep)
switch (ep->fid.fclass) {
case FI_CLASS_EP:
sock_ep = container_of(ep, struct sock_ep, ep);
- rx_ctx = sock_ep->rx_ctx;
+ rx_ctx = sock_ep->attr->rx_ctx;
break;
case FI_CLASS_RX_CTX:
@@ -525,7 +560,7 @@ static ssize_t sock_rx_size_left(struct fid_ep *ep)
return -FI_EINVAL;
}
- return rx_ctx->num_left;
+ return rx_ctx->enabled ? rx_ctx->num_left : -FI_EOPBADSTATE;
}
static ssize_t sock_tx_size_left(struct fid_ep *ep)
@@ -537,7 +572,7 @@ static ssize_t sock_tx_size_left(struct fid_ep *ep)
switch (ep->fid.fclass) {
case FI_CLASS_EP:
sock_ep = container_of(ep, struct sock_ep, ep);
- tx_ctx = sock_ep->tx_ctx;
+ tx_ctx = sock_ep->attr->tx_ctx;
break;
case FI_CLASS_TX_CTX:
@@ -549,6 +584,9 @@ static ssize_t sock_tx_size_left(struct fid_ep *ep)
return -FI_EINVAL;
}
+ if (!tx_ctx->enabled)
+ return -FI_EOPBADSTATE;
+
fastlock_acquire(&tx_ctx->wlock);
num_left = rbavail(&tx_ctx->rb)/SOCK_EP_TX_ENTRY_SZ;
fastlock_release(&tx_ctx->wlock);
@@ -584,77 +622,98 @@ static int sock_ep_close(struct fid *fid)
return -FI_EINVAL;
}
- if (atomic_get(&sock_ep->ref) || atomic_get(&sock_ep->num_rx_ctx) ||
- atomic_get(&sock_ep->num_tx_ctx))
+ if (sock_ep->is_alias) {
+ atomic_dec(&sock_ep->attr->ref);
+ return 0;
+ }
+ if (atomic_get(&sock_ep->attr->ref) || atomic_get(&sock_ep->attr->num_rx_ctx) ||
+ atomic_get(&sock_ep->attr->num_tx_ctx))
return -FI_EBUSY;
- if (sock_ep->ep_type == FI_EP_MSG) {
- sock_ep->cm.do_listen = 0;
- if (write(sock_ep->cm.signal_fds[0], &c, 1) != 1)
+ if (sock_ep->attr->ep_type == FI_EP_MSG) {
+ sock_ep->attr->cm.do_listen = 0;
+ if (ofi_write_socket(sock_ep->attr->cm.signal_fds[0], &c, 1) != 1)
SOCK_LOG_DBG("Failed to signal\n");
- if (sock_ep->cm.listener_thread &&
- pthread_join(sock_ep->cm.listener_thread, NULL)) {
+ if (sock_ep->attr->cm.listener_thread &&
+ pthread_join(sock_ep->attr->cm.listener_thread, NULL)) {
SOCK_LOG_ERROR("pthread join failed (%d)\n", errno);
}
- close(sock_ep->cm.signal_fds[0]);
- close(sock_ep->cm.signal_fds[1]);
+ ofi_close_socket(sock_ep->attr->cm.signal_fds[0]);
+ ofi_close_socket(sock_ep->attr->cm.signal_fds[1]);
} else {
- if (sock_ep->av)
- atomic_dec(&sock_ep->av->ref);
+ if (sock_ep->attr->av)
+ atomic_dec(&sock_ep->attr->av->ref);
+ }
+ if (sock_ep->attr->av) {
+ fastlock_acquire(&sock_ep->attr->av->list_lock);
+ fid_list_remove(&sock_ep->attr->av->ep_list, &sock_ep->attr->lock, &sock_ep->ep.fid);
+ fastlock_release(&sock_ep->attr->av->list_lock);
}
- pthread_mutex_lock(&sock_ep->domain->pe->list_lock);
- if (sock_ep->tx_shared) {
- fastlock_acquire(&sock_ep->tx_ctx->lock);
- dlist_remove(&sock_ep->tx_ctx_entry);
- fastlock_release(&sock_ep->tx_ctx->lock);
+ pthread_mutex_lock(&sock_ep->attr->domain->pe->list_lock);
+ if (sock_ep->attr->tx_shared) {
+ fastlock_acquire(&sock_ep->attr->tx_ctx->lock);
+ dlist_remove(&sock_ep->attr->tx_ctx_entry);
+ fastlock_release(&sock_ep->attr->tx_ctx->lock);
}
- if (sock_ep->rx_shared) {
- fastlock_acquire(&sock_ep->rx_ctx->lock);
- dlist_remove(&sock_ep->rx_ctx_entry);
- fastlock_release(&sock_ep->rx_ctx->lock);
+ if (sock_ep->attr->rx_shared) {
+ fastlock_acquire(&sock_ep->attr->rx_ctx->lock);
+ dlist_remove(&sock_ep->attr->rx_ctx_entry);
+ fastlock_release(&sock_ep->attr->rx_ctx->lock);
}
- pthread_mutex_unlock(&sock_ep->domain->pe->list_lock);
+ pthread_mutex_unlock(&sock_ep->attr->domain->pe->list_lock);
- sock_ep->listener.do_listen = 0;
- if (write(sock_ep->listener.signal_fds[0], &c, 1) != 1)
- SOCK_LOG_DBG("Failed to signal\n");
+ if (sock_ep->attr->listener.do_listen) {
+ sock_ep->attr->listener.do_listen = 0;
+ if (ofi_write_socket(sock_ep->attr->listener.signal_fds[0], &c, 1) != 1)
+ SOCK_LOG_DBG("Failed to signal\n");
- if (sock_ep->listener.listener_thread &&
- pthread_join(sock_ep->listener.listener_thread, NULL)) {
- SOCK_LOG_ERROR("pthread join failed (%d)\n", errno);
+ if (sock_ep->attr->listener.listener_thread &&
+ pthread_join(sock_ep->attr->listener.listener_thread, NULL)) {
+ SOCK_LOG_ERROR("pthread join failed (%d)\n", errno);
+ }
+
+ ofi_close_socket(sock_ep->attr->listener.signal_fds[0]);
+ ofi_close_socket(sock_ep->attr->listener.signal_fds[1]);
}
- close(sock_ep->listener.signal_fds[0]);
- close(sock_ep->listener.signal_fds[1]);
- fastlock_destroy(&sock_ep->cm.lock);
+ fastlock_destroy(&sock_ep->attr->cm.lock);
+
+ if (sock_ep->attr->fclass != FI_CLASS_SEP) {
+ if (!sock_ep->attr->tx_shared)
+ sock_pe_remove_tx_ctx(sock_ep->attr->tx_array[0]);
- if (sock_ep->fclass != FI_CLASS_SEP && !sock_ep->tx_shared) {
- sock_pe_remove_tx_ctx(sock_ep->tx_array[0]);
- sock_tx_ctx_free(sock_ep->tx_array[0]);
+ sock_tx_ctx_close(sock_ep->attr->tx_array[0]);
+ sock_tx_ctx_free(sock_ep->attr->tx_array[0]);
}
- if (sock_ep->fclass != FI_CLASS_SEP && !sock_ep->rx_shared) {
- sock_pe_remove_rx_ctx(sock_ep->rx_array[0]);
- sock_rx_ctx_free(sock_ep->rx_array[0]);
+ if (sock_ep->attr->fclass != FI_CLASS_SEP) {
+ if (!sock_ep->attr->rx_shared)
+ sock_pe_remove_rx_ctx(sock_ep->attr->rx_array[0]);
+
+ sock_rx_ctx_close(sock_ep->attr->rx_array[0]);
+ sock_rx_ctx_free(sock_ep->attr->rx_array[0]);
}
- free(sock_ep->tx_array);
- free(sock_ep->rx_array);
+ free(sock_ep->attr->tx_array);
+ free(sock_ep->attr->rx_array);
- if (sock_ep->src_addr)
- free(sock_ep->src_addr);
- if (sock_ep->dest_addr)
- free(sock_ep->dest_addr);
+ if (sock_ep->attr->src_addr)
+ free(sock_ep->attr->src_addr);
+ if (sock_ep->attr->dest_addr)
+ free(sock_ep->attr->dest_addr);
- sock_fabric_remove_service(sock_ep->domain->fab,
- atoi(sock_ep->listener.service));
+ fastlock_acquire(&sock_ep->attr->domain->pe->lock);
+ idm_reset(&sock_ep->attr->conn_idm);
+ idm_reset(&sock_ep->attr->av_idm);
+ sock_conn_map_destroy(sock_ep->attr);
+ fastlock_release(&sock_ep->attr->domain->pe->lock);
- sock_conn_map_destroy(&sock_ep->cmap);
- atomic_dec(&sock_ep->domain->ref);
- fastlock_destroy(&sock_ep->lock);
+ atomic_dec(&sock_ep->attr->domain->ref);
+ fastlock_destroy(&sock_ep->attr->lock);
+ free(sock_ep->attr);
free(sock_ep);
return 0;
}
@@ -670,6 +729,10 @@ static int sock_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
struct sock_tx_ctx *tx_ctx;
struct sock_rx_ctx *rx_ctx;
+ ret = ofi_ep_bind_valid(&sock_prov, bfid, flags);
+ if (ret)
+ return ret;
+
switch (fid->fclass) {
case FI_CLASS_EP:
ep = container_of(fid, struct sock_ep, ep.fid);
@@ -686,7 +749,7 @@ static int sock_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
switch (bfid->fclass) {
case FI_CLASS_EQ:
eq = container_of(bfid, struct sock_eq, eq.fid);
- ep->eq = eq;
+ ep->attr->eq = eq;
break;
case FI_CLASS_MR:
@@ -694,58 +757,30 @@ static int sock_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
case FI_CLASS_CQ:
cq = container_of(bfid, struct sock_cq, cq_fid.fid);
- if (ep->domain != cq->domain)
+ if (ep->attr->domain != cq->domain)
return -FI_EINVAL;
if (flags & FI_SEND) {
- ep->comp.send_cq = cq;
- if (flags & FI_SELECTIVE_COMPLETION)
- ep->comp.send_cq_event = 1;
- }
-
- if (flags & FI_RECV) {
- ep->comp.recv_cq = cq;
- if (flags & FI_SELECTIVE_COMPLETION)
- ep->comp.recv_cq_event = 1;
- }
-
- if (flags & FI_SEND) {
- for (i = 0; i < ep->ep_attr.tx_ctx_cnt; i++) {
- tx_ctx = ep->tx_array[i];
+ for (i = 0; i < ep->attr->ep_attr.tx_ctx_cnt; i++) {
+ tx_ctx = ep->attr->tx_array[i];
if (!tx_ctx)
continue;
- ret = sock_ctx_bind_cq(&tx_ctx->fid.ctx.fid,
- bfid, flags);
+ ret = sock_ctx_bind_cq(&tx_ctx->fid.ctx.fid, bfid, flags);
if (ret)
return ret;
}
}
if (flags & FI_RECV) {
- for (i = 0; i < ep->ep_attr.rx_ctx_cnt; i++) {
- rx_ctx = ep->rx_array[i];
+ for (i = 0; i < ep->attr->ep_attr.rx_ctx_cnt; i++) {
+ rx_ctx = ep->attr->rx_array[i];
if (!rx_ctx)
continue;
- if (rx_ctx->ctx.fid.fclass == FI_CLASS_SRX_CTX) {
- if (flags & FI_RECV) {
- ep->comp.recv_cq = cq;
- if (flags & FI_SELECTIVE_COMPLETION)
- ep->comp.recv_cq_event = 1;
- }
-
- fastlock_acquire(&cq->list_lock);
- dlist_insert_tail(&rx_ctx->cq_entry,
- &cq->rx_list);
- fastlock_release(&cq->list_lock);
- continue;
- }
-
- ret = sock_ctx_bind_cq(&rx_ctx->ctx.fid,
- bfid, flags);
+ ret = sock_ctx_bind_cq(&rx_ctx->ctx.fid, bfid, flags);
if (ret)
return ret;
}
@@ -754,36 +789,17 @@ static int sock_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
case FI_CLASS_CNTR:
cntr = container_of(bfid, struct sock_cntr, cntr_fid.fid);
- if (ep->domain != cntr->domain)
+ if (ep->attr->domain != cntr->domain)
return -FI_EINVAL;
- if (flags & FI_SEND)
- ep->comp.send_cntr = cntr;
-
- if (flags & FI_RECV)
- ep->comp.recv_cntr = cntr;
-
- if (flags & FI_READ)
- ep->comp.read_cntr = cntr;
-
- if (flags & FI_WRITE)
- ep->comp.write_cntr = cntr;
-
- if (flags & FI_REMOTE_READ)
- ep->comp.rem_read_cntr = cntr;
-
- if (flags & FI_REMOTE_WRITE)
- ep->comp.rem_write_cntr = cntr;
-
if (flags & FI_SEND || flags & FI_WRITE || flags & FI_READ) {
- for (i = 0; i < ep->ep_attr.tx_ctx_cnt; i++) {
- tx_ctx = ep->tx_array[i];
+ for (i = 0; i < ep->attr->ep_attr.tx_ctx_cnt; i++) {
+ tx_ctx = ep->attr->tx_array[i];
if (!tx_ctx)
continue;
- ret = sock_ctx_bind_cntr(&tx_ctx->fid.ctx.fid,
- bfid, flags);
+ ret = sock_ctx_bind_cntr(&tx_ctx->fid.ctx.fid, bfid, flags);
if (ret)
return ret;
}
@@ -791,31 +807,13 @@ static int sock_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
if (flags & FI_RECV || flags & FI_REMOTE_READ ||
flags & FI_REMOTE_WRITE) {
- for (i = 0; i < ep->ep_attr.rx_ctx_cnt; i++) {
- rx_ctx = ep->rx_array[i];
+ for (i = 0; i < ep->attr->ep_attr.rx_ctx_cnt; i++) {
+ rx_ctx = ep->attr->rx_array[i];
if (!rx_ctx)
continue;
- if (rx_ctx->ctx.fid.fclass == FI_CLASS_SRX_CTX) {
- if (flags & FI_RECV)
- rx_ctx->comp.recv_cntr = cntr;
-
- if (flags & FI_REMOTE_READ)
- rx_ctx->comp.rem_read_cntr = cntr;
-
- if (flags & FI_REMOTE_WRITE)
- rx_ctx->comp.rem_write_cntr = cntr;
-
- fastlock_acquire(&cntr->list_lock);
- dlist_insert_tail(&rx_ctx->cntr_entry,
- &cntr->rx_list);
- fastlock_release(&cntr->list_lock);
- continue;
- }
-
- ret = sock_ctx_bind_cntr(&rx_ctx->ctx.fid, bfid,
- flags);
+ ret = sock_ctx_bind_cntr(&rx_ctx->ctx.fid, bfid, flags);
if (ret)
return ret;
}
@@ -824,49 +822,58 @@ static int sock_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
case FI_CLASS_AV:
av = container_of(bfid, struct sock_av, av_fid.fid);
- if (ep->domain != av->domain)
+ if (ep->attr->domain != av->domain)
return -FI_EINVAL;
- ep->av = av;
+ ep->attr->av = av;
atomic_inc(&av->ref);
- if (ep->tx_ctx &&
- ep->tx_ctx->fid.ctx.fid.fclass == FI_CLASS_TX_CTX) {
- ep->tx_ctx->av = av;
+ if (ep->attr->tx_ctx &&
+ ep->attr->tx_ctx->fid.ctx.fid.fclass == FI_CLASS_TX_CTX) {
+ ep->attr->tx_ctx->av = av;
}
- if (ep->rx_ctx &&
- ep->rx_ctx->ctx.fid.fclass == FI_CLASS_RX_CTX)
- ep->rx_ctx->av = av;
+ if (ep->attr->rx_ctx &&
+ ep->attr->rx_ctx->ctx.fid.fclass == FI_CLASS_RX_CTX)
+ ep->attr->rx_ctx->av = av;
- for (i = 0; i < ep->ep_attr.tx_ctx_cnt; i++) {
- if (ep->tx_array[i])
- ep->tx_array[i]->av = av;
+ for (i = 0; i < ep->attr->ep_attr.tx_ctx_cnt; i++) {
+ if (ep->attr->tx_array[i])
+ ep->attr->tx_array[i]->av = av;
}
- for (i = 0; i < ep->ep_attr.rx_ctx_cnt; i++) {
- if (ep->rx_array[i])
- ep->rx_array[i]->av = av;
+ for (i = 0; i < ep->attr->ep_attr.rx_ctx_cnt; i++) {
+ if (ep->attr->rx_array[i])
+ ep->attr->rx_array[i]->av = av;
}
-
+ fastlock_acquire(&av->list_lock);
+ ret = fid_list_insert(&av->ep_list, &ep->attr->lock, &ep->ep.fid);
+ if (ret) {
+ SOCK_LOG_ERROR("Error in adding fid in the EP list\n");
+ fastlock_release(&av->list_lock);
+ return ret;
+ }
+ fastlock_release(&av->list_lock);
break;
case FI_CLASS_STX_CTX:
tx_ctx = container_of(bfid, struct sock_tx_ctx, fid.stx.fid);
fastlock_acquire(&tx_ctx->lock);
- dlist_insert_tail(&ep->tx_ctx_entry, &tx_ctx->ep_list);
+ dlist_insert_tail(&ep->attr->tx_ctx_entry, &tx_ctx->ep_list);
fastlock_release(&tx_ctx->lock);
- ep->tx_ctx = tx_ctx;
- ep->tx_array[0] = tx_ctx;
+
+ ep->attr->tx_ctx->use_shared = 1;
+ ep->attr->tx_ctx->stx_ctx = tx_ctx;
break;
case FI_CLASS_SRX_CTX:
rx_ctx = container_of(bfid, struct sock_rx_ctx, ctx);
fastlock_acquire(&rx_ctx->lock);
- dlist_insert_tail(&ep->rx_ctx_entry, &rx_ctx->ep_list);
+ dlist_insert_tail(&ep->attr->rx_ctx_entry, &rx_ctx->ep_list);
fastlock_release(&rx_ctx->lock);
- ep->rx_ctx = rx_ctx;
- ep->rx_array[0] = rx_ctx;
+
+ ep->attr->rx_ctx->use_shared = 1;
+ ep->attr->rx_ctx->srx_ctx = rx_ctx;
break;
default:
@@ -878,17 +885,18 @@ static int sock_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
static int sock_ep_control(struct fid *fid, int command, void *arg)
{
+ int ret;
struct fid_ep *ep_fid;
struct fi_alias *alias;
- struct sock_ep *ep, *new_ep;
+ struct sock_ep *sock_ep, *new_ep;
switch (fid->fclass) {
case FI_CLASS_EP:
- ep = container_of(fid, struct sock_ep, ep.fid);
+ sock_ep = container_of(fid, struct sock_ep, ep.fid);
break;
case FI_CLASS_SEP:
- ep = container_of(fid, struct sock_ep, ep.fid);
+ sock_ep = container_of(fid, struct sock_ep, ep.fid);
break;
default:
@@ -901,17 +909,31 @@ static int sock_ep_control(struct fid *fid, int command, void *arg)
new_ep = calloc(1, sizeof(*new_ep));
if (!new_ep)
return -FI_ENOMEM;
- *new_ep = *ep;
- new_ep->op_flags = alias->flags;
+
+ memcpy(&new_ep->tx_attr, &sock_ep->tx_attr, sizeof(struct fi_tx_attr));
+ memcpy(&new_ep->rx_attr, &sock_ep->rx_attr, sizeof(struct fi_rx_attr));
+ ret = sock_setopflags(&new_ep->tx_attr, &new_ep->rx_attr,
+ alias->flags);
+ if (ret) {
+ free(new_ep);
+ return -FI_EINVAL;
+ }
+ new_ep->attr = sock_ep->attr;
+ new_ep->is_alias = 1;
+ memcpy(&new_ep->ep, &sock_ep->ep, sizeof(struct fid_ep));
*alias->fid = &new_ep->ep.fid;
+ atomic_inc(&new_ep->attr->ref);
break;
-
case FI_GETOPSFLAG:
- *(uint64_t *) arg = ep->op_flags;
+ ret = sock_getopflags(&sock_ep->tx_attr, &sock_ep->rx_attr, (uint64_t *) arg);
+ if (ret)
+ return -EINVAL;
break;
case FI_SETOPSFLAG:
- ep->op_flags = *(uint64_t *) arg;
- ep->op_flags |= FI_TRANSMIT_COMPLETE;
+ ret = sock_setopflags(&sock_ep->tx_attr, &sock_ep->rx_attr, *(uint64_t *) arg);
+ if (ret)
+ return -FI_EINVAL;
+ break;
break;
case FI_ENABLE:
ep_fid = container_of(fid, struct fid_ep, fid);
@@ -936,53 +958,44 @@ int sock_ep_enable(struct fid_ep *ep)
{
int i;
struct sock_ep *sock_ep;
+ struct sock_tx_ctx *tx_ctx;
+ struct sock_rx_ctx *rx_ctx;
sock_ep = container_of(ep, struct sock_ep, ep);
-
- if (sock_ep->tx_ctx &&
- sock_ep->tx_ctx->fid.ctx.fid.fclass == FI_CLASS_TX_CTX) {
- sock_ep->tx_ctx->enabled = 1;
- if (!sock_ep->tx_ctx->progress) {
- sock_pe_add_tx_ctx(sock_ep->domain->pe, sock_ep->tx_ctx);
- sock_ep->tx_ctx->progress = 1;
- }
- }
-
- if (sock_ep->rx_ctx &&
- sock_ep->rx_ctx->ctx.fid.fclass == FI_CLASS_RX_CTX) {
- sock_ep->rx_ctx->enabled = 1;
- if (!sock_ep->rx_ctx->progress) {
- sock_pe_add_rx_ctx(sock_ep->domain->pe,
- sock_ep->rx_ctx);
- sock_ep->rx_ctx->progress = 1;
- }
- }
-
- for (i = 0; i < sock_ep->ep_attr.tx_ctx_cnt; i++) {
- if (sock_ep->tx_array[i]) {
- sock_ep->tx_array[i]->enabled = 1;
- if (!sock_ep->tx_array[i]->progress) {
- sock_pe_add_tx_ctx(sock_ep->domain->pe,
- sock_ep->tx_array[i]);
- sock_ep->tx_array[i]->progress = 1;
+ for (i = 0; i < sock_ep->attr->ep_attr.tx_ctx_cnt; i++) {
+ tx_ctx = sock_ep->attr->tx_array[i];
+ if (tx_ctx) {
+ tx_ctx->enabled = 1;
+ if (tx_ctx->use_shared) {
+ if (tx_ctx->stx_ctx) {
+ sock_pe_add_tx_ctx(tx_ctx->domain->pe, tx_ctx->stx_ctx);
+ tx_ctx->stx_ctx->enabled = 1;
+ }
+ } else {
+ sock_pe_add_tx_ctx(tx_ctx->domain->pe, tx_ctx);
}
}
}
- for (i = 0; i < sock_ep->ep_attr.rx_ctx_cnt; i++) {
- if (sock_ep->rx_array[i]) {
- sock_ep->rx_array[i]->enabled = 1;
- if (!sock_ep->rx_array[i]->progress) {
- sock_pe_add_rx_ctx(sock_ep->domain->pe,
- sock_ep->rx_array[i]);
- sock_ep->rx_array[i]->progress = 1;
+ for (i = 0; i < sock_ep->attr->ep_attr.rx_ctx_cnt; i++) {
+ rx_ctx = sock_ep->attr->rx_array[i];
+ if (rx_ctx) {
+ rx_ctx->enabled = 1;
+ if (rx_ctx->use_shared) {
+ if (rx_ctx->srx_ctx) {
+ sock_pe_add_rx_ctx(rx_ctx->domain->pe, rx_ctx->srx_ctx);
+ rx_ctx->srx_ctx->enabled = 1;
+ }
+ } else {
+ sock_pe_add_rx_ctx(rx_ctx->domain->pe, rx_ctx);
}
}
}
- if (sock_ep->ep_type != FI_EP_MSG &&
- !sock_ep->listener.listener_thread && sock_conn_listen(sock_ep))
+ if (sock_ep->attr->ep_type != FI_EP_MSG &&
+ !sock_ep->attr->listener.listener_thread && sock_conn_listen(sock_ep->attr))
SOCK_LOG_ERROR("cannot start connection thread\n");
+ sock_ep->attr->is_enabled = 1;
return 0;
}
@@ -993,26 +1006,26 @@ int sock_ep_disable(struct fid_ep *ep)
sock_ep = container_of(ep, struct sock_ep, ep);
- if (sock_ep->tx_ctx &&
- sock_ep->tx_ctx->fid.ctx.fid.fclass == FI_CLASS_TX_CTX) {
- sock_ep->tx_ctx->enabled = 0;
+ if (sock_ep->attr->tx_ctx &&
+ sock_ep->attr->tx_ctx->fid.ctx.fid.fclass == FI_CLASS_TX_CTX) {
+ sock_ep->attr->tx_ctx->enabled = 0;
}
- if (sock_ep->rx_ctx &&
- sock_ep->rx_ctx->ctx.fid.fclass == FI_CLASS_RX_CTX) {
- sock_ep->rx_ctx->enabled = 0;
+ if (sock_ep->attr->rx_ctx &&
+ sock_ep->attr->rx_ctx->ctx.fid.fclass == FI_CLASS_RX_CTX) {
+ sock_ep->attr->rx_ctx->enabled = 0;
}
- for (i = 0; i < sock_ep->ep_attr.tx_ctx_cnt; i++) {
- if (sock_ep->tx_array[i])
- sock_ep->tx_array[i]->enabled = 0;
+ for (i = 0; i < sock_ep->attr->ep_attr.tx_ctx_cnt; i++) {
+ if (sock_ep->attr->tx_array[i])
+ sock_ep->attr->tx_array[i]->enabled = 0;
}
- for (i = 0; i < sock_ep->ep_attr.rx_ctx_cnt; i++) {
- if (sock_ep->rx_array[i])
- sock_ep->rx_array[i]->enabled = 0;
+ for (i = 0; i < sock_ep->attr->ep_attr.rx_ctx_cnt; i++) {
+ if (sock_ep->attr->rx_array[i])
+ sock_ep->attr->rx_array[i]->enabled = 0;
}
- sock_ep->is_disabled = 1;
+ sock_ep->attr->is_enabled = 0;
return 0;
}
@@ -1027,7 +1040,16 @@ static int sock_ep_getopt(fid_t fid, int level, int optname,
switch (optname) {
case FI_OPT_MIN_MULTI_RECV:
- *(size_t *)optval = sock_ep->min_multi_recv;
+ *(size_t *)optval = sock_ep->attr->min_multi_recv;
+ *optlen = sizeof(size_t);
+ break;
+
+ case FI_OPT_CM_DATA_SIZE:
+ if (*optlen < sizeof(size_t)) {
+ *optlen = sizeof(size_t);
+ return -FI_ETOOSMALL;
+ }
+ *((size_t *) optval) = SOCK_EP_MAX_CM_DATA_SZ;
*optlen = sizeof(size_t);
break;
@@ -1050,11 +1072,11 @@ static int sock_ep_setopt(fid_t fid, int level, int optname,
switch (optname) {
case FI_OPT_MIN_MULTI_RECV:
- sock_ep->min_multi_recv = *(size_t *)optval;
- for (i = 0; i < sock_ep->ep_attr.rx_ctx_cnt; i++) {
- if (sock_ep->rx_array[i] != NULL) {
- sock_ep->rx_array[i]->min_multi_recv =
- sock_ep->min_multi_recv;
+ sock_ep->attr->min_multi_recv = *(size_t *)optval;
+ for (i = 0; i < sock_ep->attr->ep_attr.rx_ctx_cnt; i++) {
+ if (sock_ep->attr->rx_array[i] != NULL) {
+ sock_ep->attr->rx_array[i]->min_multi_recv =
+ sock_ep->attr->min_multi_recv;
}
}
break;
@@ -1072,19 +1094,25 @@ static int sock_ep_tx_ctx(struct fid_ep *ep, int index, struct fi_tx_attr *attr,
struct sock_tx_ctx *tx_ctx;
sock_ep = container_of(ep, struct sock_ep, ep);
- if (sock_ep->fclass != FI_CLASS_SEP ||
- index >= sock_ep->ep_attr.tx_ctx_cnt)
+ if (sock_ep->attr->fclass != FI_CLASS_SEP ||
+ index >= sock_ep->attr->ep_attr.tx_ctx_cnt)
return -FI_EINVAL;
- tx_ctx = sock_tx_ctx_alloc(&sock_ep->tx_attr, context);
+ if (attr) {
+ if (fi_check_tx_attr(&sock_prov, &sock_ep->tx_attr, attr))
+ return -FI_ENODATA;
+ tx_ctx = sock_tx_ctx_alloc(attr, context, 0);
+ } else {
+ tx_ctx = sock_tx_ctx_alloc(&sock_ep->tx_attr, context, 0);
+ }
if (!tx_ctx)
return -FI_ENOMEM;
tx_ctx->tx_id = index;
- tx_ctx->ep = sock_ep;
- tx_ctx->domain = sock_ep->domain;
- tx_ctx->av = sock_ep->av;
- dlist_insert_tail(&sock_ep->tx_ctx_entry, &tx_ctx->ep_list);
+ tx_ctx->ep_attr = sock_ep->attr;
+ tx_ctx->domain = sock_ep->attr->domain;
+ tx_ctx->av = sock_ep->attr->av;
+ dlist_insert_tail(&sock_ep->attr->tx_ctx_entry, &tx_ctx->ep_list);
tx_ctx->fid.ctx.fid.ops = &sock_ctx_ops;
tx_ctx->fid.ctx.ops = &sock_ctx_ep_ops;
@@ -1094,9 +1122,9 @@ static int sock_ep_tx_ctx(struct fid_ep *ep, int index, struct fi_tx_attr *attr,
tx_ctx->fid.ctx.atomic = &sock_ep_atomic;
*tx_ep = &tx_ctx->fid.ctx;
- sock_ep->tx_array[index] = tx_ctx;
- atomic_inc(&sock_ep->num_tx_ctx);
- atomic_inc(&sock_ep->domain->ref);
+ sock_ep->attr->tx_array[index] = tx_ctx;
+ atomic_inc(&sock_ep->attr->num_tx_ctx);
+ atomic_inc(&sock_ep->attr->domain->ref);
return 0;
}
@@ -1107,30 +1135,36 @@ static int sock_ep_rx_ctx(struct fid_ep *ep, int index, struct fi_rx_attr *attr,
struct sock_rx_ctx *rx_ctx;
sock_ep = container_of(ep, struct sock_ep, ep);
- if (sock_ep->fclass != FI_CLASS_SEP ||
- index >= sock_ep->ep_attr.rx_ctx_cnt)
+ if (sock_ep->attr->fclass != FI_CLASS_SEP ||
+ index >= sock_ep->attr->ep_attr.rx_ctx_cnt)
return -FI_EINVAL;
- rx_ctx = sock_rx_ctx_alloc(attr ? attr : &sock_ep->rx_attr, context);
+ if (attr) {
+ if (fi_check_rx_attr(&sock_prov, &sock_ep->rx_attr, attr))
+ return -FI_ENODATA;
+ rx_ctx = sock_rx_ctx_alloc(attr, context, 0);
+ } else {
+ rx_ctx = sock_rx_ctx_alloc(&sock_ep->rx_attr, context, 0);
+ }
if (!rx_ctx)
return -FI_ENOMEM;
rx_ctx->rx_id = index;
- rx_ctx->ep = sock_ep;
- rx_ctx->domain = sock_ep->domain;
- rx_ctx->av = sock_ep->av;
- dlist_insert_tail(&sock_ep->rx_ctx_entry, &rx_ctx->ep_list);
+ rx_ctx->ep_attr = sock_ep->attr;
+ rx_ctx->domain = sock_ep->attr->domain;
+ rx_ctx->av = sock_ep->attr->av;
+ dlist_insert_tail(&sock_ep->attr->rx_ctx_entry, &rx_ctx->ep_list);
rx_ctx->ctx.fid.ops = &sock_ctx_ops;
rx_ctx->ctx.ops = &sock_ctx_ep_ops;
rx_ctx->ctx.msg = &sock_ep_msg_ops;
rx_ctx->ctx.tagged = &sock_ep_tagged;
- rx_ctx->min_multi_recv = sock_ep->min_multi_recv;
+ rx_ctx->min_multi_recv = sock_ep->attr->min_multi_recv;
*rx_ep = &rx_ctx->ctx;
- sock_ep->rx_array[index] = rx_ctx;
- atomic_inc(&sock_ep->num_rx_ctx);
- atomic_inc(&sock_ep->domain->ref);
+ sock_ep->attr->rx_array[index] = rx_ctx;
+ atomic_inc(&sock_ep->attr->num_rx_ctx);
+ atomic_inc(&sock_ep->attr->domain->ref);
return 0;
}
@@ -1153,7 +1187,7 @@ static int sock_verify_tx_attr(const struct fi_tx_attr *attr)
if (attr->inject_size > SOCK_EP_MAX_INJECT_SZ)
return -FI_ENODATA;
- if (attr->size > SOCK_EP_TX_SZ)
+ if (sock_get_tx_size(attr->size) > sock_get_tx_size(SOCK_EP_TX_SZ))
return -FI_ENODATA;
if (attr->iov_limit > SOCK_EP_MAX_IOV_LIMIT)
@@ -1203,7 +1237,7 @@ static int sock_verify_rx_attr(const struct fi_rx_attr *attr)
if (attr->total_buffered_recv > SOCK_EP_MAX_BUFF_RECV)
return -FI_ENODATA;
- if (attr->size > SOCK_EP_TX_SZ)
+ if (sock_get_tx_size(attr->size) > sock_get_tx_size(SOCK_EP_TX_SZ))
return -FI_ENODATA;
if (attr->iov_limit > SOCK_EP_MAX_IOV_LIMIT)
@@ -1222,7 +1256,7 @@ int sock_srx_ctx(struct fid_domain *domain,
return -FI_EINVAL;
dom = container_of(domain, struct sock_domain, dom_fid);
- rx_ctx = sock_rx_ctx_alloc(attr ? attr : &sock_srx_attr, context);
+ rx_ctx = sock_rx_ctx_alloc(attr ? attr : &sock_srx_attr, context, 0);
if (!rx_ctx)
return -FI_ENOMEM;
@@ -1233,6 +1267,7 @@ int sock_srx_ctx(struct fid_domain *domain,
rx_ctx->ctx.ops = &sock_ctx_ep_ops;
rx_ctx->ctx.msg = &sock_ep_msg_ops;
rx_ctx->ctx.tagged = &sock_ep_tagged;
+ rx_ctx->enabled = 1;
/* default config */
rx_ctx->min_multi_recv = SOCK_EP_MIN_MULTI_RECV;
@@ -1241,7 +1276,91 @@ int sock_srx_ctx(struct fid_domain *domain,
return 0;
}
-static void sock_set_fabric_attr(const struct fi_fabric_attr *hint_attr,
+int sock_get_prefix_len(uint32_t net_addr)
+{
+ int count = 0;
+ while (net_addr > 0) {
+ net_addr = net_addr >> 1;
+ count++;
+ }
+ return count;
+}
+
+#if HAVE_GETIFADDRS
+char *sock_get_fabric_name(struct sockaddr_in *src_addr)
+{
+ int ret;
+ struct ifaddrs *ifaddrs, *ifa;
+ char *fabric_name = NULL;
+ struct in_addr net_in_addr;
+ struct sockaddr_in *host_addr, *net_addr;
+ char netbuf[SOCK_MAX_NETWORK_ADDR_SZ];
+ int prefix_len;
+
+ ret = getifaddrs(&ifaddrs);
+ if (ret)
+ return NULL;
+
+ for (ifa = ifaddrs; ifa != NULL; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr == NULL || !(ifa->ifa_flags & IFF_UP) ||
+ (ifa->ifa_addr->sa_family != AF_INET))
+ continue;
+ if (ofi_equals_ipaddr((struct sockaddr_in *)ifa->ifa_addr, src_addr)) {
+ host_addr = (struct sockaddr_in *)ifa->ifa_addr;
+ net_addr = (struct sockaddr_in *)ifa->ifa_netmask;
+ /* set fabric name to the network_adress in the format of a.b.c.d/e */
+ net_in_addr.s_addr = (uint32_t)((uint32_t) host_addr->sin_addr.s_addr &
+ (uint32_t) net_addr->sin_addr.s_addr);
+ inet_ntop(host_addr->sin_family, (void *)&(net_in_addr), netbuf,
+ sizeof(netbuf));
+ prefix_len = sock_get_prefix_len(net_addr->sin_addr.s_addr);
+ snprintf(netbuf + strlen(netbuf), sizeof(netbuf) - strlen(netbuf),
+ "%s%d", "/", prefix_len);
+ fabric_name = strdup(netbuf);
+ goto out;
+ }
+ }
+out:
+ freeifaddrs(ifaddrs);
+ return fabric_name;
+}
+
+char *sock_get_domain_name(struct sockaddr_in *src_addr)
+{
+ int ret;
+ struct ifaddrs *ifaddrs, *ifa;
+ char *domain_name = NULL;
+
+ ret = getifaddrs(&ifaddrs);
+ if (ret)
+ return NULL;
+
+ for (ifa = ifaddrs; ifa != NULL; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr == NULL || !(ifa->ifa_flags & IFF_UP) ||
+ (ifa->ifa_addr->sa_family != AF_INET))
+ continue;
+ if (ofi_equals_ipaddr((struct sockaddr_in *)ifa->ifa_addr, src_addr)) {
+ domain_name = strdup(ifa->ifa_name);
+ goto out;
+ }
+ }
+out:
+ freeifaddrs(ifaddrs);
+ return domain_name;
+}
+#else
+char *sock_get_fabric_name(struct sockaddr_in *src_addr)
+{
+ return NULL;
+}
+
+char *sock_get_domain_name(struct sockaddr_in *src_addr)
+{
+ return NULL;
+}
+#endif
+
+static void sock_set_fabric_attr(void *src_addr, const struct fi_fabric_attr *hint_attr,
struct fi_fabric_attr *attr)
{
struct sock_fabric *fabric;
@@ -1253,11 +1372,16 @@ static void sock_set_fabric_attr(const struct fi_fabric_attr *hint_attr,
fabric = sock_fab_list_head();
attr->fabric = fabric ? &fabric->fab_fid : NULL;
}
- attr->name = strdup(sock_fab_name);
+
+ /* reverse lookup network address from node and assign it as fabric name */
+ attr->name = sock_get_fabric_name(src_addr);
+ if (!attr->name)
+ attr->name = strdup(sock_fab_name);
+
attr->prov_name = NULL;
}
-static void sock_set_domain_attr(const struct fi_domain_attr *hint_attr,
+static void sock_set_domain_attr(void *src_addr, const struct fi_domain_attr *hint_attr,
struct fi_domain_attr *attr)
{
struct sock_domain *domain;
@@ -1304,7 +1428,10 @@ static void sock_set_domain_attr(const struct fi_domain_attr *hint_attr,
attr->cq_data_size = sock_domain_attr.cq_data_size;
attr->resource_mgmt = sock_domain_attr.resource_mgmt;
out:
- attr->name = strdup(sock_dom_name);
+ /* reverse lookup interface from node and assign it as domain name */
+ attr->name = sock_get_domain_name(src_addr);
+ if (!attr->name)
+ attr->name = strdup(sock_fab_name);
}
@@ -1324,10 +1451,11 @@ struct fi_info *sock_fi_info(enum fi_ep_type ep_type, struct fi_info *hints,
info->mode = SOCK_MODE;
info->addr_format = FI_SOCKADDR_IN;
- if (src_addr) {
+ if (src_addr)
memcpy(info->src_addr, src_addr, sizeof(struct sockaddr_in));
- info->src_addrlen = sizeof(struct sockaddr_in);
- }
+ else
+ sock_get_src_addr_from_hostname(info->src_addr, NULL);
+ info->src_addrlen = sizeof(struct sockaddr_in);
if (dest_addr) {
info->dest_addr = calloc(1, sizeof(struct sockaddr_in));
@@ -1353,11 +1481,11 @@ struct fi_info *sock_fi_info(enum fi_ep_type ep_type, struct fi_info *hints,
if (hints->handle)
info->handle = hints->handle;
- sock_set_domain_attr(hints->domain_attr, info->domain_attr);
- sock_set_fabric_attr(hints->fabric_attr, info->fabric_attr);
+ sock_set_domain_attr(info->src_addr, hints->domain_attr, info->domain_attr);
+ sock_set_fabric_attr(info->src_addr, hints->fabric_attr, info->fabric_attr);
} else {
- sock_set_domain_attr(NULL, info->domain_attr);
- sock_set_fabric_attr(NULL, info->fabric_attr);
+ sock_set_domain_attr(info->src_addr, NULL, info->domain_attr);
+ sock_set_fabric_attr(info->src_addr, NULL, info->fabric_attr);
}
info->ep_attr->type = ep_type;
@@ -1380,7 +1508,7 @@ int sock_get_src_addr_from_hostname(struct sockaddr_in *src_addr,
ai.ai_family = AF_INET;
ai.ai_socktype = SOCK_STREAM;
- sock_getnodename(hostname, sizeof(hostname));
+ ofi_getnodename(hostname, sizeof(hostname));
ret = getaddrinfo(hostname, service, &ai, &rai);
if (ret) {
SOCK_LOG_DBG("getaddrinfo failed!\n");
@@ -1394,20 +1522,20 @@ int sock_get_src_addr_from_hostname(struct sockaddr_in *src_addr,
static int sock_ep_assign_src_addr(struct sock_ep *sock_ep, struct fi_info *info)
{
- sock_ep->src_addr = calloc(1, sizeof(struct sockaddr_in));
- if (!sock_ep->src_addr)
+ sock_ep->attr->src_addr = calloc(1, sizeof(struct sockaddr_in));
+ if (!sock_ep->attr->src_addr)
return -FI_ENOMEM;
if (info && info->dest_addr)
- return sock_get_src_addr(info->dest_addr, sock_ep->src_addr);
+ return sock_get_src_addr(info->dest_addr, sock_ep->attr->src_addr);
else
- return sock_get_src_addr_from_hostname(sock_ep->src_addr, NULL);
+ return sock_get_src_addr_from_hostname(sock_ep->attr->src_addr, NULL);
}
int sock_alloc_endpoint(struct fid_domain *domain, struct fi_info *info,
struct sock_ep **ep, void *context, size_t fclass)
{
- int ret, flags;
+ int ret;
struct sock_ep *sock_ep;
struct sock_tx_ctx *tx_ctx;
struct sock_rx_ctx *rx_ctx;
@@ -1451,208 +1579,219 @@ int sock_alloc_endpoint(struct fid_domain *domain, struct fi_info *info,
default:
ret = -FI_EINVAL;
- goto err;
+ goto err1;
}
- sock_ep->fclass = fclass;
+ sock_ep->attr = (struct sock_ep_attr *) calloc(1, sizeof(struct sock_ep_attr));
+ if (!sock_ep->attr) {
+ ret = -FI_ENOMEM;
+ goto err1;
+ }
+ sock_ep->attr->fclass = fclass;
*ep = sock_ep;
if (info) {
- sock_ep->info.caps = info->caps;
- sock_ep->info.addr_format = FI_SOCKADDR_IN;
+ sock_ep->attr->info.caps = info->caps;
+ sock_ep->attr->info.addr_format = FI_SOCKADDR_IN;
if (info->ep_attr) {
- sock_ep->ep_type = info->ep_attr->type;
- sock_ep->ep_attr.tx_ctx_cnt = info->ep_attr->tx_ctx_cnt;
- sock_ep->ep_attr.rx_ctx_cnt = info->ep_attr->rx_ctx_cnt;
+ sock_ep->attr->ep_type = info->ep_attr->type;
+ sock_ep->attr->ep_attr.tx_ctx_cnt = info->ep_attr->tx_ctx_cnt;
+ sock_ep->attr->ep_attr.rx_ctx_cnt = info->ep_attr->rx_ctx_cnt;
}
if (info->src_addr) {
- sock_ep->src_addr = calloc(1, sizeof(struct sockaddr_in));
- if (!sock_ep->src_addr) {
+ sock_ep->attr->src_addr = calloc(1, sizeof(struct sockaddr_in));
+ if (!sock_ep->attr->src_addr) {
ret = -FI_ENOMEM;
- goto err;
+ goto err2;
}
- memcpy(sock_ep->src_addr, info->src_addr,
+ memcpy(sock_ep->attr->src_addr, info->src_addr,
sizeof(struct sockaddr_in));
}
if (info->dest_addr) {
- sock_ep->dest_addr = calloc(1, sizeof(struct sockaddr_in));
- if (!sock_ep->dest_addr) {
+ sock_ep->attr->dest_addr = calloc(1, sizeof(struct sockaddr_in));
+ if (!sock_ep->attr->dest_addr) {
ret = -FI_ENOMEM;
- goto err;
+ goto err2;
}
- memcpy(sock_ep->dest_addr, info->dest_addr,
+ memcpy(sock_ep->attr->dest_addr, info->dest_addr,
sizeof(struct sockaddr_in));
}
if (info->tx_attr) {
sock_ep->tx_attr = *info->tx_attr;
- sock_ep->op_flags = info->tx_attr->op_flags;
+ if (!(sock_ep->tx_attr.op_flags & (FI_INJECT_COMPLETE |
+ FI_TRANSMIT_COMPLETE | FI_DELIVERY_COMPLETE)))
+ sock_ep->tx_attr.op_flags |= FI_TRANSMIT_COMPLETE;
sock_ep->tx_attr.size = sock_ep->tx_attr.size ?
- sock_ep->tx_attr.size :
- (SOCK_EP_TX_SZ * SOCK_EP_TX_ENTRY_SZ);
- sock_ep->op_flags |= FI_TRANSMIT_COMPLETE;
+ sock_ep->tx_attr.size : SOCK_EP_TX_SZ;
}
- if (info->rx_attr) {
+ if (info->rx_attr)
sock_ep->rx_attr = *info->rx_attr;
- sock_ep->op_flags |= info->rx_attr->op_flags;
- }
- sock_ep->info.handle = info->handle;
+ sock_ep->attr->info.handle = info->handle;
}
- if (!sock_ep->src_addr && sock_ep_assign_src_addr(sock_ep, info)) {
+ if (!sock_ep->attr->src_addr && sock_ep_assign_src_addr(sock_ep, info)) {
SOCK_LOG_ERROR("failed to get src_address\n");
ret = -FI_EINVAL;
- goto err;
+ goto err2;
}
- atomic_initialize(&sock_ep->ref, 0);
- atomic_initialize(&sock_ep->num_tx_ctx, 0);
- atomic_initialize(&sock_ep->num_rx_ctx, 0);
- fastlock_init(&sock_ep->lock);
- dlist_init(&sock_ep->conn_list);
+ atomic_initialize(&sock_ep->attr->ref, 0);
+ atomic_initialize(&sock_ep->attr->num_tx_ctx, 0);
+ atomic_initialize(&sock_ep->attr->num_rx_ctx, 0);
+ fastlock_init(&sock_ep->attr->lock);
- if (sock_ep->ep_attr.tx_ctx_cnt == FI_SHARED_CONTEXT)
- sock_ep->tx_shared = 1;
- if (sock_ep->ep_attr.rx_ctx_cnt == FI_SHARED_CONTEXT)
- sock_ep->rx_shared = 1;
+ if (sock_ep->attr->ep_attr.tx_ctx_cnt == FI_SHARED_CONTEXT)
+ sock_ep->attr->tx_shared = 1;
+ if (sock_ep->attr->ep_attr.rx_ctx_cnt == FI_SHARED_CONTEXT)
+ sock_ep->attr->rx_shared = 1;
- if (sock_ep->fclass != FI_CLASS_SEP) {
- sock_ep->ep_attr.tx_ctx_cnt = 1;
- sock_ep->ep_attr.rx_ctx_cnt = 1;
+ if (sock_ep->attr->fclass != FI_CLASS_SEP) {
+ sock_ep->attr->ep_attr.tx_ctx_cnt = 1;
+ sock_ep->attr->ep_attr.rx_ctx_cnt = 1;
}
- sock_ep->tx_array = calloc(sock_ep->ep_attr.tx_ctx_cnt,
+ sock_ep->attr->tx_array = calloc(sock_ep->attr->ep_attr.tx_ctx_cnt,
sizeof(struct sock_tx_ctx *));
- if (!sock_ep->tx_array) {
+ if (!sock_ep->attr->tx_array) {
ret = -FI_ENOMEM;
- goto err;
+ goto err2;
}
- sock_ep->rx_array = calloc(sock_ep->ep_attr.rx_ctx_cnt,
+ sock_ep->attr->rx_array = calloc(sock_ep->attr->ep_attr.rx_ctx_cnt,
sizeof(struct sock_rx_ctx *));
- if (!sock_ep->rx_array) {
+ if (!sock_ep->attr->rx_array) {
ret = -FI_ENOMEM;
- goto err;
+ goto err2;
}
- if (sock_ep->fclass != FI_CLASS_SEP &&
- sock_ep->ep_attr.tx_ctx_cnt != FI_SHARED_CONTEXT) {
+ if (sock_ep->attr->fclass != FI_CLASS_SEP) {
/* default tx ctx */
- tx_ctx = sock_tx_ctx_alloc(&sock_ep->tx_attr, context);
+ tx_ctx = sock_tx_ctx_alloc(&sock_ep->tx_attr, context,
+ sock_ep->attr->tx_shared);
if (!tx_ctx) {
ret = -FI_ENOMEM;
- goto err;
+ goto err2;
}
- tx_ctx->ep = sock_ep;
+ tx_ctx->ep_attr = sock_ep->attr;
tx_ctx->domain = sock_dom;
tx_ctx->tx_id = 0;
- dlist_insert_tail(&sock_ep->tx_ctx_entry, &tx_ctx->ep_list);
- sock_ep->tx_array[0] = tx_ctx;
- sock_ep->tx_ctx = tx_ctx;
- }
+ dlist_insert_tail(&sock_ep->attr->tx_ctx_entry, &tx_ctx->ep_list);
+ sock_ep->attr->tx_array[0] = tx_ctx;
+ sock_ep->attr->tx_ctx = tx_ctx;
- if (sock_ep->fclass != FI_CLASS_SEP &&
- sock_ep->ep_attr.rx_ctx_cnt != FI_SHARED_CONTEXT) {
/* default rx_ctx */
- rx_ctx = sock_rx_ctx_alloc(&sock_ep->rx_attr, context);
+ rx_ctx = sock_rx_ctx_alloc(&sock_ep->rx_attr, context,
+ sock_ep->attr->rx_shared);
if (!rx_ctx) {
ret = -FI_ENOMEM;
- goto err;
+ goto err2;
}
- rx_ctx->ep = sock_ep;
+ rx_ctx->ep_attr = sock_ep->attr;
rx_ctx->domain = sock_dom;
rx_ctx->rx_id = 0;
- dlist_insert_tail(&sock_ep->rx_ctx_entry, &rx_ctx->ep_list);
- sock_ep->rx_array[0] = rx_ctx;
- sock_ep->rx_ctx = rx_ctx;
+ dlist_insert_tail(&sock_ep->attr->rx_ctx_entry, &rx_ctx->ep_list);
+ sock_ep->attr->rx_array[0] = rx_ctx;
+ sock_ep->attr->rx_ctx = rx_ctx;
}
/* default config */
- sock_ep->min_multi_recv = SOCK_EP_MIN_MULTI_RECV;
+ sock_ep->attr->min_multi_recv = SOCK_EP_MIN_MULTI_RECV;
if (info)
- memcpy(&sock_ep->info, info, sizeof(struct fi_info));
+ memcpy(&sock_ep->attr->info, info, sizeof(struct fi_info));
- sock_ep->domain = sock_dom;
- fastlock_init(&sock_ep->cm.lock);
- if (sock_ep->ep_type == FI_EP_MSG) {
- dlist_init(&sock_ep->cm.msg_list);
+ sock_ep->attr->domain = sock_dom;
+ fastlock_init(&sock_ep->attr->cm.lock);
+ if (sock_ep->attr->ep_type == FI_EP_MSG) {
+ dlist_init(&sock_ep->attr->cm.msg_list);
if (socketpair(AF_UNIX, SOCK_STREAM, 0,
- sock_ep->cm.signal_fds) < 0) {
+ sock_ep->attr->cm.signal_fds) < 0) {
ret = -FI_EINVAL;
- goto err;
+ goto err2;
}
- flags = fcntl(sock_ep->cm.signal_fds[1], F_GETFL, 0);
- if (fcntl(sock_ep->cm.signal_fds[1], F_SETFL, flags | O_NONBLOCK))
- SOCK_LOG_ERROR("fcntl failed");
+ if (fi_fd_nonblock(sock_ep->attr->cm.signal_fds[1]))
+ SOCK_LOG_ERROR("fi_fd_nonblock failed");
}
if (sock_conn_map_init(sock_ep, sock_cm_def_map_sz)) {
SOCK_LOG_ERROR("failed to init connection map: %s\n", strerror(errno));
ret = -FI_EINVAL;
- goto err;
+ goto err2;
}
atomic_inc(&sock_dom->ref);
return 0;
-err:
- if (sock_ep->src_addr)
- free(sock_ep->src_addr);
- if (sock_ep->dest_addr)
- free(sock_ep->dest_addr);
+err2:
+ if (sock_ep->attr) {
+ free(sock_ep->attr->src_addr);
+ free(sock_ep->attr->dest_addr);
+ free(sock_ep->attr);
+ }
+err1:
free(sock_ep);
return ret;
}
-struct sock_conn *sock_ep_lookup_conn(struct sock_ep *ep, fi_addr_t index,
+void sock_ep_remove_conn(struct sock_ep_attr *attr, struct sock_conn *conn)
+{
+ sock_pe_poll_del(attr->domain->pe, conn->sock_fd);
+ idm_clear(&attr->conn_idm, conn->sock_fd);
+ sock_conn_release_entry(&attr->cmap, conn);
+}
+
+struct sock_conn *sock_ep_lookup_conn(struct sock_ep_attr *attr, fi_addr_t index,
struct sockaddr_in *addr)
{
int i;
uint16_t idx;
struct sock_conn *conn;
- idx = (ep->ep_type == FI_EP_MSG) ? index : index & ep->av->mask;
- conn = idm_lookup(&ep->av_idm, idx);
- if (conn && conn != SOCK_CM_CONN_IN_PROGRESS) {
- assert(sock_compare_addr(&conn->addr, addr));
+ idx = (attr->ep_type == FI_EP_MSG) ? index : index & attr->av->mask;
+
+ conn = idm_lookup(&attr->av_idm, idx);
+ if (conn && conn != SOCK_CM_CONN_IN_PROGRESS)
return conn;
- }
- for (i = 0; i < ep->cmap.used; i++) {
- if (sock_compare_addr(&ep->cmap.table[i].addr, addr))
- return &ep->cmap.table[i];
+ for (i = 0; i < attr->cmap.used; i++) {
+ if (!attr->cmap.table[i].connected)
+ continue;
+
+ if (ofi_equals_sockaddr(&attr->cmap.table[i].addr, addr))
+ return &attr->cmap.table[i];
}
return conn;
}
-int sock_ep_get_conn(struct sock_ep *ep, struct sock_tx_ctx *tx_ctx,
+int sock_ep_get_conn(struct sock_ep_attr *attr, struct sock_tx_ctx *tx_ctx,
fi_addr_t index, struct sock_conn **pconn)
{
struct sock_conn *conn;
- uint64_t av_index = (ep->ep_type == FI_EP_MSG) ? 0 : (index & ep->av->mask);
+ uint64_t av_index = (attr->ep_type == FI_EP_MSG) ? 0 : (index & attr->av->mask);
struct sockaddr_in *addr;
- if (ep->ep_type == FI_EP_MSG)
- addr = ep->dest_addr;
+ if (attr->ep_type == FI_EP_MSG)
+ addr = attr->dest_addr;
else
- addr = (struct sockaddr_in *)&ep->av->table[av_index].addr;
+ addr = (struct sockaddr_in *)&attr->av->table[av_index].addr;
- fastlock_acquire(&ep->cmap.lock);
- conn = sock_ep_lookup_conn(ep, av_index, addr);
+ fastlock_acquire(&attr->cmap.lock);
+ conn = sock_ep_lookup_conn(attr, av_index, addr);
if (!conn) {
conn = SOCK_CM_CONN_IN_PROGRESS;
- idm_set(&ep->av_idm, av_index, conn);
+ if (idm_set(&attr->av_idm, av_index, conn) < 0)
+ SOCK_LOG_ERROR("idm_set failed\n");
}
- fastlock_release(&ep->cmap.lock);
+ fastlock_release(&attr->cmap.lock);
if (conn == SOCK_CM_CONN_IN_PROGRESS)
- conn = sock_ep_connect(ep, av_index);
+ conn = sock_ep_connect(attr, av_index);
if (!conn) {
SOCK_LOG_ERROR("Error in connecting: %s\n", strerror(errno));
@@ -1663,5 +1802,5 @@ int sock_ep_get_conn(struct sock_ep *ep, struct sock_tx_ctx *tx_ctx,
}
*pconn = conn;
- return conn->address_published ? 0 : sock_conn_send_src_addr(ep, tx_ctx, conn);
+ return conn->address_published ? 0 : sock_conn_send_src_addr(attr, tx_ctx, conn);
}
diff --git a/prov/sockets/src/sock_ep_dgram.c b/prov/sockets/src/sock_ep_dgram.c
index cb361c9..56d6ce2 100644
--- a/prov/sockets/src/sock_ep_dgram.c
+++ b/prov/sockets/src/sock_ep_dgram.c
@@ -109,7 +109,8 @@ static int sock_dgram_verify_rx_attr(const struct fi_rx_attr *attr)
if (attr->total_buffered_recv > sock_dgram_rx_attr.total_buffered_recv)
return -FI_ENODATA;
- if (attr->size > sock_dgram_rx_attr.size)
+ if (sock_get_tx_size(attr->size) >
+ sock_get_tx_size(sock_dgram_rx_attr.size))
return -FI_ENODATA;
if (attr->iov_limit > sock_dgram_rx_attr.iov_limit)
@@ -132,7 +133,8 @@ static int sock_dgram_verify_tx_attr(const struct fi_tx_attr *attr)
if (attr->inject_size > sock_dgram_tx_attr.inject_size)
return -FI_ENODATA;
- if (attr->size > sock_dgram_tx_attr.size)
+ if (sock_get_tx_size(attr->size) >
+ sock_get_tx_size(sock_dgram_tx_attr.size))
return -FI_ENODATA;
if (attr->iov_limit > sock_dgram_tx_attr.iov_limit)
@@ -203,7 +205,9 @@ int sock_dgram_fi_info(void *src_addr, void *dest_addr, struct fi_info *hints,
return -FI_ENOMEM;
*(*info)->tx_attr = sock_dgram_tx_attr;
+ (*info)->tx_attr->size = sock_get_tx_size(sock_dgram_tx_attr.size);
*(*info)->rx_attr = sock_dgram_rx_attr;
+ (*info)->rx_attr->size = sock_get_tx_size(sock_dgram_rx_attr.size);
*(*info)->ep_attr = sock_dgram_ep_attr;
if (hints && hints->ep_attr) {
@@ -271,7 +275,7 @@ static int sock_dgram_endpoint(struct fid_domain *domain, struct fi_info *info,
return ret;
if (!info || !info->ep_attr)
- (*ep)->ep_attr = sock_dgram_ep_attr;
+ (*ep)->attr->ep_attr = sock_dgram_ep_attr;
if (!info || !info->tx_attr)
(*ep)->tx_attr = sock_dgram_tx_attr;
diff --git a/prov/sockets/src/sock_ep_msg.c b/prov/sockets/src/sock_ep_msg.c
index 28060fa..a6d8c65 100644
--- a/prov/sockets/src/sock_ep_msg.c
+++ b/prov/sockets/src/sock_ep_msg.c
@@ -111,7 +111,8 @@ static int sock_msg_verify_rx_attr(const struct fi_rx_attr *attr)
if (attr->total_buffered_recv > sock_msg_rx_attr.total_buffered_recv)
return -FI_ENODATA;
- if (attr->size > sock_msg_rx_attr.size)
+ if (sock_get_tx_size(attr->size) >
+ sock_get_tx_size(sock_msg_rx_attr.size))
return -FI_ENODATA;
if (attr->iov_limit > sock_msg_rx_attr.iov_limit)
@@ -134,7 +135,8 @@ static int sock_msg_verify_tx_attr(const struct fi_tx_attr *attr)
if (attr->inject_size > sock_msg_tx_attr.inject_size)
return -FI_ENODATA;
- if (attr->size > sock_msg_tx_attr.size)
+ if (sock_get_tx_size(attr->size) >
+ sock_get_tx_size(sock_msg_tx_attr.size))
return -FI_ENODATA;
if (attr->iov_limit > sock_msg_tx_attr.iov_limit)
@@ -204,7 +206,9 @@ int sock_msg_fi_info(void *src_addr, void *dest_addr, struct fi_info *hints,
return -FI_ENOMEM;
*(*info)->tx_attr = sock_msg_tx_attr;
+ (*info)->tx_attr->size = sock_get_tx_size(sock_msg_tx_attr.size);
*(*info)->rx_attr = sock_msg_rx_attr;
+ (*info)->rx_attr->size = sock_get_tx_size(sock_msg_rx_attr.size);
*(*info)->ep_attr = sock_msg_ep_attr;
if (hints && hints->ep_attr) {
@@ -251,10 +255,14 @@ static int sock_ep_cm_getname(fid_t fid, void *addr, size_t *addrlen)
case FI_CLASS_EP:
case FI_CLASS_SEP:
sock_ep = container_of(fid, struct sock_ep, ep.fid);
- memcpy(addr, sock_ep->src_addr, len);
+ if (sock_ep->attr->is_enabled == 0)
+ return -FI_EOPBADSTATE;
+ memcpy(addr, sock_ep->attr->src_addr, len);
break;
case FI_CLASS_PEP:
sock_pep = container_of(fid, struct sock_pep, pep.fid);
+ if (!sock_pep->name_set)
+ return -FI_EOPBADSTATE;
memcpy(addr, &sock_pep->src_addr, len);
break;
default:
@@ -268,7 +276,7 @@ static int sock_ep_cm_getname(fid_t fid, void *addr, size_t *addrlen)
static int sock_pep_create_listener(struct sock_pep *pep)
{
- int optval, ret;
+ int ret;
socklen_t addr_size;
struct sockaddr_in addr;
struct addrinfo *s_res = NULL, *p;
@@ -277,12 +285,10 @@ static int sock_pep_create_listener(struct sock_pep *pep)
char sa_port[NI_MAXSERV] = {0};
pep->cm.do_listen = 1;
-
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_INET;
- hints.ai_socktype = SOCK_DGRAM;
+ hints.ai_socktype = SOCK_STREAM;
hints.ai_flags = AI_PASSIVE;
- hints.ai_protocol = IPPROTO_UDP;
memcpy(sa_ip, inet_ntoa(pep->src_addr.sin_addr), INET_ADDRSTRLEN);
sprintf(sa_port, "%d", ntohs(pep->src_addr.sin_port));
@@ -294,31 +300,27 @@ static int sock_pep_create_listener(struct sock_pep *pep)
return -FI_EINVAL;
}
+ SOCK_LOG_DBG("binding pep listener to %s\n", sa_port);
for (p = s_res; p; p = p->ai_next) {
pep->cm.sock = socket(p->ai_family, p->ai_socktype,
p->ai_protocol);
if (pep->cm.sock >= 0) {
- optval = 1;
- if (setsockopt(pep->cm.sock, SOL_SOCKET, SO_REUSEADDR,
- &optval, sizeof(optval)))
- SOCK_LOG_ERROR("setsockopt failed\n");
-
+ sock_set_sockopts(pep->cm.sock);
if (!bind(pep->cm.sock, s_res->ai_addr, s_res->ai_addrlen))
break;
- close(pep->cm.sock);
+ SOCK_LOG_ERROR("failed to bind listener: %s\n", strerror(errno));
+ ofi_close_socket(pep->cm.sock);
pep->cm.sock = -1;
}
}
freeaddrinfo(s_res);
- if (pep->cm.sock < 0)
+ if (pep->cm.sock < 0) {
+ SOCK_LOG_ERROR("failed to create listener: %s\n", strerror(errno));
return -FI_EIO;
+ }
- optval = 1;
- if (setsockopt(pep->cm.sock, SOL_SOCKET, SO_REUSEADDR, &optval,
- sizeof optval))
- SOCK_LOG_ERROR("setsockopt failed\n");
-
+ sock_set_sockopt_reuseaddr(pep->cm.sock);
if (pep->src_addr.sin_port == 0) {
addr_size = sizeof(addr);
if (getsockname(pep->cm.sock, (struct sockaddr *)&addr, &addr_size))
@@ -333,8 +335,14 @@ static int sock_pep_create_listener(struct sock_pep *pep)
return -FI_EINVAL;
}
+ if (listen(pep->cm.sock, sock_cm_def_map_sz)) {
+ SOCK_LOG_ERROR("failed to listen socket: %s\n", strerror(errno));
+ return -errno;
+ }
+
+ pep->name_set = 1;
SOCK_LOG_DBG("Listener thread bound to %s:%d\n",
- sa_ip, ntohs(pep->src_addr.sin_port));
+ sa_ip, ntohs(pep->src_addr.sin_port));
return 0;
}
@@ -350,10 +358,10 @@ static int sock_ep_cm_setname(fid_t fid, void *addr, size_t addrlen)
case FI_CLASS_EP:
case FI_CLASS_SEP:
sock_ep = container_of(fid, struct sock_ep, ep.fid);
- if (sock_ep->listener.listener_thread)
+ if (sock_ep->attr->listener.listener_thread)
return -FI_EINVAL;
- memcpy(sock_ep->src_addr, addr, addrlen);
- return sock_conn_listen(sock_ep);
+ memcpy(sock_ep->attr->src_addr, addr, addrlen);
+ return sock_conn_listen(sock_ep->attr);
case FI_CLASS_PEP:
sock_pep = container_of(fid, struct sock_pep, pep.fid);
if (sock_pep->cm.listener_thread)
@@ -374,487 +382,351 @@ static int sock_ep_cm_getpeer(struct fid_ep *ep, void *addr, size_t *addrlen)
sock_ep = container_of(ep, struct sock_ep, ep);
len = MIN(*addrlen, sizeof(struct sockaddr_in));
- memcpy(addr, sock_ep->dest_addr, len);
+ memcpy(addr, sock_ep->attr->dest_addr, len);
*addrlen = sizeof(struct sockaddr_in);
return (len == sizeof(struct sockaddr_in)) ? 0 : -FI_ETOOSMALL;
}
-static int sock_ep_cm_create_socket(void)
-{
- int sock, optval;
- sock = socket(AF_INET, SOCK_DGRAM, 0);
- if (sock < 0)
- return 0;
-
- optval = 1;
- if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR,
- &optval, sizeof(optval)))
- SOCK_LOG_ERROR("setsockopt failed\n");
- return sock;
-}
-
-static int sock_ep_cm_enqueue_msg(struct sock_cm_entry *cm,
- const struct sockaddr_in *addr,
- void *msg, size_t len,
- fid_t fid, struct sock_eq *eq)
-{
- char c = 0;
- int ret = 0;
- struct sock_cm_msg_list_entry *list_entry;
-
- list_entry = calloc(1, sizeof(*list_entry) + len);
- if (!list_entry)
- return -FI_ENOMEM;
-
- list_entry->msg_len = len;
- memcpy(&list_entry->msg[0], msg, len);
- memcpy(&list_entry->addr, addr, sizeof(*addr));
- list_entry->fid = fid;
- list_entry->eq = eq;
-
- fastlock_acquire(&cm->lock);
- dlist_insert_tail(&list_entry->entry, &cm->msg_list);
- fastlock_release(&cm->lock);
-
- ret = write(cm->signal_fds[0], &c, 1);
- if (ret != 1) {
- SOCK_LOG_DBG("failed to signal\n");
- ret = -FI_EIO;
- } else {
- ret = 0;
- SOCK_LOG_DBG("Enqueued CM Msg\n");
- }
- return ret;
-}
-
-static int sock_ep_cm_send_msg(struct sock_cm_entry *cm,
- const struct sockaddr_in *addr,
- void *msg, size_t len)
-{
- int ret;
- char sa_ip[INET_ADDRSTRLEN] = {0};
-
- memcpy(sa_ip, inet_ntoa(addr->sin_addr), INET_ADDRSTRLEN);
- SOCK_LOG_DBG("Sending message to %s:%d\n", sa_ip, ntohs(addr->sin_port));
-
- ret = sendto(cm->sock, (char *) msg, len, 0,
- (struct sockaddr *) addr, sizeof(*addr));
- SOCK_LOG_DBG("Total Sent: %d\n", ret);
- return (ret == len) ? 0 : -1;
-}
-
-static void sock_ep_cm_release_entry(struct sock_cm_msg_list_entry *msg_entry)
-{
- struct sock_conn_hdr *msg_hdr;
- struct fi_eq_cm_entry cm_entry;
- struct sock_ep *sock_ep;
-
- msg_hdr = (struct sock_conn_hdr *) msg_entry->msg;
- if (msg_hdr->type == SOCK_CONN_SHUTDOWN) {
- memset(&cm_entry, 0, sizeof(cm_entry));
- cm_entry.fid = msg_entry->fid;
- sock_ep = container_of(cm_entry.fid, struct sock_ep, ep.fid);
- if (sock_eq_report_event(msg_entry->eq, FI_SHUTDOWN, &cm_entry,
- sizeof(cm_entry), 0))
- SOCK_LOG_ERROR("Error in writing to EQ\n");
- sock_ep->cm.shutdown_received = 1;
- sock_ep_disable(&sock_ep->ep);
- } else {
- if (sock_eq_report_error(msg_entry->eq, msg_entry->fid, NULL,
- 0, FI_ETIMEDOUT, -FI_ETIMEDOUT, NULL, 0))
- SOCK_LOG_ERROR("failed to report error\n");
- }
-
- dlist_remove(&msg_entry->entry);
- free(msg_entry);
-}
-
-static void sock_ep_cm_flush_msg(struct sock_cm_entry *cm)
+static int sock_cm_send(int fd, const void *buf, int len)
{
- struct dlist_entry *entry, *next_entry;
- struct sock_cm_msg_list_entry *msg_entry;
- fastlock_acquire(&cm->lock);
- for (entry = cm->msg_list.next; entry != &cm->msg_list;) {
- msg_entry = container_of(entry,
- struct sock_cm_msg_list_entry, entry);
- next_entry = entry->next;
-
- if (msg_entry->timestamp_ms != 0 &&
- fi_gettime_ms() - msg_entry->timestamp_ms < SOCK_CM_COMM_TIMEOUT) {
- entry = next_entry;
- continue;
- }
-
- msg_entry->timestamp_ms = fi_gettime_ms();
- msg_entry->retry++;
-
- if (msg_entry->retry > SOCK_EP_MAX_RETRY) {
- sock_ep_cm_release_entry(msg_entry);
- entry = next_entry;
- continue;
+ int ret, done = 0;
+ while (done != len) {
+ ret = ofi_write_socket(fd, (const char*) buf + done, len - done);
+ if (ret < 0) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK)
+ continue;
+ SOCK_LOG_ERROR("failed to write to fd: %s\n", strerror(errno));
+ return -FI_EIO;
}
-
- if (sock_ep_cm_send_msg(cm, &msg_entry->addr,
- &msg_entry->msg, msg_entry->msg_len))
- SOCK_LOG_DBG("Failed to send out cm message\n");
- entry = next_entry;
+ done += ret;
}
- fastlock_release(&cm->lock);
-}
-
-static int sock_ep_cm_send_ack(struct sock_cm_entry *cm,
- struct sockaddr_in *addr, uint64_t msg_id)
-{
- int ret;
- struct sock_conn_response conn_response;
-
- memset(&conn_response, 0, sizeof(conn_response));
- conn_response.hdr.type = SOCK_CONN_ACK;
- conn_response.hdr.msg_id = msg_id;
-
- ret = sendto(cm->sock, &conn_response, sizeof(conn_response), 0,
- (struct sockaddr *) addr, sizeof(*addr));
- SOCK_LOG_DBG("Total Sent: %d\n", ret);
- sock_ep_cm_flush_msg(cm);
- return (ret == sizeof(conn_response)) ? 0 : -1;
+ return 0;
}
-static void sock_ep_cm_handle_ack(struct sock_cm_entry *cm,
- struct sock_conn_hdr *hdr)
+static int sock_cm_recv(int fd, void *buf, int len)
{
- struct sock_conn_hdr *msg_hdr;
- struct dlist_entry *entry;
- struct sock_cm_msg_list_entry *msg_entry;
- struct fi_eq_cm_entry cm_entry;
- struct sock_ep *sock_ep;
-
- sock_ep = container_of(cm, struct sock_ep, cm);
- fastlock_acquire(&cm->lock);
- for (entry = cm->msg_list.next; entry != &cm->msg_list;) {
-
- msg_entry = container_of(entry, struct sock_cm_msg_list_entry,
- entry);
- msg_hdr = (struct sock_conn_hdr *) msg_entry->msg;
-
- if (msg_hdr->msg_id == hdr->msg_id) {
- switch (msg_hdr->type) {
- case SOCK_CONN_SHUTDOWN:
- SOCK_LOG_DBG("Got ack for SOCK_CONN_SHUTDOWN\n");
- memset(&cm_entry, 0, sizeof(cm_entry));
- cm_entry.fid = &sock_ep->ep.fid;
- if (sock_ep->cm.shutdown_received)
- break;
-
- if (sock_eq_report_event(sock_ep->eq, FI_SHUTDOWN,
- &cm_entry,
- sizeof(cm_entry), 0))
- SOCK_LOG_ERROR("Error in writing to EQ\n");
- break;
-
- case SOCK_CONN_ACCEPT:
- SOCK_LOG_DBG("Got ack for SOCK_CONN_ACCEPT\n");
- memset(&cm_entry, 0, sizeof(cm_entry));
- cm_entry.fid = &sock_ep->ep.fid;
- sock_ep_enable(&sock_ep->ep);
-
- if (sock_eq_report_event(sock_ep->eq,
- FI_CONNECTED, &cm_entry,
- sizeof(cm_entry), 0))
- SOCK_LOG_ERROR("Error in writing to EQ\n");
- break;
-
- default:
- break;
- }
- dlist_remove(entry);
- free(msg_entry);
- break;
+ int ret, done = 0;
+ while (done != len) {
+ ret = recv(fd, (char*) buf + done, len - done, 0);
+ if (ret <= 0) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK)
+ continue;
+ SOCK_LOG_ERROR("failed to read from fd: %s\n", strerror(errno));
+ return -FI_EIO;
}
- entry = entry->next;
+ done += ret;
}
- fastlock_release(&cm->lock);
+ return 0;
}
-
-static void *sock_msg_ep_listener_thread(void *data)
+static void sock_ep_wait_shutdown(struct sock_ep *ep)
{
- struct pollfd poll_fds[2];
- struct sock_ep *ep = (struct sock_ep *)data;
- struct sock_conn_response *conn_response;
- struct fi_eq_cm_entry *cm_entry;
-
- struct sockaddr_in from_addr;
- socklen_t addr_len;
- int ret, user_data_sz, entry_sz, timeout;
+ int ret, do_report = 0;
char tmp = 0;
+ struct pollfd poll_fds[2];
+ struct sock_conn_hdr msg;
+ struct fi_eq_cm_entry cm_entry = {0};
- ep->cm.sock = sock_ep_cm_create_socket();
- if (!ep->cm.sock) {
- SOCK_LOG_ERROR("Cannot open socket\n");
- return NULL;
- }
-
- SOCK_LOG_DBG("Starting listener thread for EP: %p\n", ep);
- conn_response = calloc(1, sizeof(*conn_response) + SOCK_EP_MAX_CM_DATA_SZ);
- if (!conn_response) {
- SOCK_LOG_ERROR("cannot allocate\n");
- return NULL;
- }
-
- cm_entry = calloc(1, sizeof(*cm_entry) + SOCK_EP_MAX_CM_DATA_SZ);
- if (!cm_entry) {
- free(conn_response);
- SOCK_LOG_ERROR("cannot allocate\n");
- return NULL;
- }
-
- poll_fds[0].fd = ep->cm.sock;
- poll_fds[1].fd = ep->cm.signal_fds[1];
+ poll_fds[0].fd = ep->attr->cm.sock;
+ poll_fds[1].fd = ep->attr->cm.signal_fds[1];
poll_fds[0].events = poll_fds[1].events = POLLIN;
- while (*((volatile int*) &ep->cm.do_listen)) {
- timeout = dlist_empty(&ep->cm.msg_list) ? -1 :
- SOCK_CM_COMM_TIMEOUT;
- ret = poll(poll_fds, 2, timeout);
+ while (*((volatile int*) &ep->attr->cm.do_listen)) {
+ ret = poll(poll_fds, 2, -1);
if (ret > 0) {
if (poll_fds[1].revents & POLLIN) {
- ret = read(ep->cm.signal_fds[1], &tmp, 1);
+ ret = ofi_read_socket(ep->attr->cm.signal_fds[1], &tmp, 1);
if (ret != 1) {
SOCK_LOG_DBG("Invalid signal\n");
break;
}
- sock_ep_cm_flush_msg(&ep->cm);
continue;
}
} else {
- if (ret == 0) {
- sock_ep_cm_flush_msg(&ep->cm);
- continue;
- } else {
- break;
- }
+ break;
}
- addr_len = sizeof(from_addr);
- ret = recvfrom(ep->cm.sock, (char *) conn_response,
- sizeof(*conn_response) + SOCK_EP_MAX_CM_DATA_SZ,
- 0, (struct sockaddr *) &from_addr, &addr_len);
- if (ret <= 0)
- continue;
-
- SOCK_LOG_DBG("Total received: %d\n", ret);
-
- if (ret < sizeof(*conn_response))
- continue;
-
- if (conn_response->hdr.type != SOCK_CONN_ACK)
- sock_ep_cm_send_ack(&ep->cm, &from_addr,
- conn_response->hdr.msg_id);
-
- user_data_sz = ret - sizeof(*conn_response);
- switch (conn_response->hdr.type) {
-
- case SOCK_CONN_ACK:
- SOCK_LOG_DBG("Received SOCK_CONN_ACK\n");
- sock_ep_cm_handle_ack(&ep->cm, &conn_response->hdr);
+ if (sock_cm_recv(ep->attr->cm.sock, &msg, sizeof(msg)))
break;
- case SOCK_CONN_ACCEPT:
- SOCK_LOG_DBG("Received SOCK_CONN_ACCEPT\n");
-
- entry_sz = sizeof(*cm_entry) + user_data_sz;
- memset(cm_entry, 0, sizeof(*cm_entry));
- cm_entry->fid = &ep->ep.fid;
-
- memcpy(&ep->cm_addr, &from_addr, sizeof(from_addr));
- memcpy(&cm_entry->data, &conn_response->user_data,
- user_data_sz);
-
- if (ep->is_disabled || ep->cm.shutdown_received)
- break;
+ if (msg.type == SOCK_CONN_SHUTDOWN)
+ break;
+ }
- ((struct sockaddr_in *) ep->dest_addr)->sin_port =
- conn_response->hdr.s_port;
+ fastlock_acquire(&ep->attr->cm.lock);
+ if (ep->attr->cm.is_connected) {
+ do_report = 1;
+ ep->attr->cm.is_connected = 0;
+ }
+ fastlock_release(&ep->attr->cm.lock);
- sock_ep_enable(&ep->ep);
- if (sock_eq_report_event(ep->eq, FI_CONNECTED, cm_entry,
- entry_sz, 0))
- SOCK_LOG_ERROR("Error in writing to EQ\n");
- break;
- case SOCK_CONN_REJECT:
- SOCK_LOG_DBG("Received SOCK_CONN_REJECT\n");
+ if (do_report) {
+ cm_entry.fid = &ep->ep.fid;
+ SOCK_LOG_DBG("reporting FI_SHUTDOWN\n");
+ if (sock_eq_report_event(ep->attr->eq, FI_SHUTDOWN,
+ &cm_entry, sizeof(cm_entry), 0))
+ SOCK_LOG_ERROR("Error in writing to EQ\n");
+ }
+ ofi_close_socket(ep->attr->cm.sock);
+}
- if (ep->is_disabled || ep->cm.shutdown_received)
- break;
+static void sock_ep_cm_report_connect_fail(struct sock_ep *ep,
+ void *param, size_t paramlen)
+{
+ SOCK_LOG_DBG("reporting FI_REJECT\n");
+ if (sock_eq_report_error(ep->attr->eq, &ep->ep.fid, NULL, 0,
+ FI_ECONNREFUSED, -FI_ECONNREFUSED,
+ param, paramlen))
+ SOCK_LOG_ERROR("Error in writing to EQ\n");
+}
- if (sock_eq_report_error(ep->eq, &ep->ep.fid, NULL, 0,
- FI_ECONNREFUSED,
- -FI_ECONNREFUSED,
- &conn_response->user_data,
- user_data_sz))
- SOCK_LOG_ERROR("Error in writing to EQ\n");
- goto out;
+static void *sock_ep_cm_connect_handler(void *data)
+{
+ int sock_fd, ret;
+ struct sock_conn_req_handle *handle = data;
+ struct sock_conn_req *req = handle->req;
+ struct sock_conn_hdr response;
+ struct sock_ep *ep = handle->ep;
+ void *param = NULL;
+ struct fi_eq_cm_entry *cm_entry = NULL;
+ int cm_data_sz, response_port;
+
+ sock_fd = socket(AF_INET, SOCK_STREAM, 0);
+ if (sock_fd < 0) {
+ SOCK_LOG_ERROR("no socket\n");
+ sock_ep_cm_report_connect_fail(handle->ep, NULL, 0);
+ goto out;
+ }
- case SOCK_CONN_SHUTDOWN:
- SOCK_LOG_DBG("Received SOCK_CONN_SHUTDOWN\n");
+ sock_set_sockopts_conn(sock_fd);
+ ret = connect(sock_fd, (struct sockaddr *)&handle->dest_addr,
+ sizeof(handle->dest_addr));
+ if (ret < 0) {
+ SOCK_LOG_ERROR("connect failed : %s\n", strerror(errno));
+ goto err;
+ }
- entry_sz = sizeof(*cm_entry);
- memset(cm_entry, 0, sizeof(*cm_entry));
- cm_entry->fid = &ep->ep.fid;
+ if (sock_cm_send(sock_fd, req, sizeof(*req)))
+ goto err;
+ if (handle->paramlen && sock_cm_send(sock_fd, handle->cm_data, handle->paramlen))
+ goto err;
- if (ep->cm.shutdown_received)
- break;
+ if (sock_cm_recv(sock_fd, &response, sizeof(response)))
+ goto err;
- sock_ep_disable(&ep->ep);
- ep->cm.shutdown_received = 1;
- if (sock_eq_report_event(ep->eq, FI_SHUTDOWN, cm_entry,
- entry_sz, 0))
- SOCK_LOG_ERROR("Error in writing to EQ\n");
- goto out;
+ cm_data_sz = ntohs(response.cm_data_sz);
+ response_port = ntohs(response.port);
+ if (cm_data_sz) {
+ param = calloc(1, cm_data_sz);
+ if (!param)
+ goto err;
- default:
- SOCK_LOG_ERROR("Invalid event: %d\n", conn_response->hdr.type);
- break;
- }
+ if (sock_cm_recv(sock_fd, param, cm_data_sz))
+ goto err;
}
+ if (response.type == SOCK_CONN_REJECT) {
+ sock_ep_cm_report_connect_fail(handle->ep, param, cm_data_sz);
+ ofi_close_socket(sock_fd);
+ } else {
+ cm_entry = calloc(1, sizeof(*cm_entry) + SOCK_EP_MAX_CM_DATA_SZ);
+ if (!cm_entry)
+ goto err;
+
+ cm_entry->fid = &ep->ep.fid;
+ memcpy(&cm_entry->data, param, cm_data_sz);
+ ep->attr->cm.is_connected = 1;
+ ep->attr->cm.do_listen = 1;
+ ep->attr->cm.sock = sock_fd;
+ ep->attr->msg_dest_port = response_port;
+ SOCK_LOG_DBG("got accept - port: %d\n", response_port);
+
+ SOCK_LOG_DBG("Reporting FI_CONNECTED\n");
+ if (sock_eq_report_event(ep->attr->eq, FI_CONNECTED, cm_entry,
+ sizeof(*cm_entry) + cm_data_sz, 0))
+ SOCK_LOG_ERROR("Error in writing to EQ\n");
+ sock_ep_wait_shutdown(ep);
+ }
+ goto out;
+err:
+ SOCK_LOG_ERROR("io failed : %s\n", strerror(errno));
+ sock_ep_cm_report_connect_fail(handle->ep, NULL, 0);
+ ofi_close_socket(sock_fd);
out:
- free(conn_response);
+ free(param);
free(cm_entry);
- close(ep->cm.sock);
- ep->cm.listener_thread = 0L;
+ free(handle->req);
+ free(handle);
return NULL;
}
static int sock_ep_cm_connect(struct fid_ep *ep, const void *addr,
- const void *param, size_t paramlen)
+ const void *param, size_t paramlen)
{
- struct sock_conn_req *req;
+ struct sock_conn_req *req = NULL;
+ struct sock_conn_req_handle *handle = NULL;
struct sock_ep *_ep;
struct sock_eq *_eq;
- int ret = 0;
_ep = container_of(ep, struct sock_ep, ep);
- _eq = _ep->eq;
- if (!_eq || paramlen > SOCK_EP_MAX_CM_DATA_SZ)
+ _eq = _ep->attr->eq;
+ if (!_eq || !addr || (paramlen > SOCK_EP_MAX_CM_DATA_SZ))
return -FI_EINVAL;
- if (!_ep->listener.listener_thread && sock_conn_listen(_ep))
+ if (!_ep->attr->listener.listener_thread && sock_conn_listen(_ep->attr))
return -FI_EINVAL;
- req = calloc(1, sizeof(*req) + paramlen);
+ if (!_ep->attr->dest_addr) {
+ _ep->attr->dest_addr = calloc(1, sizeof(*_ep->attr->dest_addr));
+ if (!_ep->attr->dest_addr)
+ return -FI_ENOMEM;
+ }
+ memcpy(_ep->attr->dest_addr, addr, sizeof(*_ep->attr->dest_addr));
+
+ req = calloc(1, sizeof(*req));
if (!req)
return -FI_ENOMEM;
- ((struct sockaddr *) addr)->sa_family = AF_INET;
+ handle = calloc(1, sizeof(*handle));
+ if (!handle)
+ goto out;
req->hdr.type = SOCK_CONN_REQ;
- req->hdr.msg_id = _ep->cm.next_msg_id++;
- req->info = _ep->info;
- memcpy(&req->src_addr, _ep->src_addr, sizeof(req->src_addr));
- memcpy(&req->dest_addr, _ep->info.dest_addr, sizeof(req->dest_addr));
- req->tx_attr = *_ep->info.tx_attr;
- req->rx_attr = *_ep->info.rx_attr;
- req->ep_attr = *_ep->info.ep_attr;
- req->domain_attr = *_ep->info.domain_attr;
- req->fabric_attr = *_ep->info.fabric_attr;
- req->fabric_attr.fabric = NULL;
- req->domain_attr.domain = NULL;
-
- if (param && paramlen)
- memcpy(&req->user_data, param, paramlen);
-
- memcpy(&_ep->cm_addr, addr, sizeof(struct sockaddr_in));
- if (sock_ep_cm_enqueue_msg(&_ep->cm, addr, req,
- sizeof(*req) + paramlen,
- &_ep->ep.fid, _eq)) {
- ret = -FI_EIO;
- goto err;
+ req->hdr.port = htons(_ep->attr->msg_src_port);
+ req->hdr.cm_data_sz = htons(paramlen);
+ req->caps = _ep->attr->info.caps;
+ memcpy(&req->src_addr, _ep->attr->src_addr, sizeof(req->src_addr));
+ memcpy(&handle->dest_addr, addr, sizeof(handle->dest_addr));
+
+ handle->ep = _ep;
+ handle->req = req;
+ if (paramlen) {
+ handle->paramlen = paramlen;
+ memcpy(handle->cm_data, param, paramlen);
}
- free (req);
- return 0;
+ if (_ep->attr->cm.listener_thread &&
+ pthread_join(_ep->attr->cm.listener_thread, NULL))
+ SOCK_LOG_DBG("failed to join cm listener\n");
-err:
+ if (pthread_create(&_ep->attr->cm.listener_thread, NULL,
+ sock_ep_cm_connect_handler, handle)) {
+ SOCK_LOG_ERROR("failed to create cm thread\n");
+ goto out;
+ }
+ return 0;
+out:
free(req);
- return ret;
+ free(handle);
+ return -FI_ENOMEM;
+}
+
+static void *sock_cm_accept_handler(void *data)
+{
+ int ret;
+ struct sock_conn_hdr reply;
+ struct sock_conn_req_handle *hreq = data;
+ struct sock_ep_attr *ep_attr;
+ struct fi_eq_cm_entry cm_entry;
+
+ ep_attr = hreq->ep->attr;
+ ep_attr->msg_dest_port = ntohs(hreq->req->hdr.port);
+
+ reply.type = SOCK_CONN_ACCEPT;
+ reply.port = htons(ep_attr->msg_src_port);
+ reply.cm_data_sz = htons(hreq->paramlen);
+ ret = sock_cm_send(hreq->sock_fd, &reply, sizeof(reply));
+ if (ret) {
+ SOCK_LOG_ERROR("failed to reply\n");
+ return NULL;
+ }
+
+ if (hreq->paramlen && sock_cm_send(hreq->sock_fd, hreq->cm_data, hreq->paramlen)) {
+ SOCK_LOG_ERROR("failed to send userdata\n");
+ return NULL;
+ }
+
+ cm_entry.fid = &hreq->ep->ep.fid;
+ SOCK_LOG_DBG("reporting FI_CONNECTED\n");
+ if (sock_eq_report_event(ep_attr->eq, FI_CONNECTED, &cm_entry,
+ sizeof(cm_entry), 0))
+ SOCK_LOG_ERROR("Error in writing to EQ\n");
+ ep_attr->cm.is_connected = 1;
+ ep_attr->cm.do_listen = 1;
+ ep_attr->cm.sock = hreq->sock_fd;
+ sock_ep_wait_shutdown(hreq->ep);
+
+ if (pthread_join(hreq->req_handler, NULL))
+ SOCK_LOG_DBG("failed to join req-handler\n");
+ free(hreq->req);
+ free(hreq);
+ return NULL;
}
-static int sock_ep_cm_accept(struct fid_ep *ep, const void *param,
- size_t paramlen)
+static int sock_ep_cm_accept(struct fid_ep *ep, const void *param, size_t paramlen)
{
struct sock_conn_req_handle *handle;
- struct sock_conn_req *req;
- struct sock_conn_response *response;
- struct sockaddr_in *addr;
struct sock_ep *_ep;
- int ret = 0;
_ep = container_of(ep, struct sock_ep, ep);
- if (!_ep->eq || paramlen > SOCK_EP_MAX_CM_DATA_SZ)
- return -FI_EINVAL;
-
- if (_ep->is_disabled || _ep->cm.shutdown_received)
+ if (!_ep->attr->eq || paramlen > SOCK_EP_MAX_CM_DATA_SZ)
return -FI_EINVAL;
- if (!_ep->listener.listener_thread && sock_conn_listen(_ep))
+ if (!_ep->attr->listener.listener_thread && sock_conn_listen(_ep->attr))
return -FI_EINVAL;
- response = calloc(1, sizeof(*response) + paramlen);
- if (!response)
- return -FI_ENOMEM;
-
- handle = container_of(_ep->info.handle, struct sock_conn_req_handle,
- handle);
+ handle = container_of(_ep->attr->info.handle,
+ struct sock_conn_req_handle, handle);
if (!handle || handle->handle.fclass != FI_CLASS_CONNREQ) {
SOCK_LOG_ERROR("invalid handle for cm_accept\n");
- free(response);
return -FI_EINVAL;
}
- req = handle->req;
- memcpy(&response->hdr, &req->hdr, sizeof(response->hdr));
- if (param && paramlen)
- memcpy(&response->user_data, param, paramlen);
-
- addr = &req->from_addr;
- memcpy(&_ep->cm_addr, addr, sizeof(*addr));
+ handle->ep = _ep;
+ handle->paramlen = 0;
+ handle->is_accepted = 1;
+ if (paramlen) {
+ handle->paramlen = paramlen;
+ memcpy(handle->cm_data, param, paramlen);
+ }
- response->hdr.type = SOCK_CONN_ACCEPT;
- req->hdr.msg_id = _ep->cm.next_msg_id++;
- response->hdr.s_port = htons(atoi(_ep->listener.service));
+ if (_ep->attr->cm.listener_thread &&
+ pthread_join(_ep->attr->cm.listener_thread, NULL))
+ SOCK_LOG_DBG("failed to join cm listener\n");
- if (sock_ep_cm_enqueue_msg(&_ep->cm, addr, response,
- sizeof(*response) + paramlen,
- &_ep->ep.fid, _ep->eq)) {
- ret = -FI_EIO;
- goto out;
+ if (pthread_create(&_ep->attr->cm.listener_thread, NULL,
+ sock_cm_accept_handler, handle)) {
+ SOCK_LOG_ERROR("Couldnt create accept handler\n");
+ return -FI_ENOMEM;
}
-
-out:
- free(handle);
- free(req);
- free(response);
- _ep->info.handle = NULL;
- return ret;
+ return 0;
}
static int sock_ep_cm_shutdown(struct fid_ep *ep, uint64_t flags)
{
- struct sock_conn_response response;
struct sock_ep *_ep;
+ struct fi_eq_cm_entry cm_entry = {0};
+ struct sock_conn_hdr msg = {0};
+ char c = 0;
_ep = container_of(ep, struct sock_ep, ep);
- memset(&response, 0, sizeof(response));
-
- response.hdr.type = SOCK_CONN_SHUTDOWN;
- response.hdr.msg_id = _ep->cm.next_msg_id++;
-
- if (sock_ep_cm_enqueue_msg(&_ep->cm, &_ep->cm_addr, &response,
- sizeof(response), &_ep->ep.fid, _ep->eq)) {
- return -FI_EIO;
+ fastlock_acquire(&_ep->attr->cm.lock);
+ if (_ep->attr->cm.is_connected) {
+ msg.type = SOCK_CONN_SHUTDOWN;
+ if (sock_cm_send(_ep->attr->cm.sock, &msg, sizeof(msg)))
+ SOCK_LOG_DBG("failed to send shutdown msg\n");
+ _ep->attr->cm.is_connected = 0;
+ _ep->attr->cm.do_listen = 0;
+ if (ofi_write_socket(_ep->attr->cm.signal_fds[0], &c, 1) != 1)
+ SOCK_LOG_DBG("Failed to signal\n");
+
+ cm_entry.fid = &_ep->ep.fid;
+ SOCK_LOG_DBG("reporting FI_SHUTDOWN\n");
+ if (sock_eq_report_event(_ep->attr->eq, FI_SHUTDOWN,
+ &cm_entry, sizeof(cm_entry), 0))
+ SOCK_LOG_ERROR("Error in writing to EQ\n");
}
-
+ fastlock_release(&_ep->attr->cm.lock);
sock_ep_disable(ep);
return 0;
}
@@ -905,11 +777,11 @@ static int sock_msg_endpoint(struct fid_domain *domain, struct fi_info *info,
if (info && info->handle && info->handle->fclass == FI_CLASS_PEP) {
pep = container_of(info->handle, struct sock_pep, pep.fid);
- memcpy((*ep)->src_addr, &pep->src_addr, sizeof *(*ep)->src_addr);
+ memcpy((*ep)->attr->src_addr, &pep->src_addr, sizeof *(*ep)->attr->src_addr);
}
if (!info || !info->ep_attr)
- (*ep)->ep_attr = sock_msg_ep_attr;
+ (*ep)->attr->ep_attr = sock_msg_ep_attr;
if (!info || !info->tx_attr)
(*ep)->tx_attr = sock_msg_tx_attr;
@@ -930,13 +802,6 @@ int sock_msg_ep(struct fid_domain *domain, struct fi_info *info,
if (ret)
return ret;
- endpoint->cm.do_listen = 1;
- if (pthread_create(&endpoint->cm.listener_thread, NULL,
- sock_msg_ep_listener_thread, endpoint)) {
- SOCK_LOG_ERROR("Couldn't create listener thread\n");
- return -FI_EINVAL;
- }
-
*ep = &endpoint->ep;
return 0;
}
@@ -968,7 +833,7 @@ static int sock_pep_fi_close(fid_t fid)
pep = container_of(fid, struct sock_pep, pep.fid);
pep->cm.do_listen = 0;
- ret = write(pep->cm.signal_fds[0], &c, 1);
+ ret = ofi_write_socket(pep->cm.signal_fds[0], &c, 1);
if (ret != 1)
SOCK_LOG_DBG("Failed to signal\n");
@@ -977,8 +842,8 @@ static int sock_pep_fi_close(fid_t fid)
SOCK_LOG_DBG("pthread join failed\n");
}
- close(pep->cm.signal_fds[0]);
- close(pep->cm.signal_fds[1]);
+ ofi_close_socket(pep->cm.signal_fds[0]);
+ ofi_close_socket(pep->cm.signal_fds[1]);
fastlock_destroy(&pep->cm.lock);
free(pep);
@@ -993,151 +858,189 @@ static struct fi_ops sock_pep_fi_ops = {
.ops_open = fi_no_ops_open,
};
-static struct fi_info *sock_ep_msg_process_info(struct sock_conn_req *req)
+static struct fi_info *sock_ep_msg_get_info(struct sock_pep *pep,
+ struct sock_conn_req *req)
+{
+ struct fi_info hints;
+ uint64_t requested, supported;
+
+ requested = req->caps & SOCK_EP_MSG_PRI_CAP;
+ supported = pep->info.caps & SOCK_EP_MSG_PRI_CAP;
+ supported = (supported & FI_RMA) ?
+ (supported | FI_REMOTE_READ | FI_REMOTE_WRITE) : supported;
+ if ((requested | supported) != supported)
+ return NULL;
+
+ hints = pep->info;
+ hints.caps = req->caps;
+ return sock_fi_info(FI_EP_MSG, &hints,
+ &pep->src_addr, &req->src_addr);
+}
+
+static void *sock_pep_req_handler(void *data)
{
- req->info.src_addr = &req->src_addr;
- req->info.dest_addr = &req->dest_addr;
- req->info.tx_attr = &req->tx_attr;
- req->info.rx_attr = &req->rx_attr;
- req->info.ep_attr = &req->ep_attr;
- req->info.domain_attr = &req->domain_attr;
- req->info.fabric_attr = &req->fabric_attr;
- req->info.domain_attr->name = NULL;
- req->info.fabric_attr->name = NULL;
- req->info.fabric_attr->prov_name = NULL;
- if (sock_verify_info(&req->info)) {
- SOCK_LOG_ERROR("incoming conn_req not supported\n");
- errno = EINVAL;
+ int ret, entry_sz;
+ struct fi_info *info;
+ struct sock_conn_req *conn_req = NULL;
+ struct fi_eq_cm_entry *cm_entry = NULL;
+ struct sock_conn_req_handle *handle = data;
+ int req_cm_data_sz;
+ char c = 0;
+
+ conn_req = calloc(1, sizeof(*conn_req) + SOCK_EP_MAX_CM_DATA_SZ);
+ if (!conn_req) {
+ SOCK_LOG_ERROR("cannot allocate memory\n");
+ goto err;
+ }
+
+ ret = sock_cm_recv(handle->sock_fd, conn_req, sizeof(*conn_req));
+ if (ret) {
+ SOCK_LOG_ERROR("IO failed\n");
+ goto err;
+ }
+
+ req_cm_data_sz = ntohs(conn_req->hdr.cm_data_sz);
+ if (req_cm_data_sz) {
+ ret = sock_cm_recv(handle->sock_fd, conn_req->cm_data,
+ req_cm_data_sz);
+ if (ret) {
+ SOCK_LOG_ERROR("IO failed for cm-data\n");
+ goto err;
+ }
+ }
+
+ info = sock_ep_msg_get_info(handle->pep, conn_req);
+ if (info == NULL) {
+ handle->paramlen = 0;
+ fastlock_acquire(&handle->pep->cm.lock);
+ dlist_insert_tail(&handle->entry, &handle->pep->cm.msg_list);
+ fastlock_release(&handle->pep->cm.lock);
+
+ if (ofi_write_socket(handle->pep->cm.signal_fds[0], &c, 1) != 1)
+ SOCK_LOG_DBG("Failed to signal\n");
+ free(conn_req);
return NULL;
}
- return sock_fi_info(FI_EP_MSG, &req->info,
- req->info.dest_addr, req->info.src_addr);
+ cm_entry = calloc(1, sizeof(*cm_entry) + req_cm_data_sz);
+ if (!cm_entry) {
+ SOCK_LOG_ERROR("cannot allocate memory\n");
+ goto err;
+ }
+
+ handle->handle.fclass = FI_CLASS_CONNREQ;
+ handle->req = conn_req;
+
+ entry_sz = sizeof(*cm_entry) + req_cm_data_sz;
+ cm_entry->fid = &handle->pep->pep.fid;
+ cm_entry->info = info;
+ cm_entry->info->handle = &handle->handle;
+ memcpy(cm_entry->data, conn_req->cm_data, req_cm_data_sz);
+
+ SOCK_LOG_DBG("reporting conn-req to EQ\n");
+ if (sock_eq_report_event(handle->pep->eq, FI_CONNREQ, cm_entry, entry_sz, 0))
+ SOCK_LOG_ERROR("Error in writing to EQ\n");
+
+ free(cm_entry);
+ return NULL;
+err:
+ ofi_close_socket(handle->sock_fd);
+ free(cm_entry);
+ free(conn_req);
+ free(handle);
+ return NULL;
+}
+
+static void sock_pep_check_msg_list(struct sock_pep *pep)
+{
+ struct dlist_entry *entry;
+ struct sock_conn_req_handle *hreq;
+ struct sock_conn_hdr reply;
+
+ fastlock_acquire(&pep->cm.lock);
+ while (!dlist_empty(&pep->cm.msg_list)) {
+ entry = pep->cm.msg_list.next;
+ dlist_remove(entry);
+ hreq = container_of(entry, struct sock_conn_req_handle, entry);
+
+ reply.type = SOCK_CONN_REJECT;
+ reply.cm_data_sz = htons(hreq->paramlen);
+
+ SOCK_LOG_DBG("sending reject message\n");
+ if (sock_cm_send(hreq->sock_fd, &reply, sizeof(reply))) {
+ SOCK_LOG_ERROR("failed to reply\n");
+ break;
+ }
+
+ if (hreq->paramlen && sock_cm_send(hreq->sock_fd, hreq->cm_data,
+ hreq->paramlen)) {
+ SOCK_LOG_ERROR("failed to send userdata\n");
+ break;
+ }
+
+ if (pthread_join(hreq->req_handler, NULL))
+ SOCK_LOG_DBG("failed to join req-handler\n");
+ ofi_close_socket(hreq->sock_fd);
+ free(hreq->req);
+ free(hreq);
+ }
+ fastlock_release(&pep->cm.lock);
}
static void *sock_pep_listener_thread(void *data)
{
struct sock_pep *pep = (struct sock_pep *) data;
struct sock_conn_req_handle *handle = NULL;
- struct sock_conn_req *conn_req = NULL;
- struct fi_eq_cm_entry *cm_entry;
- struct sockaddr_in from_addr;
struct pollfd poll_fds[2];
- socklen_t addr_len;
- int ret = 0, user_data_sz, entry_sz, timeout;
+ int ret = 0, conn_fd;
char tmp = 0;
SOCK_LOG_DBG("Starting listener thread for PEP: %p\n", pep);
- cm_entry = calloc(1, sizeof(*cm_entry) + SOCK_EP_MAX_CM_DATA_SZ);
- if (!cm_entry) {
- SOCK_LOG_ERROR("cannot allocate\n");
- return NULL;
- }
-
poll_fds[0].fd = pep->cm.sock;
poll_fds[1].fd = pep->cm.signal_fds[1];
poll_fds[0].events = poll_fds[1].events = POLLIN;
while (*((volatile int *) &pep->cm.do_listen)) {
- timeout = dlist_empty(&pep->cm.msg_list) ? -1 :
- SOCK_CM_COMM_TIMEOUT;
- if (poll(poll_fds, 2, timeout) > 0) {
+ ret = poll(poll_fds, 2, -1);
+ if (ret > 0) {
if (poll_fds[1].revents & POLLIN) {
- ret = read(pep->cm.signal_fds[1], &tmp, 1);
+ ret = ofi_read_socket(pep->cm.signal_fds[1], &tmp, 1);
if (ret != 1)
SOCK_LOG_DBG("Invalid signal\n");
- sock_ep_cm_flush_msg(&pep->cm);
+ sock_pep_check_msg_list(pep);
continue;
}
} else {
- if (ret == 0) {
- sock_ep_cm_flush_msg(&pep->cm);
- continue;
- } else {
- break;
- }
- }
-
- if (handle == NULL) {
- handle = calloc(1, sizeof(*handle));
- if (!handle)
- break;
+ break;
}
- if (conn_req == NULL) {
- conn_req = calloc(1, sizeof(*conn_req) +
- SOCK_EP_MAX_CM_DATA_SZ);
- if (!conn_req) {
- SOCK_LOG_ERROR("cannot allocate\n");
- break;
- }
+ conn_fd = accept(pep->cm.sock, NULL, 0);
+ if (conn_fd < 0) {
+ SOCK_LOG_ERROR("failed to accept: %d\n", errno);
+ continue;
}
- handle->handle.fclass = FI_CLASS_CONNREQ;
- handle->req = conn_req;
-
- addr_len = sizeof(struct sockaddr_in);
- ret = recvfrom(pep->cm.sock, (char *) conn_req,
- sizeof(*conn_req) + SOCK_EP_MAX_CM_DATA_SZ, 0,
- (struct sockaddr *) &from_addr, &addr_len);
- SOCK_LOG_DBG("Total received: %d\n", ret);
-
- if (ret <= 0)
- continue;
- memcpy(&conn_req->from_addr, &from_addr, sizeof(struct sockaddr_in));
- SOCK_LOG_DBG("CM msg received: %d\n", ret);
- memset(cm_entry, 0, sizeof(*cm_entry));
-
- if (conn_req->hdr.type != SOCK_CONN_ACK)
- sock_ep_cm_send_ack(&pep->cm, &from_addr,
- conn_req->hdr.msg_id);
-
- switch (conn_req->hdr.type) {
- case SOCK_CONN_ACK:
- SOCK_LOG_DBG("Received SOCK_CONN_ACK\n");
- sock_ep_cm_handle_ack(&pep->cm, &conn_req->hdr);
+ sock_set_sockopts_conn(conn_fd);
+ handle = calloc(1, sizeof(*handle));
+ if (!handle) {
+ SOCK_LOG_ERROR("cannot allocate memory\n");
+ ofi_close_socket(conn_fd);
break;
+ }
- case SOCK_CONN_REQ:
- SOCK_LOG_DBG("Received SOCK_CONN_REQ\n");
-
- user_data_sz = ret - sizeof(*conn_req);
- entry_sz = sizeof(*cm_entry) + user_data_sz;
-
- if (ret < sizeof(*conn_req)) {
- SOCK_LOG_ERROR("Invalid connection request\n");
- break;
- }
-
- cm_entry->fid = &pep->pep.fid;
- cm_entry->info = sock_ep_msg_process_info(conn_req);
- if (!cm_entry->info)
- goto out;
- cm_entry->info->handle = &handle->handle;
-
- memcpy(&cm_entry->data, &conn_req->user_data,
- user_data_sz);
- handle = NULL;
- conn_req = NULL;
-
- if (sock_eq_report_event(pep->eq, FI_CONNREQ, cm_entry,
- entry_sz, 0))
- SOCK_LOG_ERROR("Error in writing to EQ\n");
- break;
+ handle->sock_fd = conn_fd;
+ handle->pep = pep;
- default:
- SOCK_LOG_ERROR("Invalid event: %d\n", conn_req->hdr.type);
- goto out;
+ if (pthread_create(&handle->req_handler, NULL,
+ sock_pep_req_handler, handle)) {
+ SOCK_LOG_ERROR("failed to create req handler\n");
+ ofi_close_socket(conn_fd);
+ free(handle);
}
}
-out:
- if (conn_req)
- free(conn_req);
- if (handle)
- free(handle);
- free(cm_entry);
- close(pep->cm.sock);
+ SOCK_LOG_DBG("PEP listener thread exiting\n");
+ ofi_close_socket(pep->cm.sock);
return NULL;
}
@@ -1171,43 +1074,28 @@ static int sock_pep_reject(struct fid_pep *pep, fid_t handle,
{
struct sock_conn_req_handle *hreq;
struct sock_conn_req *req;
- struct sockaddr_in *addr;
struct sock_pep *_pep;
- struct sock_conn_response *response;
- int ret = 0;
+ char c = 0;
_pep = container_of(pep, struct sock_pep, pep);
hreq = container_of(handle, struct sock_conn_req_handle, handle);
req = hreq->req;
- if (!req || hreq->handle.fclass != FI_CLASS_CONNREQ)
+ if (!req || hreq->handle.fclass != FI_CLASS_CONNREQ || hreq->is_accepted)
return -FI_EINVAL;
- response = (struct sock_conn_response *)
- calloc(1, sizeof(*response) + paramlen);
- if (!response)
- return -FI_ENOMEM;
-
- memcpy(&response->hdr, &req->hdr, sizeof(struct sock_conn_hdr));
- if (param && paramlen)
- memcpy(&response->user_data, param, paramlen);
-
- addr = &req->from_addr;
- response->hdr.type = SOCK_CONN_REJECT;
- req->hdr.msg_id = _pep->cm.next_msg_id++;
-
- if (sock_ep_cm_enqueue_msg(&_pep->cm, addr, req,
- sizeof(struct sock_conn_response),
- &_pep->pep.fid, _pep->eq)) {
- ret = -FI_EIO;
- goto out;
+ hreq->paramlen = 0;
+ if (paramlen) {
+ memcpy(hreq->cm_data, param, paramlen);
+ hreq->paramlen = paramlen;
}
- ret = 0;
-out:
- free(hreq);
- free(req);
- free(response);
- return ret;
+ fastlock_acquire(&_pep->cm.lock);
+ dlist_insert_tail(&hreq->entry, &_pep->cm.msg_list);
+ fastlock_release(&_pep->cm.lock);
+
+ if (ofi_write_socket(_pep->cm.signal_fds[0], &c, 1) != 1)
+ SOCK_LOG_DBG("Failed to signal\n");
+ return 0;
}
static struct fi_ops_cm sock_pep_cm_ops = {
@@ -1222,6 +1110,31 @@ static struct fi_ops_cm sock_pep_cm_ops = {
.shutdown = fi_no_shutdown,
};
+int sock_pep_getopt(fid_t fid, int level, int optname,
+ void *optval, size_t *optlen)
+{
+ if (level != FI_OPT_ENDPOINT || optname != FI_OPT_CM_DATA_SIZE)
+ return -FI_ENOPROTOOPT;
+
+ if (*optlen < sizeof(size_t)) {
+ *optlen = sizeof(size_t);
+ return -FI_ETOOSMALL;
+ }
+ *((size_t *) optval) = SOCK_EP_MAX_CM_DATA_SZ;
+ *optlen = sizeof(size_t);
+ return 0;
+}
+
+static struct fi_ops_ep sock_pep_ops = {
+ .size = sizeof(struct fi_ops_ep),
+ .getopt = sock_pep_getopt,
+ .setopt = fi_no_setopt,
+ .tx_ctx = fi_no_tx_ctx,
+ .rx_ctx = fi_no_rx_ctx,
+ .rx_size_left = fi_no_rx_size_left,
+ .tx_size_left = fi_no_tx_size_left,
+};
+
int sock_msg_sep(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **sep, void *context)
{
@@ -1293,7 +1206,7 @@ int sock_msg_passive_ep(struct fid_fabric *fabric, struct fi_info *info,
_pep->pep.fid.context = context;
_pep->pep.fid.ops = &sock_pep_fi_ops;
_pep->pep.cm = &sock_pep_cm_ops;
- _pep->pep.ops = NULL;
+ _pep->pep.ops = &sock_pep_ops;
fastlock_init(&_pep->cm.lock);
_pep->sock_fab = container_of(fabric, struct sock_fabric, fab_fid);
diff --git a/prov/sockets/src/sock_ep_rdm.c b/prov/sockets/src/sock_ep_rdm.c
index 5b9e8f8..8f49d6e 100644
--- a/prov/sockets/src/sock_ep_rdm.c
+++ b/prov/sockets/src/sock_ep_rdm.c
@@ -118,7 +118,8 @@ static int sock_rdm_verify_rx_attr(const struct fi_rx_attr *attr)
return -FI_ENODATA;
}
- if (attr->size > sock_rdm_rx_attr.size) {
+ if (sock_get_tx_size(attr->size) >
+ sock_get_tx_size(sock_rdm_rx_attr.size)) {
SOCK_LOG_DBG("Rx size too large\n");
return -FI_ENODATA;
}
@@ -151,7 +152,8 @@ static int sock_rdm_verify_tx_attr(const struct fi_tx_attr *attr)
return -FI_ENODATA;
}
- if (attr->size > sock_rdm_tx_attr.size) {
+ if (sock_get_tx_size(attr->size) >
+ sock_get_tx_size(sock_rdm_tx_attr.size)) {
SOCK_LOG_DBG("Tx size too large\n");
return -FI_ENODATA;
}
@@ -247,7 +249,9 @@ int sock_rdm_fi_info(void *src_addr, void *dest_addr, struct fi_info *hints,
return -FI_ENOMEM;
*(*info)->tx_attr = sock_rdm_tx_attr;
+ (*info)->tx_attr->size = sock_get_tx_size(sock_rdm_tx_attr.size);
*(*info)->rx_attr = sock_rdm_rx_attr;
+ (*info)->rx_attr->size = sock_get_tx_size(sock_rdm_rx_attr.size);
*(*info)->ep_attr = sock_rdm_ep_attr;
if (hints && hints->ep_attr) {
@@ -315,7 +319,7 @@ static int sock_rdm_endpoint(struct fid_domain *domain, struct fi_info *info,
return ret;
if (!info || !info->ep_attr)
- (*ep)->ep_attr = sock_rdm_ep_attr;
+ (*ep)->attr->ep_attr = sock_rdm_ep_attr;
if (!info || !info->tx_attr)
(*ep)->tx_attr = sock_rdm_tx_attr;
diff --git a/prov/sockets/src/sock_epoll.c b/prov/sockets/src/sock_epoll.c
index 2b47137..ffbdd57 100644
--- a/prov/sockets/src/sock_epoll.c
+++ b/prov/sockets/src/sock_epoll.c
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2014 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -75,21 +76,35 @@ int sock_epoll_create(struct sock_epoll_set *set, int size)
int sock_epoll_add(struct sock_epoll_set *set, int fd)
{
+ int ret;
struct epoll_event event;
+
+ memset(&event, 0, sizeof(event));
event.data.fd = fd;
event.events = EPOLLIN;
if (set->used == set->size)
return -1;
- set->used++;
- return epoll_ctl(set->fd, EPOLL_CTL_ADD, fd, &event);
+ ret = epoll_ctl(set->fd, EPOLL_CTL_ADD, fd, &event);
+ if (!ret)
+ set->used++;
+
+ return ret;
}
int sock_epoll_del(struct sock_epoll_set *set, int fd)
{
- set->used--;
- return epoll_ctl(set->fd, EPOLL_CTL_DEL, fd, NULL);
+ int ret;
+
+ if (!set->used)
+ return -1;
+
+ ret = epoll_ctl(set->fd, EPOLL_CTL_DEL, fd, NULL);
+ if (!ret)
+ set->used--;
+
+ return ret;
}
int sock_epoll_wait(struct sock_epoll_set *set, int timeout)
@@ -108,6 +123,7 @@ void sock_epoll_close(struct sock_epoll_set *set)
{
free(set->events);
close(set->fd);
+ set->used = 0;
}
#else
@@ -177,6 +193,7 @@ int sock_epoll_get_fd_at_index(struct sock_epoll_set *set, int index)
void sock_epoll_close(struct sock_epoll_set *set)
{
free(set->pollfds);
+ set->used = 0;
}
#endif
diff --git a/prov/sockets/src/sock_eq.c b/prov/sockets/src/sock_eq.c
index 2b459f8..7eb86e6 100644
--- a/prov/sockets/src/sock_eq.c
+++ b/prov/sockets/src/sock_eq.c
@@ -155,6 +155,7 @@ static ssize_t sock_eq_readerr(struct fid_eq *eq, struct fi_eq_err_entry *buf,
}
dlistfd_remove(list, &sock_eq->err_list);
+ dlistfd_reset(&sock_eq->list);
free(entry);
}
diff --git a/prov/sockets/src/sock_fabric.c b/prov/sockets/src/sock_fabric.c
index 516132f..81a665f 100644
--- a/prov/sockets/src/sock_fabric.c
+++ b/prov/sockets/src/sock_fabric.c
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2014 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -205,10 +206,6 @@ int sock_verify_fabric_attr(struct fi_fabric_attr *attr)
if (!attr)
return 0;
- if (attr->name &&
- strcmp(attr->name, sock_fab_name))
- return -FI_ENODATA;
-
if (attr->prov_version) {
if (attr->prov_version !=
FI_VERSION(SOCK_MAJOR_VERSION, SOCK_MINOR_VERSION))
@@ -356,9 +353,6 @@ static int sock_fabric(struct fi_fabric_attr *attr,
{
struct sock_fabric *fab;
- if (strcmp(attr->name, sock_fab_name))
- return -FI_EINVAL;
-
fab = calloc(1, sizeof(*fab));
if (!fab)
return -FI_ENOMEM;
@@ -381,58 +375,6 @@ static int sock_fabric(struct fi_fabric_attr *attr,
return 0;
}
-static struct sock_service_entry *sock_fabric_find_service(struct sock_fabric *fab,
- int service)
-{
- struct dlist_entry *entry;
- struct sock_service_entry *service_entry;
-
- for (entry = fab->service_list.next; entry != &fab->service_list;
- entry = entry->next) {
- service_entry = container_of(entry,
- struct sock_service_entry, entry);
- if (service_entry->service == service)
- return service_entry;
- }
- return NULL;
-}
-
-int sock_fabric_check_service(struct sock_fabric *fab, int service)
-{
- struct sock_service_entry *entry;
-
- fastlock_acquire(&fab->lock);
- entry = sock_fabric_find_service(fab, service);
- fastlock_release(&fab->lock);
- return (entry == NULL) ? 1 : 0;
-}
-
-void sock_fabric_add_service(struct sock_fabric *fab, int service)
-{
- struct sock_service_entry *entry;
-
- entry = calloc(1, sizeof(*entry));
- if (!entry)
- return;
-
- entry->service = service;
- fastlock_acquire(&fab->lock);
- dlist_insert_tail(&entry->entry, &fab->service_list);
- fastlock_release(&fab->lock);
-}
-
-void sock_fabric_remove_service(struct sock_fabric *fab, int service)
-{
- struct sock_service_entry *service_entry;
- fastlock_acquire(&fab->lock);
- service_entry = sock_fabric_find_service(fab, service);
- if (service_entry) {
- dlist_remove(&service_entry->entry);
- free(service_entry);
- }
- fastlock_release(&fab->lock);
-}
-
int sock_get_src_addr(struct sockaddr_in *dest_addr,
struct sockaddr_in *src_addr)
{
@@ -447,7 +389,6 @@ int sock_get_src_addr(struct sockaddr_in *dest_addr,
ret = connect(sock, (struct sockaddr *) dest_addr, len);
if (ret) {
SOCK_LOG_DBG("Failed to connect udp socket\n");
-
ret = sock_get_src_addr_from_hostname(src_addr, NULL);
goto out;
}
@@ -459,10 +400,23 @@ int sock_get_src_addr(struct sockaddr_in *dest_addr,
ret = -errno;
}
out:
- close(sock);
+ ofi_close_socket(sock);
return ret;
}
+static int sock_fi_checkinfo(struct fi_info *info, struct fi_info *hints)
+{
+ if (hints && hints->domain_attr && hints->domain_attr->name &&
+ strcmp(info->domain_attr->name, hints->domain_attr->name))
+ return -FI_ENODATA;
+
+ if (hints && hints->fabric_attr && hints->fabric_attr->name &&
+ strcmp(info->fabric_attr->name, hints->fabric_attr->name))
+ return -FI_ENODATA;
+
+ return 0;
+}
+
static int sock_ep_getinfo(const char *node, const char *service, uint64_t flags,
struct fi_info *hints, enum fi_ep_type ep_type,
struct fi_info **info)
@@ -486,7 +440,6 @@ static int sock_ep_getinfo(const char *node, const char *service, uint64_t flags
return -FI_ENODATA;
}
src_addr = (struct sockaddr_in *) rai->ai_addr;
-
if (hints && hints->dest_addr)
dest_addr = hints->dest_addr;
} else {
@@ -533,90 +486,92 @@ static int sock_ep_getinfo(const char *node, const char *service, uint64_t flags
if (rai)
freeaddrinfo(rai);
+
+ if (ret == 0)
+ return sock_fi_checkinfo(*info, hints);
+
return ret;
}
-void sock_getnodename(char *buf, int buflen)
+void sock_insert_loopback_addr(struct slist *addr_list)
{
- int ret;
- struct addrinfo ai, *rai = NULL;
- struct ifaddrs *ifaddrs, *ifa;
+ struct sock_host_list_entry *addr_entry;
- ret = gethostname(buf, buflen);
- if (ret == 0) {
- memset(&ai, 0, sizeof(ai));
- ai.ai_family = AF_INET;
- ret = getaddrinfo(buf, NULL, &ai, &rai);
- if (!ret) {
- freeaddrinfo(rai);
- return;
- }
- }
+ addr_entry = calloc(1, sizeof(struct sock_host_list_entry));
+ strncpy(addr_entry->hostname, "127.0.0.1", sizeof(addr_entry->hostname));
+ slist_insert_tail(&addr_entry->entry, addr_list);
+}
#if HAVE_GETIFADDRS
+void sock_get_list_of_addr(struct slist *addr_list)
+{
+ int ret;
+ struct sock_host_list_entry *addr_entry;
+ struct ifaddrs *ifaddrs, *ifa;
+
ret = getifaddrs(&ifaddrs);
- for(ifa = ifaddrs; ifa != NULL; ifa = ifa->ifa_next) {
- if (ifa->ifa_addr == NULL || (ifa->ifa_addr->sa_family != AF_INET))
- continue;
-
- ret = getnameinfo(ifa->ifa_addr, sizeof(struct sockaddr_in),
- buf, buflen, NULL, 0, NI_NUMERICHOST);
- if (ret == 0) {
- freeifaddrs(ifaddrs);
- return;
+ if (!ret) {
+ for (ifa = ifaddrs; ifa != NULL; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr == NULL || !(ifa->ifa_flags & IFF_UP) ||
+ (ifa->ifa_addr->sa_family != AF_INET) ||
+ !strcmp(ifa->ifa_name, "lo"))
+ continue;
+
+ addr_entry = calloc(1, sizeof(struct sock_host_list_entry));
+ ret = getnameinfo(ifa->ifa_addr, sizeof(struct sockaddr_in),
+ addr_entry->hostname, sizeof(addr_entry->hostname),
+ NULL, 0, NI_NUMERICHOST);
+ if (ret) {
+ SOCK_LOG_DBG("getnameinfo failed: %d\n", ret);
+ free(addr_entry);
+ continue;
+ }
+ slist_insert_tail(&addr_entry->entry, addr_list);
}
+ freeifaddrs(ifaddrs);
}
- freeifaddrs(ifaddrs);
-#endif
- /* no reasonable address found, try loopback */
- strncpy(buf, "127.0.0.1", buflen);
+ // Always add loopback address at the end
+ sock_insert_loopback_addr(addr_list);
+}
+#else
+void sock_get_list_of_addr(struct slist *addr_list)
+{
+ sock_insert_loopback_addr(addr_list);
}
+#endif
-static int sock_getinfo(uint32_t version, const char *node, const char *service,
- uint64_t flags, struct fi_info *hints, struct fi_info **info)
+int sock_node_getinfo(const char *node, const char *service, uint64_t flags,
+ struct fi_info *hints, struct fi_info **info,
+ struct fi_info **tail)
{
- struct fi_info *cur, *tail;
enum fi_ep_type ep_type;
- char hostname[HOST_NAME_MAX];
+ struct fi_info *cur;
int ret;
- if (!(flags & FI_SOURCE) && hints && hints->src_addr &&
- (hints->src_addrlen != sizeof(struct sockaddr_in)))
- return -FI_ENODATA;
-
- if (((!node && !service) || (flags & FI_SOURCE)) &&
- hints && hints->dest_addr &&
- (hints->dest_addrlen != sizeof(struct sockaddr_in)))
- return -FI_ENODATA;
-
- ret = sock_verify_info(hints);
- if (ret)
- return ret;
-
- if (!node && !service && !hints) {
- flags |= FI_SOURCE;
- sock_getnodename(hostname, sizeof(hostname));
- node = hostname;
- }
-
- if (!node && !service && !(flags & FI_SOURCE) && !hints->dest_addr) {
- sock_getnodename(hostname, sizeof(hostname));
- node = hostname;
- }
-
if (hints && hints->ep_attr) {
switch (hints->ep_attr->type) {
case FI_EP_RDM:
case FI_EP_DGRAM:
case FI_EP_MSG:
- return sock_ep_getinfo(node, service, flags, hints,
- hints->ep_attr->type, info);
+ ret = sock_ep_getinfo(node, service, flags, hints,
+ hints->ep_attr->type, &cur);
+ if (ret) {
+ if (ret == -FI_ENODATA)
+ return ret;
+ goto err;
+ }
+
+ if (!*info)
+ *info = cur;
+ else
+ (*tail)->next = cur;
+ for (*tail = cur; (*tail)->next; *tail = (*tail)->next)
+ ;
+ return 0;
default:
break;
}
}
-
- *info = tail = NULL;
for (ep_type = FI_EP_MSG; ep_type <= FI_EP_RDM; ep_type++) {
ret = sock_ep_getinfo(node, service, flags,
hints, ep_type, &cur);
@@ -629,8 +584,8 @@ static int sock_getinfo(uint32_t version, const char *node, const char *service,
if (!*info)
*info = cur;
else
- tail->next = cur;
- for (tail = cur; tail->next; tail = tail->next)
+ (*tail)->next = cur;
+ for (*tail = cur; (*tail)->next; *tail = (*tail)->next)
;
}
if (!*info) {
@@ -646,6 +601,127 @@ err_no_free:
return ret;
}
+static int sock_match_src_addr(struct slist_entry *entry, const void *src_addr)
+{
+ struct sock_host_list_entry *host_entry;
+ host_entry = container_of(entry, struct sock_host_list_entry, entry);
+
+ return (strcmp(host_entry->hostname, (char *) src_addr) == 0);
+}
+
+static int sock_addr_matches_interface(struct slist *addr_list, struct sockaddr_in *src_addr)
+{
+ struct slist_entry *entry;
+
+ /* Always match if it's localhost */
+ if (src_addr->sin_addr.s_addr == htonl(INADDR_LOOPBACK))
+ return 1;
+
+ entry = slist_find_first_match(addr_list, sock_match_src_addr,
+ inet_ntoa(src_addr->sin_addr));
+
+ return entry ? 1 : 0;
+}
+
+static int sock_node_matches_interface(struct slist *addr_list, const char *node)
+{
+ struct addrinfo ai, *rai = NULL;
+ struct sockaddr_in addr;
+ int ret;
+
+ memset(&ai, 0, sizeof(ai));
+ ai.ai_family = AF_INET;
+ ai.ai_socktype = SOCK_STREAM;
+
+ ret = getaddrinfo(node, 0, &ai, &rai);
+ if (ret) {
+ SOCK_LOG_DBG("getaddrinfo failed!\n");
+ return -FI_EINVAL;
+ }
+ addr = *(struct sockaddr_in *)rai->ai_addr;
+ freeaddrinfo(rai);
+
+ return sock_addr_matches_interface(addr_list, &addr);
+}
+
+static void sock_free_addr_list(struct slist *addr_list)
+{
+ struct slist_entry *entry;
+ struct sock_host_list_entry *host_entry;
+
+ while (!slist_empty(addr_list)) {
+ entry = slist_remove_head(addr_list);
+ host_entry = container_of(entry, struct sock_host_list_entry,
+ entry);
+ free(host_entry);
+ }
+}
+
+static int sock_getinfo(uint32_t version, const char *node, const char *service,
+ uint64_t flags, struct fi_info *hints,
+ struct fi_info **info)
+{
+ int ret = 0;
+ struct slist addr_list;
+ struct slist_entry *entry;
+ struct sock_host_list_entry *host_entry;
+ struct fi_info *tail;
+
+ if (!(flags & FI_SOURCE) && hints && hints->src_addr &&
+ (hints->src_addrlen != sizeof(struct sockaddr_in)))
+ return -FI_ENODATA;
+
+ if (((!node && !service) || (flags & FI_SOURCE)) &&
+ hints && hints->dest_addr &&
+ (hints->dest_addrlen != sizeof(struct sockaddr_in)))
+ return -FI_ENODATA;
+
+ ret = sock_verify_info(hints);
+ if (ret)
+ return ret;
+
+ slist_init(&addr_list);
+ /* Returns loopback address if no other interfaces are available */
+ sock_get_list_of_addr(&addr_list);
+
+ ret = 1;
+ if (flags & FI_SOURCE) {
+ if (node)
+ ret = sock_node_matches_interface(&addr_list, node);
+ } else if (hints && hints->src_addr) {
+ ret = sock_addr_matches_interface(&addr_list, (struct sockaddr_in *)hints->src_addr);
+ }
+ if (!ret) {
+ SOCK_LOG_ERROR("Couldn't find a match with local interfaces\n");
+ return -FI_ENODATA;
+ }
+
+ *info = tail = NULL;
+ if (node ||
+ (!(flags & FI_SOURCE) && hints && hints->src_addr) ||
+ (!(flags & FI_SOURCE) && hints && hints->dest_addr)) {
+ sock_free_addr_list(&addr_list);
+ return sock_node_getinfo(node, service, flags, hints, info, &tail);
+ }
+
+ while (!slist_empty(&addr_list)) {
+ entry = slist_remove_head(&addr_list);
+ host_entry = container_of(entry, struct sock_host_list_entry, entry);
+ node = host_entry->hostname;
+ flags |= FI_SOURCE;
+ ret = sock_node_getinfo(node, service, flags, hints, info, &tail);
+ free(host_entry);
+ if (ret) {
+ if (ret == -FI_ENODATA)
+ continue;
+ sock_free_addr_list(&addr_list);
+ return ret;
+ }
+ }
+
+ return (!*info) ? ret : 0;
+}
+
static void fi_sockets_fini(void)
{
fastlock_destroy(&sock_list_lock);
@@ -654,7 +730,7 @@ static void fi_sockets_fini(void)
struct fi_provider sock_prov = {
.name = sock_prov_name,
.version = FI_VERSION(SOCK_MAJOR_VERSION, SOCK_MINOR_VERSION),
- .fi_version = FI_VERSION(1, 3),
+ .fi_version = FI_VERSION(1, 4),
.getinfo = sock_getinfo,
.fabric = sock_fabric,
.cleanup = fi_sockets_fini
diff --git a/prov/sockets/src/sock_msg.c b/prov/sockets/src/sock_msg.c
index c7074ad..ce4eb33 100644
--- a/prov/sockets/src/sock_msg.c
+++ b/prov/sockets/src/sock_msg.c
@@ -64,15 +64,18 @@ ssize_t sock_ep_recvmsg(struct fid_ep *ep, const struct fi_msg *msg,
struct sock_rx_ctx *rx_ctx;
struct sock_rx_entry *rx_entry;
struct sock_ep *sock_ep;
+ uint64_t op_flags;
switch (ep->fid.fclass) {
case FI_CLASS_EP:
sock_ep = container_of(ep, struct sock_ep, ep);
- rx_ctx = sock_ep->rx_ctx;
+ rx_ctx = sock_ep->attr->rx_ctx;
+ op_flags = sock_ep->rx_attr.op_flags;
break;
case FI_CLASS_RX_CTX:
case FI_CLASS_SRX_CTX:
rx_ctx = container_of(ep, struct sock_rx_ctx, ctx);
+ op_flags = rx_ctx->attr.op_flags;
break;
default:
SOCK_LOG_ERROR("Invalid ep type\n");
@@ -88,7 +91,7 @@ ssize_t sock_ep_recvmsg(struct fid_ep *ep, const struct fi_msg *msg,
return -FI_EOPBADSTATE;
if (flags & SOCK_USE_OP_FLAGS)
- flags |= rx_ctx->attr.op_flags;
+ flags |= op_flags;
if (flags & FI_TRIGGER) {
ret = sock_queue_msg_op(ep, msg, flags, SOCK_OP_RECV);
@@ -172,21 +175,26 @@ ssize_t sock_ep_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
uint64_t flags)
{
int ret, i;
- uint64_t total_len;
+ uint64_t total_len, op_flags;
struct sock_op tx_op;
union sock_iov tx_iov;
struct sock_conn *conn;
struct sock_tx_ctx *tx_ctx;
struct sock_ep *sock_ep;
+ struct sock_ep_attr *ep_attr;
switch (ep->fid.fclass) {
case FI_CLASS_EP:
sock_ep = container_of(ep, struct sock_ep, ep);
- tx_ctx = sock_ep->tx_ctx;
+ ep_attr = sock_ep->attr;
+ tx_ctx = sock_ep->attr->tx_ctx->use_shared ?
+ sock_ep->attr->tx_ctx->stx_ctx : sock_ep->attr->tx_ctx;
+ op_flags = sock_ep->tx_attr.op_flags;
break;
case FI_CLASS_TX_CTX:
tx_ctx = container_of(ep, struct sock_tx_ctx, fid.ctx);
- sock_ep = tx_ctx->ep;
+ ep_attr = tx_ctx->ep_attr;
+ op_flags = tx_ctx->attr.op_flags;
break;
default:
SOCK_LOG_ERROR("Invalid EP type\n");
@@ -201,10 +209,10 @@ ssize_t sock_ep_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
if (!tx_ctx->enabled)
return -FI_EOPBADSTATE;
- if (sock_drop_packet(sock_ep))
+ if (sock_drop_packet(ep_attr))
return 0;
- ret = sock_ep_get_conn(sock_ep, tx_ctx, msg->addr, &conn);
+ ret = sock_ep_get_conn(ep_attr, tx_ctx, msg->addr, &conn);
if (ret)
return ret;
@@ -213,7 +221,7 @@ ssize_t sock_ep_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
SOCK_EP_SET_TX_OP_FLAGS(flags);
if (flags & SOCK_USE_OP_FLAGS)
- flags |= tx_ctx->attr.op_flags;
+ flags |= op_flags;
if (flags & FI_TRIGGER) {
ret = sock_queue_msg_op(ep, msg, flags, SOCK_OP_SEND);
@@ -252,7 +260,7 @@ ssize_t sock_ep_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
sock_tx_ctx_write_op_send(tx_ctx, &tx_op, flags, (uintptr_t) msg->context,
msg->addr, (uintptr_t) msg->msg_iov[0].iov_base,
- sock_ep, conn);
+ ep_attr, conn);
if (flags & FI_REMOTE_CQ_DATA)
sock_tx_ctx_write(tx_ctx, &msg->data, sizeof(msg->data));
@@ -385,15 +393,18 @@ ssize_t sock_ep_trecvmsg(struct fid_ep *ep,
struct sock_rx_ctx *rx_ctx;
struct sock_rx_entry *rx_entry;
struct sock_ep *sock_ep;
+ uint64_t op_flags;
switch (ep->fid.fclass) {
case FI_CLASS_EP:
sock_ep = container_of(ep, struct sock_ep, ep);
- rx_ctx = sock_ep->rx_ctx;
+ rx_ctx = sock_ep->attr->rx_ctx;
+ op_flags = sock_ep->rx_attr.op_flags;
break;
case FI_CLASS_RX_CTX:
case FI_CLASS_SRX_CTX:
rx_ctx = container_of(ep, struct sock_rx_ctx, ctx);
+ op_flags = rx_ctx->attr.op_flags;
break;
default:
SOCK_LOG_ERROR("Invalid ep type\n");
@@ -409,7 +420,7 @@ ssize_t sock_ep_trecvmsg(struct fid_ep *ep,
return -FI_EOPBADSTATE;
if (flags & SOCK_USE_OP_FLAGS)
- flags |= rx_ctx->attr.op_flags;
+ flags |= op_flags;
flags &= ~FI_MULTI_RECV;
if (flags & FI_TRIGGER) {
@@ -503,21 +514,26 @@ ssize_t sock_ep_tsendmsg(struct fid_ep *ep,
const struct fi_msg_tagged *msg, uint64_t flags)
{
int ret, i;
- uint64_t total_len;
+ uint64_t total_len, op_flags;
struct sock_op tx_op;
union sock_iov tx_iov;
struct sock_conn *conn;
struct sock_tx_ctx *tx_ctx;
struct sock_ep *sock_ep;
+ struct sock_ep_attr *ep_attr;
switch (ep->fid.fclass) {
case FI_CLASS_EP:
sock_ep = container_of(ep, struct sock_ep, ep);
- tx_ctx = sock_ep->tx_ctx;
+ tx_ctx = sock_ep->attr->tx_ctx->use_shared ?
+ sock_ep->attr->tx_ctx->stx_ctx : sock_ep->attr->tx_ctx;
+ ep_attr = sock_ep->attr;
+ op_flags = sock_ep->tx_attr.op_flags;
break;
case FI_CLASS_TX_CTX:
tx_ctx = container_of(ep, struct sock_tx_ctx, fid.ctx);
- sock_ep = tx_ctx->ep;
+ ep_attr = tx_ctx->ep_attr;
+ op_flags = tx_ctx->attr.op_flags;
break;
default:
SOCK_LOG_ERROR("Invalid EP type\n");
@@ -532,16 +548,16 @@ ssize_t sock_ep_tsendmsg(struct fid_ep *ep,
if (!tx_ctx->enabled)
return -FI_EOPBADSTATE;
- if (sock_drop_packet(sock_ep))
+ if (sock_drop_packet(ep_attr))
return 0;
- ret = sock_ep_get_conn(sock_ep, tx_ctx, msg->addr, &conn);
+ ret = sock_ep_get_conn(ep_attr, tx_ctx, msg->addr, &conn);
if (ret)
return ret;
SOCK_EP_SET_TX_OP_FLAGS(flags);
if (flags & SOCK_USE_OP_FLAGS)
- flags |= tx_ctx->attr.op_flags;
+ flags |= op_flags;
if (flags & FI_TRIGGER) {
ret = sock_queue_tmsg_op(ep, msg, flags, SOCK_OP_TSEND);
@@ -580,7 +596,7 @@ ssize_t sock_ep_tsendmsg(struct fid_ep *ep,
sock_tx_ctx_write_op_tsend(tx_ctx, &tx_op, flags,
(uintptr_t) msg->context, msg->addr,
(uintptr_t) msg->msg_iov[0].iov_base,
- sock_ep, conn, msg->tag);
+ ep_attr, conn, msg->tag);
if (flags & FI_REMOTE_CQ_DATA)
sock_tx_ctx_write(tx_ctx, &msg->data, sizeof(msg->data));
diff --git a/prov/sockets/src/sock_poll.c b/prov/sockets/src/sock_poll.c
index e648bd9..89b56b9 100644
--- a/prov/sockets/src/sock_poll.c
+++ b/prov/sockets/src/sock_poll.c
@@ -142,8 +142,10 @@ static int sock_poll_poll(struct fid_poll *pollset, void **context, int count)
cntr_fid);
sock_cntr_progress(cntr);
pthread_mutex_lock(&cntr->mut);
- if (atomic_get(&cntr->value) >=
- atomic_get(&cntr->threshold)) {
+ if (atomic_get(&cntr->value) !=
+ atomic_get(&cntr->last_read_val)) {
+ atomic_set(&cntr->last_read_val,
+ atomic_get(&cntr->value));
*context++ = cntr->cntr_fid.fid.context;
ret_count++;
}
diff --git a/prov/sockets/src/sock_progress.c b/prov/sockets/src/sock_progress.c
index bd22357..4683db2 100644
--- a/prov/sockets/src/sock_progress.c
+++ b/prov/sockets/src/sock_progress.c
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2014 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -125,7 +126,7 @@ static inline void sock_pe_discard_field(struct sock_pe_entry *pe_entry)
{
size_t ret;
if (!pe_entry->rem)
- return;
+ goto out;
SOCK_LOG_DBG("Remaining for %p: %ld\n", pe_entry, pe_entry->rem);
ret = sock_comm_discard(pe_entry, pe_entry->rem);
@@ -135,6 +136,7 @@ static inline void sock_pe_discard_field(struct sock_pe_entry *pe_entry)
if (pe_entry->rem == 0)
pe_entry->conn->rx_pe_entry = NULL;
+ out:
if (pe_entry->done_len == pe_entry->total_len && !pe_entry->rem) {
SOCK_LOG_DBG("Discard complete for %p\n", pe_entry);
pe_entry->is_complete = 1;
@@ -158,6 +160,7 @@ static void sock_pe_release_entry(struct sock_pe *pe,
if (pe_entry->is_pool_entry) {
rbfree(&pe_entry->comm_buf);
+ dlist_remove(&pe_entry->entry);
util_buf_release(pe->pe_rx_pool, pe_entry);
return;
}
@@ -200,6 +203,7 @@ static struct sock_pe_entry *sock_pe_acquire_entry(struct sock_pe *pe)
if (rbinit(&pe_entry->comm_buf, SOCK_PE_OVERFLOW_COMM_BUFF_SZ))
SOCK_LOG_ERROR("failed to init comm-cache\n");
pe_entry->cache_sz = SOCK_PE_OVERFLOW_COMM_BUFF_SZ;
+ dlist_insert_tail(&pe_entry->entry, &pe->pool_list);
}
} else {
pe->num_free_entries--;
@@ -213,7 +217,7 @@ static struct sock_pe_entry *sock_pe_acquire_entry(struct sock_pe *pe)
return pe_entry;
}
-static void sock_pe_report_tx_completion(struct sock_pe_entry *pe_entry)
+static void sock_pe_report_send_cq_completion(struct sock_pe_entry *pe_entry)
{
int ret = 0;
if (!(pe_entry->flags & SOCK_NO_COMPLETION)) {
@@ -224,9 +228,6 @@ static void sock_pe_report_tx_completion(struct sock_pe_entry *pe_entry)
pe_entry->comp->send_cq, pe_entry->addr, pe_entry);
}
- if (pe_entry->comp->send_cntr)
- sock_cntr_inc(pe_entry->comp->send_cntr);
-
if (ret < 0) {
SOCK_LOG_ERROR("Failed to report completion %p\n",
pe_entry);
@@ -240,10 +241,16 @@ static void sock_pe_report_tx_completion(struct sock_pe_entry *pe_entry)
}
}
-static void sock_pe_report_rx_completion(struct sock_pe_entry *pe_entry)
+static void sock_pe_report_send_completion(struct sock_pe_entry *pe_entry)
{
- int ret = 0;
+ sock_pe_report_send_cq_completion(pe_entry);
+ if (pe_entry->comp->send_cntr)
+ sock_cntr_inc(pe_entry->comp->send_cntr);
+}
+static void sock_pe_report_recv_cq_completion(struct sock_pe_entry *pe_entry)
+{
+ int ret = 0;
if (pe_entry->comp->recv_cq &&
(!pe_entry->comp->recv_cq_event ||
(pe_entry->flags & FI_COMPLETION)))
@@ -251,10 +258,6 @@ static void sock_pe_report_rx_completion(struct sock_pe_entry *pe_entry)
pe_entry->comp->recv_cq, pe_entry->addr,
pe_entry);
- if (pe_entry->comp->recv_cntr)
- sock_cntr_inc(pe_entry->comp->recv_cntr);
-
-
if (ret < 0) {
SOCK_LOG_ERROR("Failed to report completion %p\n", pe_entry);
if (pe_entry->comp->eq) {
@@ -267,6 +270,13 @@ static void sock_pe_report_rx_completion(struct sock_pe_entry *pe_entry)
}
}
+static void sock_pe_report_recv_completion(struct sock_pe_entry *pe_entry)
+{
+ sock_pe_report_recv_cq_completion(pe_entry);
+ if (pe_entry->comp->recv_cntr)
+ sock_cntr_inc(pe_entry->comp->recv_cntr);
+}
+
static void sock_pe_report_mr_completion(struct sock_domain *domain,
struct sock_pe_entry *pe_entry)
{
@@ -275,7 +285,7 @@ static void sock_pe_report_mr_completion(struct sock_domain *domain,
for (i = 0; i < pe_entry->msg_hdr.dest_iov_len; i++) {
fastlock_acquire(&domain->lock);
- mr = sock_mr_get_entry(domain, pe_entry->pe.rx.rx_iov[i].iov.key);
+ mr = ofi_mr_retrieve(domain->mr_heap, pe_entry->pe.rx.rx_iov[i].iov.key);
fastlock_release(&domain->lock);
if (!mr || (!mr->cq && !mr->cntr))
continue;
@@ -294,6 +304,10 @@ static void sock_pe_report_remote_write(struct sock_rx_ctx *rx_ctx,
pe_entry->buf = pe_entry->pe.rx.rx_iov[0].iov.addr;
pe_entry->data_len = pe_entry->pe.rx.rx_iov[0].iov.len;
+ if (pe_entry->flags & FI_REMOTE_CQ_DATA) {
+ sock_pe_report_recv_cq_completion(pe_entry);
+ }
+
if ((!pe_entry->comp->rem_write_cntr &&
!(pe_entry->msg_hdr.flags & FI_REMOTE_WRITE)))
return;
@@ -305,10 +319,9 @@ static void sock_pe_report_remote_write(struct sock_rx_ctx *rx_ctx,
static void sock_pe_report_write_completion(struct sock_pe_entry *pe_entry)
{
if (!(pe_entry->flags & SOCK_NO_COMPLETION))
- sock_pe_report_tx_completion(pe_entry);
+ sock_pe_report_send_cq_completion(pe_entry);
- if (pe_entry->comp->write_cntr &&
- pe_entry->comp->write_cntr != pe_entry->comp->send_cntr)
+ if (pe_entry->comp->write_cntr)
sock_cntr_inc(pe_entry->comp->write_cntr);
}
@@ -329,20 +342,19 @@ static void sock_pe_report_remote_read(struct sock_rx_ctx *rx_ctx,
static void sock_pe_report_read_completion(struct sock_pe_entry *pe_entry)
{
if (!(pe_entry->flags & SOCK_NO_COMPLETION))
- sock_pe_report_tx_completion(pe_entry);
+ sock_pe_report_send_cq_completion(pe_entry);
- if (pe_entry->comp->read_cntr &&
- pe_entry->comp->read_cntr != pe_entry->comp->send_cntr)
+ if (pe_entry->comp->read_cntr)
sock_cntr_inc(pe_entry->comp->read_cntr);
}
-static void sock_pe_report_rx_error(struct sock_pe_entry *pe_entry, int rem)
+static void sock_pe_report_rx_error(struct sock_pe_entry *pe_entry, int rem, int err)
{
if (pe_entry->comp->recv_cntr)
sock_cntr_err_inc(pe_entry->comp->recv_cntr);
if (pe_entry->comp->recv_cq)
sock_cq_report_error(pe_entry->comp->recv_cq, pe_entry, rem,
- FI_ETRUNC, -FI_ETRUNC, NULL);
+ err, -err, NULL);
}
static void sock_pe_report_tx_rma_read_err(struct sock_pe_entry *pe_entry,
@@ -489,7 +501,7 @@ static int sock_pe_handle_ack(struct sock_pe *pe,
waiting_entry, response->pe_entry_id);
assert(waiting_entry->type == SOCK_PE_TX);
- sock_pe_report_tx_completion(waiting_entry);
+ sock_pe_report_send_completion(waiting_entry);
waiting_entry->is_complete = 1;
pe_entry->is_complete = 1;
return 0;
@@ -650,7 +662,7 @@ static int sock_pe_process_rx_read(struct sock_pe *pe,
mr = sock_mr_verify_key(rx_ctx->domain,
pe_entry->pe.rx.rx_iov[i].iov.key,
- (void *) (uintptr_t) pe_entry->pe.rx.rx_iov[i].iov.addr,
+ (uintptr_t *) &pe_entry->pe.rx.rx_iov[i].iov.addr,
pe_entry->pe.rx.rx_iov[i].iov.len,
FI_REMOTE_READ);
if (!mr) {
@@ -665,8 +677,6 @@ static int sock_pe_process_rx_read(struct sock_pe *pe,
return 0;
}
- if (mr->domain->attr.mr_mode == FI_MR_SCALABLE)
- pe_entry->pe.rx.rx_iov[i].iov.addr += mr->offset;
data_len += pe_entry->pe.rx.rx_iov[i].iov.len;
}
pe_entry->mr_checked = 1;
@@ -703,7 +713,7 @@ static int sock_pe_process_rx_write(struct sock_pe *pe,
for (i = 0; i < pe_entry->msg_hdr.dest_iov_len && !pe_entry->mr_checked; i++) {
mr = sock_mr_verify_key(rx_ctx->domain,
pe_entry->pe.rx.rx_iov[i].iov.key,
- (void *) (uintptr_t) pe_entry->pe.rx.rx_iov[i].iov.addr,
+ (uintptr_t *) &pe_entry->pe.rx.rx_iov[i].iov.addr,
pe_entry->pe.rx.rx_iov[i].iov.len,
FI_REMOTE_WRITE);
if (!mr) {
@@ -717,9 +727,6 @@ static int sock_pe_process_rx_write(struct sock_pe *pe,
SOCK_OP_WRITE_ERROR, FI_EACCES);
return 0;
}
-
- if (mr->domain->attr.mr_mode == FI_MR_SCALABLE)
- pe_entry->pe.rx.rx_iov[i].iov.addr += mr->offset;
}
pe_entry->mr_checked = 1;
@@ -740,12 +747,8 @@ static int sock_pe_process_rx_write(struct sock_pe *pe,
/* report error, if any */
if (rem) {
- sock_pe_report_rx_error(pe_entry, rem);
+ sock_pe_report_rx_error(pe_entry, rem, FI_ETRUNC);
goto out;
- } else {
- if (pe_entry->flags & FI_REMOTE_CQ_DATA) {
- sock_pe_report_rx_completion(pe_entry);
- }
}
out:
@@ -1196,7 +1199,7 @@ static int sock_pe_process_rx_atomic(struct sock_pe *pe,
for (i = 0; i < pe_entry->pe.rx.rx_op.dest_iov_len && !pe_entry->mr_checked; i++) {
mr = sock_mr_verify_key(rx_ctx->domain,
pe_entry->pe.rx.rx_iov[i].ioc.key,
- (void *) (uintptr_t) pe_entry->pe.rx.rx_iov[i].ioc.addr,
+ (uintptr_t *) &pe_entry->pe.rx.rx_iov[i].ioc.addr,
pe_entry->pe.rx.rx_iov[i].ioc.count * datatype_sz,
FI_REMOTE_WRITE);
if (!mr) {
@@ -1210,8 +1213,6 @@ static int sock_pe_process_rx_atomic(struct sock_pe *pe,
SOCK_OP_ATOMIC_ERROR, FI_EACCES);
return 0;
}
- if (mr->domain->attr.mr_mode == FI_MR_SCALABLE)
- pe_entry->pe.rx.rx_iov[i].ioc.addr += mr->offset;
}
pe_entry->mr_checked = 1;
@@ -1245,10 +1246,6 @@ static int sock_pe_process_rx_atomic(struct sock_pe *pe,
pe_entry->buf = pe_entry->pe.rx.rx_iov[0].iov.addr;
pe_entry->data_len = offset;
- if (pe_entry->flags & FI_REMOTE_CQ_DATA) {
- sock_pe_report_rx_completion(pe_entry);
- }
-
pe_entry->flags |= FI_ATOMIC;
if (pe_entry->pe.rx.rx_op.atomic.op == FI_ATOMIC_READ)
pe_entry->flags |= FI_REMOTE_READ;
@@ -1285,6 +1282,7 @@ ssize_t sock_rx_peek_recv(struct sock_rx_ctx *rx_ctx, fi_addr_t addr,
if (rx_buffered) {
pe_entry.data_len = rx_buffered->total_len;
pe_entry.tag = rx_buffered->tag;
+ pe_entry.data = rx_buffered->data;
rx_buffered->context = (uintptr_t)context;
if (flags & FI_CLAIM)
rx_buffered->is_claimed = 1;
@@ -1293,7 +1291,7 @@ ssize_t sock_rx_peek_recv(struct sock_rx_ctx *rx_ctx, fi_addr_t addr,
dlist_remove(&rx_buffered->entry);
sock_rx_release_entry(rx_buffered);
}
- sock_pe_report_rx_completion(&pe_entry);
+ sock_pe_report_recv_completion(&pe_entry);
} else {
sock_cq_report_error(rx_ctx->comp.recv_cq, &pe_entry, 0,
FI_ENOMSG, -FI_ENOMSG, NULL);
@@ -1331,6 +1329,7 @@ ssize_t sock_rx_claim_recv(struct sock_rx_ctx *rx_ctx, void *context,
pe_entry.comp = &rx_ctx->comp;
pe_entry.data_len = rx_buffered->total_len;
pe_entry.tag = rx_buffered->tag;
+ pe_entry.data = rx_buffered->data;
pe_entry.context = rx_buffered->context;
pe_entry.flags = (flags | FI_MSG | FI_RECV);
if (is_tagged)
@@ -1352,9 +1351,9 @@ ssize_t sock_rx_claim_recv(struct sock_rx_ctx *rx_ctx, void *context,
if (rem) {
SOCK_LOG_DBG("Not enough space in posted recv buffer\n");
- sock_pe_report_rx_error(&pe_entry, rem);
+ sock_pe_report_rx_error(&pe_entry, rem, FI_ETRUNC);
} else {
- sock_pe_report_rx_completion(&pe_entry);
+ sock_pe_report_recv_completion(&pe_entry);
}
dlist_remove(&rx_buffered->entry);
@@ -1431,6 +1430,7 @@ static int sock_pe_progress_buffered_rx(struct sock_rx_ctx *rx_ctx)
pe_entry.comp = rx_buffered->comp;
pe_entry.flags = rx_posted->flags;
pe_entry.flags |= (FI_MSG | FI_RECV);
+ pe_entry.addr = rx_buffered->addr;
if (rx_buffered->is_tagged)
pe_entry.flags |= FI_TAGGED;
pe_entry.flags &= ~FI_MULTI_RECV;
@@ -1446,9 +1446,9 @@ static int sock_pe_progress_buffered_rx(struct sock_rx_ctx *rx_ctx)
if (rem) {
SOCK_LOG_DBG("Not enough space in posted recv buffer\n");
- sock_pe_report_rx_error(&pe_entry, rem);
+ sock_pe_report_rx_error(&pe_entry, rem, FI_ETRUNC);
} else {
- sock_pe_report_rx_completion(&pe_entry);
+ sock_pe_report_recv_completion(&pe_entry);
}
dlist_remove(&rx_buffered->entry);
@@ -1584,13 +1584,13 @@ static int sock_pe_process_rx_send(struct sock_pe *pe,
/* report error, if any */
if (rem) {
SOCK_LOG_ERROR("Not enough space in posted recv buffer\n");
- sock_pe_report_rx_error(pe_entry, rem);
+ sock_pe_report_rx_error(pe_entry, rem, FI_ETRUNC);
pe_entry->is_error = 1;
pe_entry->rem = pe_entry->total_len - pe_entry->done_len;
goto out;
} else {
if (!rx_entry->is_buffered)
- sock_pe_report_rx_completion(pe_entry);
+ sock_pe_report_recv_completion(pe_entry);
}
out:
@@ -1615,7 +1615,7 @@ static int sock_pe_process_rx_conn_msg(struct sock_pe *pe,
struct sock_pe_entry *pe_entry)
{
uint64_t len, data_len;
- struct sock_ep *ep;
+ struct sock_ep_attr *ep_attr;
struct sock_conn_map *map;
struct sockaddr_in *addr;
struct sock_conn *conn;
@@ -1640,20 +1640,22 @@ static int sock_pe_process_rx_conn_msg(struct sock_pe *pe,
inet_ntoa(((struct sockaddr_in *)pe_entry->comm_addr)->sin_addr),
ntohs(((struct sockaddr_in *)pe_entry->comm_addr)->sin_port));
- ep = pe_entry->conn->ep;
- map = &ep->cmap;
+ ep_attr = pe_entry->conn->ep_attr;
+ map = &ep_attr->cmap;
addr = (struct sockaddr_in *) pe_entry->comm_addr;
pe_entry->conn->addr = *addr;
- index = (ep->ep_type == FI_EP_MSG) ? 0 : sock_av_get_addr_index(ep->av, addr);
+ index = (ep_attr->ep_type == FI_EP_MSG) ? 0 : sock_av_get_addr_index(ep_attr->av, addr);
if (index != -1) {
fastlock_acquire(&map->lock);
- conn = sock_ep_lookup_conn(ep, index, addr);
- if (conn == NULL || conn == SOCK_CM_CONN_IN_PROGRESS)
- idm_set(&ep->av_idm, index, pe_entry->conn);
+ conn = sock_ep_lookup_conn(ep_attr, index, addr);
+ if (conn == NULL || conn == SOCK_CM_CONN_IN_PROGRESS) {
+ if (idm_set(&ep_attr->av_idm, index, pe_entry->conn) < 0)
+ SOCK_LOG_ERROR("idm_set failed\n");
+ }
fastlock_release(&map->lock);
}
- pe_entry->conn->av_index = (ep->ep_type == FI_EP_MSG || index == -1) ?
+ pe_entry->conn->av_index = (ep_attr->ep_type == FI_EP_MSG || index == -1) ?
FI_ADDR_NOTAVAIL : index;
pe_entry->is_complete = 1;
@@ -1826,25 +1828,31 @@ static int sock_pe_progress_tx_atomic(struct sock_pe *pe,
return 0;
len += entry_len;
- /* cmp data */
datatype_sz = fi_datatype_size(pe_entry->pe.tx.tx_op.atomic.datatype);
- for (i = 0; i < pe_entry->pe.tx.tx_op.atomic.cmp_iov_len; i++) {
+ if (pe_entry->flags & FI_INJECT) {
+ /* cmp data */
if (sock_pe_send_field(pe_entry,
- (void *) (uintptr_t) pe_entry->pe.tx.tx_iov[i].cmp.ioc.addr,
- pe_entry->pe.tx.tx_iov[i].cmp.ioc.count *
- datatype_sz, len))
+ &pe_entry->pe.tx.inject[0] + pe_entry->pe.tx.tx_op.src_iov_len,
+ pe_entry->pe.tx.tx_op.atomic.cmp_iov_len, len))
return 0;
- len += (pe_entry->pe.tx.tx_iov[i].cmp.ioc.count * datatype_sz);
- }
-
- /* data */
- if (pe_entry->flags & FI_INJECT) {
+ len += pe_entry->pe.tx.tx_op.atomic.cmp_iov_len;
+ /* data */
if (sock_pe_send_field(pe_entry,
&pe_entry->pe.tx.inject[0],
pe_entry->pe.tx.tx_op.src_iov_len, len))
return 0;
len += pe_entry->pe.tx.tx_op.src_iov_len;
} else {
+ /* cmp data */
+ for (i = 0; i < pe_entry->pe.tx.tx_op.atomic.cmp_iov_len; i++) {
+ if (sock_pe_send_field(pe_entry,
+ (void *) (uintptr_t) pe_entry->pe.tx.tx_iov[i].cmp.ioc.addr,
+ pe_entry->pe.tx.tx_iov[i].cmp.ioc.count *
+ datatype_sz, len))
+ return 0;
+ len += (pe_entry->pe.tx.tx_iov[i].cmp.ioc.count * datatype_sz);
+ }
+ /* data */
for (i = 0; i < pe_entry->pe.tx.tx_op.src_iov_len; i++) {
if (pe_entry->pe.tx.tx_op.atomic.op != FI_ATOMIC_READ) {
if (sock_pe_send_field(pe_entry,
@@ -2037,7 +2045,7 @@ static int sock_pe_progress_tx_send(struct sock_pe *pe,
SOCK_LOG_DBG("Send complete\n");
if (pe_entry->flags & FI_INJECT_COMPLETE) {
- sock_pe_report_tx_completion(pe_entry);
+ sock_pe_report_send_completion(pe_entry);
pe_entry->is_complete = 1;
}
}
@@ -2140,8 +2148,20 @@ static int sock_pe_progress_rx_pe_entry(struct sock_pe *pe,
{
int ret;
- if (pe_entry->conn->disconnected)
+ if (sock_comm_is_disconnected(pe_entry)) {
+ SOCK_LOG_DBG("conn disconnected: removing fd from pollset\n");
+ if (pe_entry->conn->sock_fd != -1) {
+ fastlock_acquire(&pe_entry->ep_attr->cmap.lock);
+ sock_ep_remove_conn(pe_entry->ep_attr, pe_entry->conn);
+ fastlock_release(&pe_entry->ep_attr->cmap.lock);
+ }
+
+ if (pe_entry->pe.rx.header_read)
+ sock_pe_report_rx_error(pe_entry, 0, FI_EIO);
+
+ sock_pe_release_entry(pe, pe_entry);
return 0;
+ }
if (pe_entry->pe.rx.pending_send) {
sock_pe_progress_pending_ack(pe, pe_entry);
@@ -2176,7 +2196,7 @@ out:
}
static void sock_pe_new_rx_entry(struct sock_pe *pe, struct sock_rx_ctx *rx_ctx,
- struct sock_ep *ep, struct sock_conn *conn)
+ struct sock_ep_attr *ep_attr, struct sock_conn *conn)
{
struct sock_pe_entry *pe_entry;
@@ -2185,17 +2205,17 @@ static void sock_pe_new_rx_entry(struct sock_pe *pe, struct sock_rx_ctx *rx_ctx,
pe_entry->conn = conn;
pe_entry->type = SOCK_PE_RX;
- pe_entry->ep = ep;
+ pe_entry->ep_attr = ep_attr;
pe_entry->is_complete = 0;
pe_entry->done_len = 0;
- if (ep->ep_type == FI_EP_MSG || !ep->av)
+ if (ep_attr->ep_type == FI_EP_MSG || !ep_attr->av)
pe_entry->addr = FI_ADDR_NOTAVAIL;
else
pe_entry->addr = conn->av_index;
if (rx_ctx->ctx.fid.fclass == FI_CLASS_SRX_CTX)
- pe_entry->comp = &ep->comp;
+ pe_entry->comp = &ep_attr->rx_ctx->comp;
else
pe_entry->comp = &rx_ctx->comp;
@@ -2213,7 +2233,7 @@ static int sock_pe_new_tx_entry(struct sock_pe *pe, struct sock_tx_ctx *tx_ctx)
int i, datatype_sz;
struct sock_msg_hdr *msg_hdr;
struct sock_pe_entry *pe_entry;
- struct sock_ep *ep;
+ struct sock_ep_attr *ep_attr;
pe_entry = sock_pe_acquire_entry(pe);
memset(&pe_entry->pe.tx, 0, sizeof(pe_entry->pe.tx));
@@ -2223,7 +2243,7 @@ static int sock_pe_new_tx_entry(struct sock_pe *pe, struct sock_tx_ctx *tx_ctx)
pe_entry->is_complete = 0;
pe_entry->done_len = 0;
pe_entry->conn = NULL;
- pe_entry->ep = tx_ctx->ep;
+ pe_entry->ep_attr = tx_ctx->ep_attr;
pe_entry->pe.tx.tx_ctx = tx_ctx;
dlist_insert_tail(&pe_entry->ctx_entry, &tx_ctx->pe_entry_list);
@@ -2238,15 +2258,15 @@ static int sock_pe_new_tx_entry(struct sock_pe *pe, struct sock_tx_ctx *tx_ctx)
sock_tx_ctx_read_op_send(tx_ctx, &pe_entry->pe.tx.tx_op,
&pe_entry->flags, &pe_entry->context, &pe_entry->addr,
- &pe_entry->buf, &ep, &pe_entry->conn);
+ &pe_entry->buf, &ep_attr, &pe_entry->conn);
if (pe_entry->pe.tx.tx_op.op == SOCK_OP_TSEND) {
rbread(&tx_ctx->rb, &pe_entry->tag, sizeof(pe_entry->tag));
msg_hdr->msg_len += sizeof(pe_entry->tag);
}
- if (ep && tx_ctx->fclass == FI_CLASS_STX_CTX)
- pe_entry->comp = &ep->comp;
+ if (ep_attr && tx_ctx->fclass == FI_CLASS_STX_CTX)
+ pe_entry->comp = &ep_attr->tx_ctx->comp;
else
pe_entry->comp = &tx_ctx->comp;
@@ -2313,7 +2333,11 @@ static int sock_pe_new_tx_entry(struct sock_pe *pe, struct sock_tx_ctx *tx_ctx)
if (pe_entry->flags & FI_INJECT) {
rbread(&tx_ctx->rb, &pe_entry->pe.tx.inject[0],
pe_entry->pe.tx.tx_op.src_iov_len);
- msg_hdr->msg_len += pe_entry->pe.tx.tx_op.src_iov_len;
+ rbread(&tx_ctx->rb, &pe_entry->pe.tx.inject[0] +
+ pe_entry->pe.tx.tx_op.src_iov_len,
+ pe_entry->pe.tx.tx_op.atomic.cmp_iov_len);
+ msg_hdr->msg_len += pe_entry->pe.tx.tx_op.src_iov_len +
+ pe_entry->pe.tx.tx_op.atomic.cmp_iov_len;
} else {
for (i = 0; i < pe_entry->pe.tx.tx_op.src_iov_len; i++) {
rbread(&tx_ctx->rb, &pe_entry->pe.tx.tx_iov[i].src,
@@ -2323,6 +2347,12 @@ static int sock_pe_new_tx_entry(struct sock_pe *pe, struct sock_tx_ctx *tx_ctx)
msg_hdr->msg_len += datatype_sz *
pe_entry->pe.tx.tx_iov[i].src.ioc.count;
}
+ for (i = 0; i < pe_entry->pe.tx.tx_op.atomic.cmp_iov_len; i++) {
+ rbread(&tx_ctx->rb, &pe_entry->pe.tx.tx_iov[i].cmp,
+ sizeof(pe_entry->pe.tx.tx_iov[i].cmp));
+ msg_hdr->msg_len += datatype_sz *
+ pe_entry->pe.tx.tx_iov[i].cmp.ioc.count;
+ }
}
for (i = 0; i < pe_entry->pe.tx.tx_op.dest_iov_len; i++) {
@@ -2336,12 +2366,6 @@ static int sock_pe_new_tx_entry(struct sock_pe *pe, struct sock_tx_ctx *tx_ctx)
sizeof(pe_entry->pe.tx.tx_iov[i].res));
}
- for (i = 0; i < pe_entry->pe.tx.tx_op.atomic.cmp_iov_len; i++) {
- rbread(&tx_ctx->rb, &pe_entry->pe.tx.tx_iov[i].cmp,
- sizeof(pe_entry->pe.tx.tx_iov[i].cmp));
- msg_hdr->msg_len += datatype_sz *
- pe_entry->pe.tx.tx_iov[i].cmp.ioc.count;
- }
msg_hdr->dest_iov_len = pe_entry->pe.tx.tx_op.dest_iov_len;
break;
case SOCK_OP_CONN_MSG:
@@ -2353,7 +2377,6 @@ static int sock_pe_new_tx_entry(struct sock_pe *pe, struct sock_tx_ctx *tx_ctx)
SOCK_LOG_ERROR("Invalid operation type\n");
return -FI_EINVAL;
}
-
SOCK_LOG_DBG("Inserting TX-entry to PE entry %p, conn: %p\n",
pe_entry, pe_entry->conn);
@@ -2384,7 +2407,7 @@ void sock_pe_signal(struct sock_pe *pe)
fastlock_acquire(&pe->signal_lock);
if (pe->wcnt == pe->rcnt) {
- if (write(pe->signal_fds[SOCK_SIGNAL_WR_FD], &c, 1) != 1)
+ if (ofi_write_socket(pe->signal_fds[SOCK_SIGNAL_WR_FD], &c, 1) != 1)
SOCK_LOG_ERROR("Failed to signal\n");
else
pe->wcnt++;
@@ -2400,20 +2423,48 @@ void sock_pe_poll_add(struct sock_pe *pe, int fd)
fastlock_release(&pe->signal_lock);
}
+void sock_pe_poll_del(struct sock_pe *pe, int fd)
+{
+ fastlock_acquire(&pe->signal_lock);
+ if (sock_epoll_del(&pe->epoll_set, fd))
+ SOCK_LOG_DBG("failed to del from epoll set: %d, size: %d, used: %d\n",
+ fd, pe->epoll_set.size, pe->epoll_set.used);
+ fastlock_release(&pe->signal_lock);
+}
+
void sock_pe_add_tx_ctx(struct sock_pe *pe, struct sock_tx_ctx *ctx)
{
+ struct dlist_entry *entry;
+ struct sock_tx_ctx *curr_ctx;
pthread_mutex_lock(&pe->list_lock);
+ for (entry = pe->tx_list.next; entry != &pe->tx_list;
+ entry = entry->next) {
+ curr_ctx = container_of(entry, struct sock_tx_ctx, pe_entry);
+ if (curr_ctx == ctx)
+ goto out;
+ }
+
dlist_insert_tail(&ctx->pe_entry, &pe->tx_list);
sock_pe_signal(pe);
+out:
pthread_mutex_unlock(&pe->list_lock);
SOCK_LOG_DBG("TX ctx added to PE\n");
}
void sock_pe_add_rx_ctx(struct sock_pe *pe, struct sock_rx_ctx *ctx)
{
+ struct dlist_entry *entry;
+ struct sock_rx_ctx *curr_ctx;
pthread_mutex_lock(&pe->list_lock);
+ for (entry = pe->rx_list.next; entry != &pe->rx_list;
+ entry = entry->next) {
+ curr_ctx = container_of(entry, struct sock_rx_ctx, pe_entry);
+ if (curr_ctx == ctx)
+ goto out;
+ }
dlist_insert_tail(&ctx->pe_entry, &pe->rx_list);
sock_pe_signal(pe);
+out:
pthread_mutex_unlock(&pe->list_lock);
SOCK_LOG_DBG("RX ctx added to PE\n");
}
@@ -2432,14 +2483,14 @@ void sock_pe_remove_rx_ctx(struct sock_rx_ctx *rx_ctx)
pthread_mutex_unlock(&rx_ctx->domain->pe->list_lock);
}
-static int sock_pe_progress_rx_ep(struct sock_pe *pe, struct sock_ep *ep,
+static int sock_pe_progress_rx_ep(struct sock_pe *pe, struct sock_ep_attr *ep_attr,
struct sock_rx_ctx *rx_ctx)
{
int ret = 0, i, fd, num_fds;
struct sock_conn *conn;
struct sock_conn_map *map;
- map = &ep->cmap;
+ map = &ep_attr->cmap;
if (!map->used)
return 0;
@@ -2454,14 +2505,17 @@ static int sock_pe_progress_rx_ep(struct sock_pe *pe, struct sock_ep *ep,
fastlock_acquire(&map->lock);
for (i = 0; i < num_fds; i++) {
fd = sock_epoll_get_fd_at_index(&map->epoll_set, i);
- conn = idm_lookup(&ep->conn_idm, fd);
+ if (fd == -1) /* failed to lookup fd due to connection failures */
+ continue;
+
+ conn = idm_lookup(&ep_attr->conn_idm, fd);
if (!conn)
SOCK_LOG_ERROR("idm_lookup failed\n");
if (!conn || conn->rx_pe_entry)
continue;
- sock_pe_new_rx_entry(pe, rx_ctx, ep, conn);
+ sock_pe_new_rx_entry(pe, rx_ctx, ep_attr, conn);
}
fastlock_release(&map->lock);
@@ -2471,7 +2525,7 @@ static int sock_pe_progress_rx_ep(struct sock_pe *pe, struct sock_ep *ep,
int sock_pe_progress_rx_ctx(struct sock_pe *pe, struct sock_rx_ctx *rx_ctx)
{
int ret = 0;
- struct sock_ep *ep;
+ struct sock_ep_attr *ep_attr;
struct dlist_entry *entry;
struct sock_pe_entry *pe_entry;
@@ -2485,15 +2539,15 @@ int sock_pe_progress_rx_ctx(struct sock_pe *pe, struct sock_rx_ctx *rx_ctx)
if (rx_ctx->ctx.fid.fclass == FI_CLASS_SRX_CTX) {
for (entry = rx_ctx->ep_list.next;
entry != &rx_ctx->ep_list;) {
- ep = container_of(entry, struct sock_ep, rx_ctx_entry);
+ ep_attr = container_of(entry, struct sock_ep_attr, rx_ctx_entry);
entry = entry->next;
- ret = sock_pe_progress_rx_ep(pe, ep, rx_ctx);
+ ret = sock_pe_progress_rx_ep(pe, ep_attr, rx_ctx);
if (ret < 0)
goto out;
}
} else {
- ep = rx_ctx->ep;
- ret = sock_pe_progress_rx_ep(pe, ep, rx_ctx);
+ ep_attr = rx_ctx->ep_attr;
+ ret = sock_pe_progress_rx_ep(pe, ep_attr, rx_ctx);
if (ret < 0)
goto out;
}
@@ -2516,19 +2570,19 @@ out:
void sock_pe_progress_rx_ctrl_ctx(struct sock_pe *pe, struct sock_rx_ctx *rx_ctx,
struct sock_tx_ctx *tx_ctx)
{
- struct sock_ep *ep;
+ struct sock_ep_attr *ep_attr;
struct dlist_entry *entry;
struct sock_pe_entry *pe_entry;
/* check for incoming data */
if (tx_ctx->fclass == FI_CLASS_STX_CTX) {
for (entry = tx_ctx->ep_list.next; entry != &tx_ctx->ep_list;) {
- ep = container_of(entry, struct sock_ep, tx_ctx_entry);
+ ep_attr = container_of(entry, struct sock_ep_attr, tx_ctx_entry);
entry = entry->next;
- sock_pe_progress_rx_ep(pe, ep, tx_ctx->rx_ctrl_ctx);
+ sock_pe_progress_rx_ep(pe, ep_attr, tx_ctx->rx_ctrl_ctx);
}
} else {
- sock_pe_progress_rx_ep(pe, tx_ctx->ep, tx_ctx->rx_ctrl_ctx);
+ sock_pe_progress_rx_ep(pe, tx_ctx->ep_attr, tx_ctx->rx_ctrl_ctx);
}
for (entry = rx_ctx->pe_entry_list.next;
@@ -2581,21 +2635,18 @@ out:
return ret;
}
-static void sock_pe_poll(struct sock_pe *pe)
+static int sock_pe_wait_ok(struct sock_pe *pe)
{
- char tmp;
- int ret;
struct dlist_entry *entry;
struct sock_tx_ctx *tx_ctx;
struct sock_rx_ctx *rx_ctx;
if (pe->waittime && ((fi_gettime_ms() - pe->waittime) < sock_pe_waittime))
- return;
+ return 0;
if (dlist_empty(&pe->tx_list) && dlist_empty(&pe->rx_list))
- return;
+ return 0;
- pthread_mutex_lock(&pe->list_lock);
if (!dlist_empty(&pe->tx_list)) {
for (entry = pe->tx_list.next;
entry != &pe->tx_list; entry = entry->next) {
@@ -2603,8 +2654,7 @@ static void sock_pe_poll(struct sock_pe *pe)
pe_entry);
if (!rbempty(&tx_ctx->rb) ||
!dlist_empty(&tx_ctx->pe_entry_list)) {
- pthread_mutex_unlock(&pe->list_lock);
- return;
+ return 0;
}
}
}
@@ -2616,12 +2666,18 @@ static void sock_pe_poll(struct sock_pe *pe)
pe_entry);
if (!dlist_empty(&rx_ctx->rx_buffered_list) ||
!dlist_empty(&rx_ctx->pe_entry_list)) {
- pthread_mutex_unlock(&pe->list_lock);
- return;
+ return 0;
}
}
}
- pthread_mutex_unlock(&pe->list_lock);
+
+ return 1;
+}
+
+static void sock_pe_wait(struct sock_pe *pe)
+{
+ char tmp;
+ int ret;
ret = sock_epoll_wait(&pe->epoll_set, -1);
if (ret < 0)
@@ -2629,7 +2685,7 @@ static void sock_pe_poll(struct sock_pe *pe)
fastlock_acquire(&pe->signal_lock);
if (pe->rcnt != pe->wcnt) {
- if (read(pe->signal_fds[SOCK_SIGNAL_RD_FD], &tmp, 1) == 1)
+ if (ofi_read_socket(pe->signal_fds[SOCK_SIGNAL_RD_FD], &tmp, 1) == 1)
pe->rcnt++;
else
SOCK_LOG_ERROR("Invalid signal\n");
@@ -2638,7 +2694,7 @@ static void sock_pe_poll(struct sock_pe *pe)
pe->waittime = fi_gettime_ms();
}
-#ifndef __APPLE__
+#if !defined __APPLE__ && !defined _WIN32
static void sock_thread_set_affinity(char *s)
{
char *saveptra = NULL, *saveptrb = NULL, *saveptrc = NULL;
@@ -2688,7 +2744,7 @@ static void sock_pe_set_affinity(void)
if (sock_pe_affinity_str == NULL)
return;
-#ifndef __APPLE__
+#if !defined __APPLE__ && !defined _WIN32
sock_thread_set_affinity(sock_pe_affinity_str);
#else
SOCK_LOG_ERROR("*** FI_SOCKETS_PE_AFFINITY is not supported on OS X\n");
@@ -2706,10 +2762,14 @@ static void *sock_pe_progress_thread(void *data)
SOCK_LOG_DBG("Progress thread started\n");
sock_pe_set_affinity();
while (*((volatile int *)&pe->do_progress)) {
- if (pe->domain->progress_mode == FI_PROGRESS_AUTO)
- sock_pe_poll(pe);
-
pthread_mutex_lock(&pe->list_lock);
+ if (pe->domain->progress_mode == FI_PROGRESS_AUTO &&
+ sock_pe_wait_ok(pe)) {
+ pthread_mutex_unlock(&pe->list_lock);
+ sock_pe_wait(pe);
+ pthread_mutex_lock(&pe->list_lock);
+ }
+
if (!dlist_empty(&pe->tx_list)) {
for (entry = pe->tx_list.next;
entry != &pe->tx_list; entry = entry->next) {
@@ -2753,6 +2813,7 @@ static void sock_pe_init_table(struct sock_pe *pe)
dlist_init(&pe->free_list);
dlist_init(&pe->busy_list);
+ dlist_init(&pe->pool_list);
for (i = 0; i < SOCK_PE_MAX_ENTRIES; i++) {
dlist_insert_head(&pe->pe_table[i].entry, &pe->free_list);
@@ -2817,8 +2878,8 @@ struct sock_pe *sock_pe_init(struct sock_domain *domain)
return pe;
err5:
- close(pe->signal_fds[0]);
- close(pe->signal_fds[1]);
+ ofi_close_socket(pe->signal_fds[0]);
+ ofi_close_socket(pe->signal_fds[1]);
err4:
sock_epoll_close(&pe->epoll_set);
err3:
@@ -2831,6 +2892,23 @@ err1:
return NULL;
}
+static void sock_pe_free_util_pool(struct sock_pe *pe)
+{
+ struct dlist_entry *entry;
+ struct sock_pe_entry *pe_entry;
+
+ while (!dlist_empty(&pe->pool_list)) {
+ entry = pe->pool_list.next;
+ pe_entry = container_of(entry, struct sock_pe_entry, entry);
+ rbfree(&pe_entry->comm_buf);
+ dlist_remove(&pe_entry->entry);
+ util_buf_release(pe->pe_rx_pool, pe_entry);
+ }
+
+ util_buf_pool_destroy(pe->pe_rx_pool);
+ util_buf_pool_destroy(pe->atomic_rx_pool);
+}
+
void sock_pe_finalize(struct sock_pe *pe)
{
int i;
@@ -2838,16 +2916,15 @@ void sock_pe_finalize(struct sock_pe *pe)
pe->do_progress = 0;
sock_pe_signal(pe);
pthread_join(pe->progress_thread, NULL);
- close(pe->signal_fds[0]);
- close(pe->signal_fds[1]);
+ ofi_close_socket(pe->signal_fds[0]);
+ ofi_close_socket(pe->signal_fds[1]);
}
for (i = 0; i < SOCK_PE_MAX_ENTRIES; i++) {
rbfree(&pe->pe_table[i].comm_buf);
}
- util_buf_pool_destroy(pe->pe_rx_pool);
- util_buf_pool_destroy(pe->atomic_rx_pool);
+ sock_pe_free_util_pool(pe);
fastlock_destroy(&pe->lock);
fastlock_destroy(&pe->signal_lock);
pthread_mutex_destroy(&pe->list_lock);
diff --git a/prov/sockets/src/sock_rma.c b/prov/sockets/src/sock_rma.c
index e626b4d..6234902 100644
--- a/prov/sockets/src/sock_rma.c
+++ b/prov/sockets/src/sock_rma.c
@@ -65,18 +65,23 @@ ssize_t sock_ep_rma_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
union sock_iov tx_iov;
struct sock_conn *conn;
struct sock_tx_ctx *tx_ctx;
- uint64_t total_len, src_len, dst_len;
+ uint64_t total_len, src_len, dst_len, op_flags;
struct sock_ep *sock_ep;
+ struct sock_ep_attr *ep_attr;
switch (ep->fid.fclass) {
case FI_CLASS_EP:
sock_ep = container_of(ep, struct sock_ep, ep);
- tx_ctx = sock_ep->tx_ctx;
+ tx_ctx = sock_ep->attr->tx_ctx->use_shared ?
+ sock_ep->attr->tx_ctx->stx_ctx : sock_ep->attr->tx_ctx;
+ ep_attr = sock_ep->attr;
+ op_flags = sock_ep->tx_attr.op_flags;
break;
case FI_CLASS_TX_CTX:
tx_ctx = container_of(ep, struct sock_tx_ctx, fid.ctx);
- sock_ep = tx_ctx->ep;
+ ep_attr = tx_ctx->ep_attr;
+ op_flags = tx_ctx->attr.op_flags;
break;
default:
@@ -93,13 +98,13 @@ ssize_t sock_ep_rma_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
if (!tx_ctx->enabled)
return -FI_EOPBADSTATE;
- ret = sock_ep_get_conn(sock_ep, tx_ctx, msg->addr, &conn);
+ ret = sock_ep_get_conn(ep_attr, tx_ctx, msg->addr, &conn);
if (ret)
return ret;
SOCK_EP_SET_TX_OP_FLAGS(flags);
if (flags & SOCK_USE_OP_FLAGS)
- flags |= tx_ctx->attr.op_flags;
+ flags |= op_flags;
if (flags & FI_TRIGGER) {
ret = sock_queue_rma_op(ep, msg, flags, SOCK_OP_READ);
@@ -125,7 +130,7 @@ ssize_t sock_ep_rma_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
sock_tx_ctx_write_op_send(tx_ctx, &tx_op, flags,
(uintptr_t) msg->context, msg->addr,
(uintptr_t) msg->msg_iov[0].iov_base,
- sock_ep, conn);
+ ep_attr, conn);
src_len = 0;
for (i = 0; i < msg->rma_iov_count; i++) {
@@ -224,18 +229,23 @@ ssize_t sock_ep_rma_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
union sock_iov tx_iov;
struct sock_conn *conn;
struct sock_tx_ctx *tx_ctx;
- uint64_t total_len, src_len, dst_len;
+ uint64_t total_len, src_len, dst_len, op_flags;
struct sock_ep *sock_ep;
+ struct sock_ep_attr *ep_attr;
switch (ep->fid.fclass) {
case FI_CLASS_EP:
sock_ep = container_of(ep, struct sock_ep, ep);
- tx_ctx = sock_ep->tx_ctx;
+ tx_ctx = sock_ep->attr->tx_ctx->use_shared ?
+ sock_ep->attr->tx_ctx->stx_ctx : sock_ep->attr->tx_ctx;
+ ep_attr = sock_ep->attr;
+ op_flags = sock_ep->tx_attr.op_flags;
break;
case FI_CLASS_TX_CTX:
tx_ctx = container_of(ep, struct sock_tx_ctx, fid.ctx);
- sock_ep = tx_ctx->ep;
+ ep_attr = tx_ctx->ep_attr;
+ op_flags = tx_ctx->attr.op_flags;
break;
default:
@@ -252,13 +262,13 @@ ssize_t sock_ep_rma_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
if (!tx_ctx->enabled)
return -FI_EOPBADSTATE;
- ret = sock_ep_get_conn(sock_ep, tx_ctx, msg->addr, &conn);
+ ret = sock_ep_get_conn(ep_attr, tx_ctx, msg->addr, &conn);
if (ret)
return ret;
SOCK_EP_SET_TX_OP_FLAGS(flags);
if (flags & SOCK_USE_OP_FLAGS)
- flags |= tx_ctx->attr.op_flags;
+ flags |= op_flags;
if (flags & FI_TRIGGER) {
ret = sock_queue_rma_op(ep, msg, flags, SOCK_OP_WRITE);
@@ -295,7 +305,7 @@ ssize_t sock_ep_rma_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
sock_tx_ctx_write_op_send(tx_ctx, &tx_op, flags,
(uintptr_t) msg->context, msg->addr,
- (uintptr_t) msg->msg_iov[0].iov_base, sock_ep, conn);
+ (uintptr_t) msg->msg_iov[0].iov_base, ep_attr, conn);
if (flags & FI_REMOTE_CQ_DATA)
sock_tx_ctx_write(tx_ctx, &msg->data, sizeof(msg->data));
diff --git a/prov/sockets/src/sock_rx_entry.c b/prov/sockets/src/sock_rx_entry.c
index c1cdbce..b8449ba 100644
--- a/prov/sockets/src/sock_rx_entry.c
+++ b/prov/sockets/src/sock_rx_entry.c
@@ -131,11 +131,6 @@ struct sock_rx_entry *sock_rx_new_buffered_entry(struct sock_rx_ctx *rx_ctx,
return rx_entry;
}
-inline size_t sock_rx_avail_len(struct sock_rx_entry *rx_entry)
-{
- return rx_entry->total_len - rx_entry->used;
-}
-
struct sock_rx_entry *sock_rx_get_entry(struct sock_rx_ctx *rx_ctx,
uint64_t addr, uint64_t tag,
uint8_t is_tagged)
diff --git a/prov/sockets/src/sock_trigger.c b/prov/sockets/src/sock_trigger.c
index 9f013de..3072425 100644
--- a/prov/sockets/src/sock_trigger.c
+++ b/prov/sockets/src/sock_trigger.c
@@ -52,8 +52,8 @@ ssize_t sock_queue_rma_op(struct fid_ep *ep, const struct fi_msg_rma *msg,
struct fi_trigger_threshold *threshold;
trigger_context = (struct fi_triggered_context *) msg->context;
- if ((trigger_context->event_type != FI_TRIGGER_THRESHOLD) ||
- (flags & FI_INJECT))
+ if ((flags & FI_INJECT) || !trigger_context ||
+ (trigger_context->event_type != FI_TRIGGER_THRESHOLD))
return -FI_EINVAL;
threshold = &trigger_context->trigger.threshold;
@@ -82,6 +82,7 @@ ssize_t sock_queue_rma_op(struct fid_ep *ep, const struct fi_msg_rma *msg,
fastlock_acquire(&cntr->trigger_lock);
dlist_insert_tail(&trigger->entry, &cntr->trigger_list);
fastlock_release(&cntr->trigger_lock);
+ sock_cntr_check_trigger_list(cntr);
return 0;
}
@@ -94,8 +95,8 @@ ssize_t sock_queue_msg_op(struct fid_ep *ep, const struct fi_msg *msg,
struct fi_trigger_threshold *threshold;
trigger_context = (struct fi_triggered_context *) msg->context;
- if ((trigger_context->event_type != FI_TRIGGER_THRESHOLD) ||
- (flags & FI_INJECT))
+ if ((flags & FI_INJECT) || !trigger_context ||
+ (trigger_context->event_type != FI_TRIGGER_THRESHOLD))
return -FI_EINVAL;
threshold = &trigger_context->trigger.threshold;
@@ -110,8 +111,8 @@ ssize_t sock_queue_msg_op(struct fid_ep *ep, const struct fi_msg *msg,
trigger->threshold = threshold->threshold;
memcpy(&trigger->op.msg.msg, msg, sizeof(*msg));
- trigger->op.msg.msg.msg_iov = &trigger->op.msg.msg.msg_iov[0];
- memcpy((void *) &trigger->op.msg.msg.msg_iov[0], &msg->msg_iov[0],
+ trigger->op.msg.msg.msg_iov = &trigger->op.msg.msg_iov[0];
+ memcpy((void *) &trigger->op.msg.msg_iov[0], &msg->msg_iov[0],
msg->iov_count * sizeof(struct iovec));
trigger->op_type = op_type;
@@ -121,6 +122,7 @@ ssize_t sock_queue_msg_op(struct fid_ep *ep, const struct fi_msg *msg,
fastlock_acquire(&cntr->trigger_lock);
dlist_insert_tail(&trigger->entry, &cntr->trigger_list);
fastlock_release(&cntr->trigger_lock);
+ sock_cntr_check_trigger_list(cntr);
return 0;
}
@@ -133,8 +135,8 @@ ssize_t sock_queue_tmsg_op(struct fid_ep *ep, const struct fi_msg_tagged *msg,
struct fi_trigger_threshold *threshold;
trigger_context = (struct fi_triggered_context *) msg->context;
- if ((trigger_context->event_type != FI_TRIGGER_THRESHOLD) ||
- (flags & FI_INJECT))
+ if ((flags & FI_INJECT) || !trigger_context ||
+ (trigger_context->event_type != FI_TRIGGER_THRESHOLD))
return -FI_EINVAL;
threshold = &trigger_context->trigger.threshold;
@@ -149,8 +151,8 @@ ssize_t sock_queue_tmsg_op(struct fid_ep *ep, const struct fi_msg_tagged *msg,
trigger->threshold = threshold->threshold;
memcpy(&trigger->op.tmsg.msg, msg, sizeof(*msg));
- trigger->op.tmsg.msg.msg_iov = &trigger->op.tmsg.msg.msg_iov[0];
- memcpy((void *) &trigger->op.tmsg.msg.msg_iov[0], &msg->msg_iov[0],
+ trigger->op.tmsg.msg.msg_iov = &trigger->op.tmsg.msg_iov[0];
+ memcpy((void *) &trigger->op.tmsg.msg_iov[0], &msg->msg_iov[0],
msg->iov_count * sizeof(struct iovec));
trigger->op_type = op_type;
@@ -160,6 +162,7 @@ ssize_t sock_queue_tmsg_op(struct fid_ep *ep, const struct fi_msg_tagged *msg,
fastlock_acquire(&cntr->trigger_lock);
dlist_insert_tail(&trigger->entry, &cntr->trigger_list);
fastlock_release(&cntr->trigger_lock);
+ sock_cntr_check_trigger_list(cntr);
return 0;
}
@@ -174,8 +177,8 @@ ssize_t sock_queue_atomic_op(struct fid_ep *ep, const struct fi_msg_atomic *msg,
struct fi_trigger_threshold *threshold;
trigger_context = (struct fi_triggered_context *) msg->context;
- if ((trigger_context->event_type != FI_TRIGGER_THRESHOLD) ||
- (flags & FI_INJECT))
+ if ((flags & FI_INJECT) || !trigger_context ||
+ (trigger_context->event_type != FI_TRIGGER_THRESHOLD))
return -FI_EINVAL;
threshold = &trigger_context->trigger.threshold;
@@ -214,5 +217,6 @@ ssize_t sock_queue_atomic_op(struct fid_ep *ep, const struct fi_msg_atomic *msg,
fastlock_acquire(&cntr->trigger_lock);
dlist_insert_tail(&trigger->entry, &cntr->trigger_list);
fastlock_release(&cntr->trigger_lock);
+ sock_cntr_check_trigger_list(cntr);
return 0;
}
diff --git a/prov/sockets/src/sock_wait.c b/prov/sockets/src/sock_wait.c
index 03ddbf9..d340ced 100644
--- a/prov/sockets/src/sock_wait.c
+++ b/prov/sockets/src/sock_wait.c
@@ -35,6 +35,7 @@
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
+#include <pthread.h>
#include "sock.h"
#include "sock_util.h"
@@ -49,7 +50,9 @@ enum {
int sock_wait_get_obj(struct fid_wait *fid, void *arg)
{
+#ifndef _WIN32 /* there is no support of wait objects on windows */
struct fi_mutex_cond mut_cond;
+#endif /* _WIN32 */
struct sock_wait *wait;
wait = container_of(fid, struct sock_wait, wait_fid.fid);
@@ -57,16 +60,17 @@ int sock_wait_get_obj(struct fid_wait *fid, void *arg)
return -FI_ENOSYS;
switch (wait->type) {
+#ifndef _WIN32
case FI_WAIT_FD:
memcpy(arg, &wait->wobj.fd[WAIT_READ_FD], sizeof(int));
break;
case FI_WAIT_MUTEX_COND:
mut_cond.mutex = &wait->wobj.mutex_cond.mutex;
- mut_cond.cond = &wait->wobj.mutex_cond.cond;
+ mut_cond.cond = &wait->wobj.mutex_cond.cond;
memcpy(arg, &mut_cond, sizeof(mut_cond));
break;
-
+#endif /* _WIN32 */
default:
SOCK_LOG_ERROR("Invalid wait obj type\n");
return -FI_EINVAL;
@@ -88,8 +92,8 @@ static int sock_wait_init(struct sock_wait *wait, enum fi_wait_obj type)
ret = fd_set_nonblock(wait->wobj.fd[WAIT_READ_FD]);
if (ret) {
- close(wait->wobj.fd[WAIT_READ_FD]);
- close(wait->wobj.fd[WAIT_WRITE_FD]);
+ ofi_close_socket(wait->wobj.fd[WAIT_READ_FD]);
+ ofi_close_socket(wait->wobj.fd[WAIT_WRITE_FD]);
return ret;
}
break;
@@ -159,7 +163,7 @@ static int sock_wait_wait(struct fid_wait *wait_fid, int timeout)
err = -FI_ETIMEDOUT;
} else {
while (err > 0) {
- ret = read(wait->wobj.fd[WAIT_READ_FD], &c, 1);
+ ret = ofi_read_socket(wait->wobj.fd[WAIT_READ_FD], &c, 1);
if (ret != 1) {
SOCK_LOG_ERROR("failed to read wait_fd\n");
err = 0;
@@ -192,7 +196,7 @@ void sock_wait_signal(struct fid_wait *wait_fid)
switch (wait->type) {
case FI_WAIT_FD:
- ret = write(wait->wobj.fd[WAIT_WRITE_FD], &c, 1);
+ ret = ofi_write_socket(wait->wobj.fd[WAIT_WRITE_FD], &c, 1);
if (ret != 1)
SOCK_LOG_ERROR("failed to signal\n");
break;
@@ -244,8 +248,8 @@ int sock_wait_close(fid_t fid)
}
if (wait->type == FI_WAIT_FD) {
- close(wait->wobj.fd[WAIT_READ_FD]);
- close(wait->wobj.fd[WAIT_WRITE_FD]);
+ ofi_close_socket(wait->wobj.fd[WAIT_READ_FD]);
+ ofi_close_socket(wait->wobj.fd[WAIT_WRITE_FD]);
}
atomic_dec(&wait->fab->ref);
diff --git a/prov/udp/src/udpx.h b/prov/udp/src/udpx.h
index ecd912b..968a931 100644
--- a/prov/udp/src/udpx.h
+++ b/prov/udp/src/udpx.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -47,7 +48,6 @@
#include <rdma/fi_endpoint.h>
#include <rdma/fi_eq.h>
#include <rdma/fi_errno.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_tagged.h>
#include <rdma/fi_trigger.h>
@@ -69,6 +69,7 @@
extern struct fi_provider udpx_prov;
+extern struct util_prov udpx_util_prov;
extern struct fi_info udpx_info;
@@ -105,6 +106,7 @@ struct udpx_ep {
udpx_tx_comp_func tx_comp;
struct udpx_rx_cirq *rxq; /* protected by rx_cq lock */
int sock;
+ int is_bound;
};
int udpx_endpoint(struct fid_domain *domain, struct fi_info *info,
diff --git a/prov/udp/src/udpx_attr.c b/prov/udp/src/udpx_attr.c
index b320423..4927dfe 100644
--- a/prov/udp/src/udpx_attr.c
+++ b/prov/udp/src/udpx_attr.c
@@ -66,10 +66,10 @@ struct fi_domain_attr udpx_domain_attr = {
.resource_mgmt = FI_RM_ENABLED,
.av_type = FI_AV_UNSPEC,
.mr_mode = FI_MR_SCALABLE,
- .cq_cnt = (1 << 16),
- .ep_cnt = (1 << 15),
- .tx_ctx_cnt = (1 << 15),
- .rx_ctx_cnt = (1 << 15),
+ .cq_cnt = 256,
+ .ep_cnt = 256,
+ .tx_ctx_cnt = 256,
+ .rx_ctx_cnt = 256,
.max_ep_tx_ctx = 1,
.max_ep_rx_ctx = 1
};
@@ -88,3 +88,9 @@ struct fi_info udpx_info = {
.domain_attr = &udpx_domain_attr,
.fabric_attr = &udpx_fabric_attr
};
+
+struct util_prov udpx_util_prov = {
+ .prov = &udpx_prov,
+ .info = &udpx_info,
+ .flags = 0,
+};
diff --git a/prov/udp/src/udpx_cq.c b/prov/udp/src/udpx_cq.c
index a5d7842..af7f442 100644
--- a/prov/udp/src/udpx_cq.c
+++ b/prov/udp/src/udpx_cq.c
@@ -35,9 +35,45 @@
#include "udpx.h"
+static int udpx_cq_close(struct fid *fid)
+{
+ int ret;
+ struct util_cq *cq;
+
+ cq = container_of(fid, struct util_cq, cq_fid.fid);
+ ret = ofi_cq_cleanup(cq);
+ if (ret)
+ return ret;
+ free(cq);
+ return 0;
+}
+
+static struct fi_ops udpx_cq_fi_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = udpx_cq_close,
+ .bind = fi_no_bind,
+ .control = fi_no_control,
+ .ops_open = fi_no_ops_open,
+};
int udpx_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
struct fid_cq **cq_fid, void *context)
{
- return util_cq_open(&udpx_prov, domain, attr, cq_fid, context);
+ int ret;
+ struct util_cq *cq;
+
+ cq = calloc(1, sizeof(*cq));
+ if (!cq)
+ return -FI_ENOMEM;
+
+ ret = ofi_cq_init(&udpx_prov, domain, attr, cq,
+ &ofi_cq_progress, context);
+ if (ret) {
+ free(cq);
+ return ret;
+ }
+
+ *cq_fid = &cq->cq_fid;
+ (*cq_fid)->fid.ops = &udpx_cq_fi_ops;
+ return 0;
}
diff --git a/prov/udp/src/udpx_domain.c b/prov/udp/src/udpx_domain.c
index 8694a87..69f0cc8 100644
--- a/prov/udp/src/udpx_domain.c
+++ b/prov/udp/src/udpx_domain.c
@@ -53,7 +53,7 @@ static int udpx_domain_close(fid_t fid)
int ret;
struct util_domain *domain;
domain = container_of(fid, struct util_domain, domain_fid.fid);
- ret = util_domain_close(domain);
+ ret = ofi_domain_close(domain);
if (ret)
return ret;
free(domain);
@@ -82,7 +82,7 @@ int udpx_domain_open(struct fid_fabric *fabric, struct fi_info *info,
if (!util_domain)
return -FI_ENOMEM;
- ret = fi_domain_init(fabric, info, util_domain, context);
+ ret = ofi_domain_init(fabric, info, util_domain, context);
if (ret)
return ret;
diff --git a/prov/udp/src/udpx_ep.c b/prov/udp/src/udpx_ep.c
index 8c1ca88..7bb9464 100644
--- a/prov/udp/src/udpx_ep.c
+++ b/prov/udp/src/udpx_ep.c
@@ -43,7 +43,10 @@ int udpx_setname(fid_t fid, void *addr, size_t addrlen)
ep = container_of(fid, struct udpx_ep, util_ep.ep_fid.fid);
ret = bind(ep->sock, addr, addrlen);
- return ret ? -errno : 0;
+ if (ret)
+ return -errno;
+ ep->is_bound = 1;
+ return 0;
}
int udpx_getname(fid_t fid, void *addr, size_t *addrlen)
@@ -96,7 +99,7 @@ static struct fi_ops_ep udpx_ep_ops = {
static void udpx_tx_comp(struct udpx_ep *ep, void *context)
{
- struct fi_cq_data_entry *comp;
+ struct fi_cq_tagged_entry *comp;
comp = cirque_tail(ep->util_ep.tx_cq->cirq);
comp->op_context = context;
@@ -116,7 +119,7 @@ static void udpx_tx_comp_signal(struct udpx_ep *ep, void *context)
static void udpx_rx_comp(struct udpx_ep *ep, void *context, uint64_t flags,
size_t len, void *buf, void *addr)
{
- struct fi_cq_data_entry *comp;
+ struct fi_cq_tagged_entry *comp;
comp = cirque_tail(ep->util_ep.rx_cq->cirq);
comp->op_context = context;
@@ -439,7 +442,11 @@ static int udpx_ep_bind(struct fid *ep_fid, struct fid *bfid, uint64_t flags)
{
struct udpx_ep *ep;
struct util_av *av;
- int ret = 0;
+ int ret;
+
+ ret = ofi_ep_bind_valid(&udpx_prov, bfid, flags);
+ if (ret)
+ return ret;
ep = container_of(ep_fid, struct udpx_ep, util_ep.ep_fid.fid);
switch (bfid->fclass) {
@@ -468,6 +475,31 @@ static int udpx_ep_bind(struct fid *ep_fid, struct fid *bfid, uint64_t flags)
return ret;
}
+static void udpx_bind_src_addr(struct udpx_ep *ep)
+{
+ int ret;
+ struct addrinfo ai, *rai = NULL;
+ char hostname[HOST_NAME_MAX];
+
+ memset(&ai, 0, sizeof(ai));
+ ai.ai_family = AF_INET;
+ ai.ai_socktype = SOCK_DGRAM;
+
+ ofi_getnodename(hostname, sizeof(hostname));
+ ret = getaddrinfo(hostname, NULL, &ai, &rai);
+ if (ret) {
+ FI_WARN(&udpx_prov, FI_LOG_EP_CTRL,
+ "getaddrinfo failed\n");
+ return;
+ }
+
+ ret = udpx_setname(&ep->util_ep.ep_fid.fid, rai->ai_addr, rai->ai_addrlen);
+ if (ret) {
+ FI_WARN(&udpx_prov, FI_LOG_EP_CTRL, "failed to set addr\n");
+ }
+ freeaddrinfo(rai);
+}
+
static int udpx_ep_ctrl(struct fid *fid, int command, void *arg)
{
struct udpx_ep *ep;
@@ -478,7 +510,10 @@ static int udpx_ep_ctrl(struct fid *fid, int command, void *arg)
if (!ep->util_ep.rx_cq || !ep->util_ep.tx_cq)
return -FI_ENOCQ;
if (!ep->util_ep.av)
- return -FI_EOPBADSTATE; /* TODO: Add FI_ENOAV */
+ return -FI_ENOAV;
+
+ if (!ep->is_bound)
+ udpx_bind_src_addr(ep);
break;
default:
return -FI_ENOSYS;
diff --git a/prov/udp/src/udpx_fabric.c b/prov/udp/src/udpx_fabric.c
index 7da684b..908510c 100644
--- a/prov/udp/src/udpx_fabric.c
+++ b/prov/udp/src/udpx_fabric.c
@@ -40,9 +40,9 @@ static struct fi_ops_fabric udpx_fabric_ops = {
.size = sizeof(struct fi_ops_fabric),
.domain = udpx_domain_open,
.passive_ep = fi_no_passive_ep,
- .eq_open = fi_eq_create,
- .wait_open = fi_wait_fd_open,
- .trywait = util_trywait
+ .eq_open = ofi_eq_create,
+ .wait_open = ofi_wait_fd_open,
+ .trywait = ofi_trywait
};
static int udpx_fabric_close(fid_t fid)
@@ -50,7 +50,7 @@ static int udpx_fabric_close(fid_t fid)
int ret;
struct util_fabric *fabric;
fabric = container_of(fid, struct util_fabric, fabric_fid.fid);
- ret = util_fabric_close(fabric);
+ ret = ofi_fabric_close(fabric);
if (ret)
return ret;
free(fabric);
@@ -75,8 +75,8 @@ int udpx_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric,
if (!util_fabric)
return -FI_ENOMEM;
- ret = fi_fabric_init(&udpx_prov, udpx_info.fabric_attr, attr,
- util_fabric, context);
+ ret = ofi_fabric_init(&udpx_prov, udpx_info.fabric_attr, attr,
+ util_fabric, context, FI_MATCH_EXACT);
if (ret)
return ret;
diff --git a/prov/udp/src/udpx_init.c b/prov/udp/src/udpx_init.c
index 8ee0e46..2c1b0b1 100644
--- a/prov/udp/src/udpx_init.c
+++ b/prov/udp/src/udpx_init.c
@@ -35,17 +35,82 @@
#include <prov.h>
#include "udpx.h"
+#include <sys/types.h>
+#include <ifaddrs.h>
+#include <net/if.h>
+
+
+#if HAVE_GETIFADDRS
+static void udpx_getinfo_ifs(struct fi_info **info)
+{
+ struct ifaddrs *ifaddrs, *ifa;
+ struct fi_info *head, *tail, *cur;
+ size_t addrlen;
+ int ret;
+
+ ret = getifaddrs(&ifaddrs);
+ if (ret)
+ return;
+
+ head = tail = NULL;
+ for (ifa = ifaddrs; ifa != NULL; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr == NULL || !(ifa->ifa_flags & IFF_UP))
+ continue;
+
+ switch (ifa->ifa_addr->sa_family) {
+ case AF_INET:
+ addrlen = sizeof(struct sockaddr_in);
+ break;
+ case AF_INET6:
+ addrlen = sizeof(struct sockaddr_in6);
+ break;
+ default:
+ continue;
+ }
+
+ cur = fi_dupinfo(*info);
+ if (!cur)
+ break;
+
+ if (!head)
+ head = cur;
+ else
+ tail->next = cur;
+ tail = cur;
+
+ if ((cur->src_addr = mem_dup(ifa->ifa_addr, addrlen)))
+ cur->src_addrlen = addrlen;
+ }
+ freeifaddrs(ifaddrs);
+
+ if (head) {
+ fi_freeinfo(*info);
+ *info = head;
+ }
+}
+#else
+#define udpx_getinfo_ifs(info) do{}while(0)
+#endif
int udpx_check_info(struct fi_info *info)
{
- return fi_check_info(&udpx_prov, &udpx_info, info);
+ return fi_check_info(&udpx_util_prov, info, FI_MATCH_EXACT);
}
static int udpx_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, struct fi_info *hints, struct fi_info **info)
{
- return util_getinfo(&udpx_prov, version, node, service, flags,
- &udpx_info, hints, info);
+ int ret;
+
+ ret = util_getinfo(&udpx_util_prov, version, node, service, flags,
+ hints, info);
+ if (ret)
+ return ret;
+
+ if (!(*info)->src_addr && !(*info)->dest_addr)
+ udpx_getinfo_ifs(info);
+
+ return 0;
}
static void udpx_fini(void)
diff --git a/prov/usnic/src/fi_ext_usnic.h b/prov/usnic/src/fi_ext_usnic.h
index 44c0a20..db2cb6a 100644
--- a/prov/usnic/src/fi_ext_usnic.h
+++ b/prov/usnic/src/fi_ext_usnic.h
@@ -51,11 +51,13 @@
/*
* usNIC specific info
*/
+/* Packed in 1.4, maintains the same alignment as <= 1.3.0 */
struct fi_usnic_cap {
const char *uc_capability;
int uc_present;
-};
+} __attribute__((packed));
+/* Packed in 1.4, maintains the same alignment as <= 1.3.0 */
struct fi_usnic_info_v1 {
uint32_t ui_link_speed;
uint32_t ui_netmask_be;
@@ -64,7 +66,7 @@ struct fi_usnic_info_v1 {
uint32_t ui_num_vf;
uint32_t ui_qp_per_vf;
uint32_t ui_cq_per_vf;
-};
+} __attribute__((packed));
struct fi_usnic_info_v2 {
/* Put all of the v1 fields at the start to provide some backward
@@ -80,11 +82,17 @@ struct fi_usnic_info_v2 {
char ui_devname[FI_EXT_USNIC_MAX_DEVNAME];
uint8_t ui_mac_addr[6];
+ /* Explicit padding to match 1.3 alignment */
+ uint8_t ui_pad0[2];
+
uint32_t ui_ipaddr_be;
uint32_t ui_prefixlen;
uint32_t ui_mtu;
uint8_t ui_link_up;
+ /* Explicit padding to match 1.3 alignment */
+ uint8_t ui_pad1[3];
+
uint32_t ui_vendor_id;
uint32_t ui_vendor_part_id;
uint32_t ui_device_id;
@@ -102,15 +110,25 @@ struct fi_usnic_info_v2 {
const char *ui_pid;
struct fi_usnic_cap **ui_caps;
-};
-
+} __attribute__((packed));
+
+/* In API version 1.2 and below, the v1 structure did not contain any 64-bit
+ * data types and therefore had a 4-byte alignment. Once v2 of the extension API
+ * was introduced in version 1.3, the extra pointers mandated an 8-byte
+ * alignment thus changing the offset of the v1 structure. This means that the
+ * alignment difference manifests when an application using v1 of the extension
+ * is compiled with Libfabric v1.1.x or v1.2.x, but then runs with libfabric.so
+ * that is v1.3.x or higher (and vice versa). Make the alignment explicit and
+ * consistent by adding an extra 32-bit padding (4 uint8_t).
+ */
struct fi_usnic_info {
uint32_t ui_version;
+ uint8_t ui_pad0[4];
union {
struct fi_usnic_info_v1 v1;
struct fi_usnic_info_v2 v2;
} ui;
-};
+} __attribute__((packed));
/*
* usNIC-specific fabric ops
diff --git a/prov/usnic/src/usdf.h b/prov/usnic/src/usdf.h
index d1d2008..374c1fa 100644
--- a/prov/usnic/src/usdf.h
+++ b/prov/usnic/src/usdf.h
@@ -39,7 +39,7 @@
#include <sys/queue.h>
#include <pthread.h>
-#include <rdma/fi_log.h>
+#include <rdma/providers/fi_log.h>
#include "usdf_progress.h"
#include "usd.h"
@@ -263,6 +263,10 @@ struct usdf_rx {
#define rx_utof(RX) (&(RX)->rx_fid)
#define rx_utofid(RX) (&(RX)->rx_fid.fid)
+enum {
+ USDF_EP_ENABLED = (1 << 0)
+};
+
struct usdf_ep {
struct fid_ep ep_fid;
struct usdf_domain *ep_domain;
@@ -276,6 +280,8 @@ struct usdf_ep {
uint8_t ep_tx_completion;
uint8_t ep_rx_completion;
+ uint32_t flags;
+
uint32_t ep_wqe; /* requested queue sizes */
uint32_t ep_rqe;
@@ -356,7 +362,7 @@ struct usdf_cq_hard {
atomic_t cqh_refcnt;
void (*cqh_progress)(struct usdf_cq_hard *hcq);
void (*cqh_post)(struct usdf_cq_hard *hcq, void *context, size_t len,
- int prov_errno);
+ int prov_errno, uint64_t flags);
TAILQ_ENTRY(usdf_cq_hard) cqh_link;
TAILQ_ENTRY(usdf_cq_hard) cqh_dom_link;
};
@@ -410,6 +416,12 @@ enum {
#define cq_fidtou(FID) container_of(FID, struct usdf_cq, cq_fid.fid)
#define cq_utof(CQ) (&(CQ)->cq_fid)
+struct usdf_err_data_entry {
+ struct slist_entry entry;
+ uint8_t seen;
+ uint8_t err_data[0];
+};
+
struct usdf_event {
uint32_t ue_event;
void *ue_buf;
@@ -438,6 +450,7 @@ struct usdf_eq {
int eq_fd;
};
+ struct slist eq_err_data;
struct fi_ops_eq eq_ops_data;
};
#define eq_ftou(FEQ) container_of(FEQ, struct usdf_eq, eq_fid)
@@ -467,6 +480,11 @@ int usdf_endpoint_open(struct fid_domain *domain, struct fi_info *info,
int usdf_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
struct fid_av **av_o, void *context);
+/* Domain name functionality */
+int usdf_domain_getname(uint32_t version, struct usd_device_attrs *dap,
+ char **name);
+bool usdf_domain_checkname(uint32_t version, struct usd_device_attrs *dap,
+ char *hint);
/* fi_ops_mr */
int usdf_reg_mr(struct fid *fid, const void *buf, size_t len,
diff --git a/prov/usnic/src/usdf_av.c b/prov/usnic/src/usdf_av.c
index f66f004..0231e43 100644
--- a/prov/usnic/src/usdf_av.c
+++ b/prov/usnic/src/usdf_av.c
@@ -64,14 +64,101 @@
#include "usdf.h"
#include "usdf_av.h"
#include "usdf_timer.h"
+#include "usdf_rdm.h"
#include "fi_ext_usnic.h"
+static int usdf_av_alloc_dest(struct usdf_dest **dest_o)
+{
+ struct usdf_dest *dest;
+
+ dest = calloc(1, sizeof(**dest_o));
+ if (dest == NULL)
+ return -errno;
+
+ SLIST_INIT(&dest->ds_rdm_rdc_list);
+
+ *dest_o = dest;
+ return 0;
+}
+
+static void usdf_av_free_dest(struct usdf_dest *dest)
+{
+ struct usdf_rdm_connection *rdc = NULL;
+
+ LIST_REMOVE(dest, ds_addresses_entry);
+
+ while (!SLIST_EMPTY(&dest->ds_rdm_rdc_list)) {
+ rdc = SLIST_FIRST(&dest->ds_rdm_rdc_list);
+ rdc->dc_dest = NULL;
+
+ SLIST_REMOVE(&dest->ds_rdm_rdc_list, rdc, usdf_rdm_connection,
+ dc_addr_link);
+ if (rdc)
+ rdc->dc_dest = NULL;
+ }
+
+ free(dest);
+}
+
+static int usdf_av_close_(struct usdf_av *av)
+{
+ struct usdf_dest *entry;
+
+ USDF_TRACE_SYS(AV, "\n");
+
+ pthread_spin_lock(&av->av_lock);
+
+ if (av->av_eq)
+ atomic_dec(&av->av_eq->eq_refcnt);
+
+ atomic_dec(&av->av_domain->dom_refcnt);
+
+ while (!LIST_EMPTY(&av->av_addresses)) {
+ entry = LIST_FIRST(&av->av_addresses);
+ usdf_av_free_dest(entry);
+ }
+
+ pthread_spin_destroy(&av->av_lock);
+ free(av);
+
+ USDF_DBG_SYS(AV, "AV successfully destroyed\n");
+
+ return 0;
+}
+
+static int usdf_av_close(struct fid *fid)
+{
+ struct usdf_av *av;
+ int pending;
+
+ USDF_TRACE_SYS(AV, "\n");
+
+ av = container_of(fid, struct usdf_av, av_fid.fid);
+ if (atomic_get(&av->av_refcnt) > 0)
+ return -FI_EBUSY;
+
+ pending = atomic_get(&av->av_active_inserts);
+ assert(pending >= 0);
+
+ if (pending) {
+ USDF_DBG_SYS(AV, "%d pending inserts, defer closing\n",
+ pending);
+ atomic_set(&av->av_closing, 1);
+ } else {
+ usdf_av_close_(av);
+ }
+
+ return 0;
+}
+
static void
usdf_av_insert_async_complete(struct usdf_av_insert *insert)
{
struct fi_eq_entry entry;
struct usdf_av *av;
+ int pending;
+ int closing;
av = insert->avi_av;
@@ -81,17 +168,16 @@ usdf_av_insert_async_complete(struct usdf_av_insert *insert)
usdf_eq_write_internal(av->av_eq,
FI_AV_COMPLETE, &entry, sizeof(entry), 0);
- pthread_spin_lock(&av->av_lock);
-
usdf_timer_free(av->av_domain->dom_fabric, insert->avi_timer);
- atomic_dec(&av->av_active_inserts);
- if (atomic_get(&av->av_active_inserts) == 0 && av->av_closing) {
- pthread_spin_destroy(&av->av_lock);
- free(av);
- } else {
- pthread_spin_unlock(&av->av_lock);
- }
+ pending = atomic_dec(&av->av_active_inserts);
+ USDF_DBG_SYS(AV, "new active insert value: %d\n", pending);
+ assert(pending >= 0);
+
+ closing = atomic_get(&av->av_closing);
+
+ if (!pending && closing)
+ usdf_av_close_(av);
free(insert);
}
@@ -123,22 +209,6 @@ usdf_post_insert_request_error(struct usdf_av_insert *insert,
USDF_EVENT_FLAG_ERROR);
}
-static int
-usdf_av_alloc_dest(struct usdf_dest **dest_o)
-{
- struct usdf_dest *dest;
-
- dest = calloc(1, sizeof(**dest_o));
- if (dest == NULL) {
- return -errno;
- }
- SLIST_INIT(&dest->ds_rdm_rdc_list);
-
- *dest_o = dest;
- return 0;
-}
-
-
/*
* Called by progression thread to look for AV completions on this domain
*/
@@ -174,6 +244,9 @@ usdf_av_insert_progress(void *v)
if (ret == 0) {
++insert->avi_successes;
*(struct usdf_dest **)req->avr_fi_addr = dest;
+
+ LIST_INSERT_HEAD(&insert->avi_av->av_addresses,
+ dest, ds_addresses_entry);
} else {
usdf_post_insert_request_error(insert, req);
}
@@ -246,9 +319,6 @@ usdf_am_insert_async(struct fid_av *fav, const void *addr, size_t count,
fp = av->av_domain->dom_fabric;
dap = fp->fab_dev_attrs;
- if (av->av_flags & FI_READ) {
- return -FI_EACCES;
- }
if (av->av_eq == NULL) {
return -FI_ENOEQ;
}
@@ -270,14 +340,15 @@ usdf_am_insert_async(struct fid_av *fav, const void *addr, size_t count,
TAILQ_INIT(&insert->avi_req_list);
insert->avi_arps_left = USDF_AV_MAX_ARPS;
+ ret = atomic_inc(&av->av_active_inserts);
+ USDF_DBG_SYS(AV, "new active insert value: %d\n", ret);
+
/* If no addresses, complete now */
if (count == 0) {
usdf_av_insert_async_complete(insert);
return 0;
}
- atomic_inc(&av->av_active_inserts);
-
req = (struct usdf_av_req *)(insert + 1);
for (i = 0; i < count; i++) {
@@ -373,6 +444,8 @@ usdf_am_insert_sync(struct fid_av *fav, const void *addr, size_t count,
htons(IP_DF);
dest->ds_dest = *u_dest;
fi_addr[i] = (fi_addr_t)dest;
+ LIST_INSERT_HEAD(&av->av_addresses, dest,
+ ds_addresses_entry);
++ret_count;
} else {
fi_addr[i] = FI_ADDR_NOTAVAIL;
@@ -447,21 +520,17 @@ usdf_am_remove(struct fid_av *fav, fi_addr_t *fi_addr, size_t count,
uint64_t flags)
{
struct usdf_dest *dest;
- struct usdf_av *av;
size_t i;
USDF_TRACE_SYS(AV, "\n");
- av = av_ftou(fav);
-
- if (av->av_flags & FI_READ) {
- return -FI_EACCES;
- }
-
for (i = 0; i < count; ++i) {
if (fi_addr[i] != FI_ADDR_NOTAVAIL) {
dest = (struct usdf_dest *)(uintptr_t)fi_addr[i];
- free(dest);
+ usdf_av_free_dest(dest);
+
+ /* Mark invalid by setting to FI_ADDR_NOTAVAIL*/
+ fi_addr[i] = FI_ADDR_NOTAVAIL;
}
}
@@ -478,6 +547,11 @@ usdf_am_lookup(struct fid_av *av, fi_addr_t fi_addr, void *addr,
USDF_TRACE_SYS(AV, "\n");
+ if (fi_addr == FI_ADDR_NOTAVAIL) {
+ USDF_WARN_SYS(AV, "invalid address, can't lookup\n");
+ return -FI_EINVAL;
+ }
+
dest = (struct usdf_dest *)(uintptr_t)fi_addr;
if (*addrlen < sizeof(sin)) {
@@ -504,7 +578,7 @@ usdf_av_straddr(struct fid_av *av, const void *addr,
sin = addr;
size = snprintf(straddr, sizeof straddr, "%s:%d",
- inet_ntoa(sin->sin_addr), sin->sin_port);
+ inet_ntoa(sin->sin_addr), ntohs(sin->sin_port));
snprintf(buf, *len, "%s", straddr);
*len = size + 1;
return buf;
@@ -534,35 +608,6 @@ usdf_av_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
return 0;
}
-static int
-usdf_av_close(struct fid *fid)
-{
- struct usdf_av *av;
-
- USDF_TRACE_SYS(AV, "\n");
-
- av = container_of(fid, struct usdf_av, av_fid.fid);
- if (atomic_get(&av->av_refcnt) > 0) {
- return -FI_EBUSY;
- }
-
- pthread_spin_lock(&av->av_lock);
-
- if (av->av_eq != NULL) {
- atomic_dec(&av->av_eq->eq_refcnt);
- }
- atomic_dec(&av->av_domain->dom_refcnt);
-
- if (atomic_get(&av->av_active_inserts) > 0) {
- av->av_closing = 1;
- pthread_spin_unlock(&av->av_lock);
- } else {
- pthread_spin_destroy(&av->av_lock);
- free(av);
- }
- return 0;
-}
-
static struct fi_ops usdf_av_fi_ops = {
.size = sizeof(struct fi_ops),
.close = usdf_av_close,
@@ -591,24 +636,68 @@ static struct fi_ops_av usdf_am_ops_sync = {
.straddr = usdf_av_straddr
};
+static int usdf_av_process_attr(struct fi_av_attr *attr)
+{
+ USDF_TRACE_SYS(AV, "\n");
+
+ if (attr == NULL) {
+ USDF_WARN_SYS(AV, "NULL AV attribute structure is invalid\n");
+ return -FI_EINVAL;
+ }
+
+ if (attr->name || attr->map_addr || (attr->flags & FI_READ)) {
+ USDF_WARN_SYS(AV, "named AVs are not supported\n");
+ return -FI_ENOSYS;
+ }
+
+ if (attr->flags & ~FI_EVENT) {
+ USDF_WARN_SYS(AV, "invalid flag, only FI_EVENT is supported\n");
+ return -FI_EINVAL;
+ }
+
+ if (attr->rx_ctx_bits) {
+ USDF_WARN_SYS(AV, "scalable endpoints not supported\n");
+ return -FI_EINVAL;
+ }
+
+ if (attr->ep_per_node > 1)
+ USDF_WARN_SYS(AV, "ep_per_node not supported, ignoring\n");
+
+ switch (attr->type) {
+ case FI_AV_UNSPEC:
+ USDF_DBG_SYS(AV, "no AV type specified, using FI_AV_MAP\n");
+ case FI_AV_MAP:
+ break;
+ case FI_AV_TABLE:
+ USDF_DBG_SYS(AV, "FI_AV_TABLE is unsupported\n");
+ return -FI_ENOSYS;
+ default:
+ USDF_WARN_SYS(AV, "unknown AV type %d, not supported",
+ attr->type);
+ return -FI_EINVAL;
+ }
+
+ return FI_SUCCESS;
+}
+
int
usdf_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
struct fid_av **av_o, void *context)
{
struct usdf_domain *udp;
struct usdf_av *av;
+ int ret;
USDF_TRACE_SYS(AV, "\n");
- if (attr == NULL || av_o == NULL)
+ if (!av_o) {
+ USDF_WARN_SYS(AV, "provided AV pointer can not be NULL\n");
return -FI_EINVAL;
-
- if ((attr->flags & ~(FI_EVENT | FI_READ)) != 0) {
- return -FI_ENOSYS;
}
- if (attr->name)
- return -FI_ENOSYS;
+ ret = usdf_av_process_attr(attr);
+ if (ret)
+ return ret;
udp = dom_ftou(domain);
@@ -622,6 +711,9 @@ usdf_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
} else {
av->av_fid.ops = &usdf_am_ops_sync;
}
+
+ LIST_INIT(&av->av_addresses);
+
av->av_fid.fid.fclass = FI_CLASS_AV;
av->av_fid.fid.context = context;
av->av_fid.fid.ops = &usdf_av_fi_ops;
@@ -629,6 +721,7 @@ usdf_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
pthread_spin_init(&av->av_lock, PTHREAD_PROCESS_PRIVATE);
atomic_initialize(&av->av_active_inserts, 0);
+ atomic_initialize(&av->av_closing, 0);
atomic_initialize(&av->av_refcnt, 0);
atomic_inc(&udp->dom_refcnt);
diff --git a/prov/usnic/src/usdf_av.h b/prov/usnic/src/usdf_av.h
index 3a46ccd..73c4adb 100644
--- a/prov/usnic/src/usdf_av.h
+++ b/prov/usnic/src/usdf_av.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2014-2016, Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -50,6 +50,7 @@ struct usdf_dest {
struct usd_dest ds_dest;
SLIST_HEAD(,usdf_rdm_connection) ds_rdm_rdc_list;
+ LIST_ENTRY(usdf_dest) ds_addresses_entry;
};
/* struct used to track async insert requests */
@@ -81,9 +82,10 @@ struct usdf_av {
uint64_t av_flags;
struct usdf_eq *av_eq;
atomic_t av_refcnt;
- int av_closing;
+ atomic_t av_closing;
atomic_t av_active_inserts;
pthread_spinlock_t av_lock;
+ LIST_HEAD(, usdf_dest) av_addresses;
};
#define av_ftou(FAV) container_of(FAV, struct usdf_av, av_fid)
#define av_fidtou(FID) container_of(FID, struct usdf_av, av_fid.fid)
diff --git a/prov/usnic/src/usdf_cm.c b/prov/usnic/src/usdf_cm.c
index e8f97f4..d639811 100644
--- a/prov/usnic/src/usdf_cm.c
+++ b/prov/usnic/src/usdf_cm.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2014-2016, Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -50,11 +50,11 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
#include "fi.h"
+#include "fi_file.h"
#include "usnic_direct.h"
#include "usdf.h"
@@ -64,7 +64,7 @@
#include "usdf_av.h"
#include "usdf_cm.h"
-static void
+void
usdf_cm_msg_connreq_cleanup(struct usdf_connreq *crp)
{
struct usdf_ep *ep;
@@ -114,7 +114,7 @@ usdf_cm_msg_accept_complete(struct usdf_connreq *crp)
ret = usdf_eq_write_internal(ep->ep_eq, FI_CONNECTED, &entry,
sizeof(entry), 0);
if (ret != sizeof(entry)) {
- usdf_cm_msg_connreq_failed(crp, ret);
+ usdf_cm_report_failure(crp, ret, false);
return 0;
}
@@ -157,9 +157,8 @@ usdf_cm_msg_accept(struct fid_ep *fep, const void *param, size_t paramlen)
ep->e.msg.ep_lcl_peer_id = ntohs(reqp->creq_peer_id);
/* start creating the dest early */
- ret = usd_create_dest_with_mac(udp->dom_dev, reqp->creq_ipaddr,
- reqp->creq_port, reqp->creq_mac,
- &ep->e.msg.ep_dest);
+ ret = usd_create_dest(udp->dom_dev, reqp->creq_ipaddr,
+ reqp->creq_port, &ep->e.msg.ep_dest);
if (ret != 0) {
goto fail;
}
@@ -185,7 +184,6 @@ usdf_cm_msg_accept(struct fid_ep *fep, const void *param, size_t paramlen)
reqp->creq_ipaddr = fp->fab_dev_attrs->uda_ipaddr_be;
reqp->creq_port =
qp->uq_attrs.uqa_local_addr.ul_addr.ul_udp.u_addr.sin_port;
- memcpy(reqp->creq_mac, fp->fab_dev_attrs->uda_mac_addr, ETH_ALEN);
reqp->creq_result = htonl(0);
reqp->creq_datalen = htonl(paramlen);
memcpy(reqp->creq_data, param, paramlen);
@@ -211,22 +209,82 @@ fail:
return ret;
}
-/*
- * Connection request attempt failed
+/* Given a connection request structure containing data, make a copy of the data
+ * that can be accessed in error entries on the EQ. The return value is the size
+ * of the data stored in the error entry. If the return value is a non-negative
+ * value, then the function has suceeded and the size and output data can be
+ * assumed to be valid. If the function fails, then the data will be NULL and
+ * the size will be a negative error value.
*/
-void
-usdf_cm_msg_connreq_failed(struct usdf_connreq *crp, int error)
+static int usdf_cm_generate_err_data(struct usdf_eq *eq,
+ struct usdf_connreq *crp, void **data)
+{
+ struct usdf_err_data_entry *err_data_entry;
+ struct usdf_connreq_msg *reqp;
+ size_t entry_size;
+ size_t data_size;
+
+ if (!eq || !crp || !data) {
+ USDF_DBG_SYS(EP_CTRL,
+ "eq, crp, or data is NULL.\n");
+ return -FI_EINVAL;
+ }
+
+ /* Initialize to NULL so data can't be used in the error case. */
+ *data = NULL;
+
+ reqp = (struct usdf_connreq_msg *) crp->cr_data;
+
+ /* This is a normal case, maybe there was no data. */
+ if (!reqp || !reqp->creq_datalen)
+ return 0;
+
+ data_size = reqp->creq_datalen;
+
+ entry_size = sizeof(*err_data_entry) + data_size;
+
+ err_data_entry = calloc(1, entry_size);
+ if (!err_data_entry) {
+ USDF_WARN_SYS(EP_CTRL,
+ "failed to allocate err data entry\n");
+ return -FI_ENOMEM;
+ }
+
+ /* This data should be copied and owned by the provider. Keep
+ * track of it in the EQ, this will be freed in the next EQ read
+ * call after it has been read.
+ */
+ memcpy(err_data_entry->err_data, reqp->creq_data, data_size);
+ slist_insert_tail(&err_data_entry->entry, &eq->eq_err_data);
+
+ *data = err_data_entry->err_data;
+
+ return data_size;
+}
+
+/* Report a connection management related failure. Sometimes there is connection
+ * event data that should be copied into the generated event. If the copy_data
+ * parameter evaluates to true, then the data will be copied.
+ *
+ * If data is to be generated for the error entry, then the connection request
+ * is assumed to have the data size in host order. If something fails during
+ * processing of the error data, then the EQ entry will still be generated
+ * without the error data.
+ */
+void usdf_cm_report_failure(struct usdf_connreq *crp, int error, bool copy_data)
{
+ struct fi_eq_err_entry err = {0};
struct usdf_pep *pep;
struct usdf_ep *ep;
struct usdf_eq *eq;
fid_t fid;
- struct fi_eq_err_entry err;
+ int ret;
- USDF_DBG_SYS(EP_CTRL, "error=%d (%s)\n", error, fi_strerror(-error));
+ USDF_DBG_SYS(EP_CTRL, "error=%d (%s)\n", error, fi_strerror(error));
pep = crp->cr_pep;
ep = crp->cr_ep;
+
if (ep != NULL) {
fid = ep_utofid(ep);
eq = ep->ep_eq;
@@ -236,13 +294,19 @@ usdf_cm_msg_connreq_failed(struct usdf_connreq *crp, int error)
eq = pep->pep_eq;
}
+ /* Try to generate the space necessary for the error data. If the
+ * function returns a number greater than or equal to 0, then it was a
+ * success. The return value is the size of the data.
+ */
+ if (copy_data) {
+ ret = usdf_cm_generate_err_data(eq, crp, &err.err_data);
+ if (ret >= 0)
+ err.err_data_size = ret;
+ }
+
err.fid = fid;
- err.context = NULL;
- err.data = 0;
err.err = -error;
- err.prov_errno = 0;
- err.err_data = NULL;
- err.err_data_size = 0;
+
usdf_eq_write_internal(eq, 0, &err, sizeof(err), USDF_EVENT_FLAG_ERROR);
usdf_cm_msg_connreq_cleanup(crp);
@@ -268,12 +332,12 @@ usdf_cm_msg_connect_cb_rd(void *v)
fp = ep->ep_domain->dom_fabric;
ret = read(crp->cr_sockfd, crp->cr_ptr, crp->cr_resid);
- if (ret == -1) {
- usdf_cm_msg_connreq_failed(crp, -errno);
- return 0;
- }
+ if (ret == -1)
+ goto report_failure_skip_data;
+ crp->cr_ptr += ret;
crp->cr_resid -= ret;
+
reqp = (struct usdf_connreq_msg *)crp->cr_data;
if (crp->cr_resid == 0 && crp->cr_ptr == crp->cr_data + sizeof(*reqp)) {
reqp->creq_datalen = ntohl(reqp->creq_datalen);
@@ -282,28 +346,31 @@ usdf_cm_msg_connect_cb_rd(void *v)
/* if resid is 0 now, completely done */
if (crp->cr_resid == 0) {
+ reqp->creq_result = ntohl(reqp->creq_result);
+
ret = epoll_ctl(fp->fab_epollfd, EPOLL_CTL_DEL,
crp->cr_sockfd, NULL);
close(crp->cr_sockfd);
crp->cr_sockfd = -1;
- entry_len = sizeof(*entry) + reqp->creq_datalen;
- entry = malloc(entry_len);
- if (entry == NULL) {
- usdf_cm_msg_connreq_failed(crp, -errno);
+ if (reqp->creq_result != FI_SUCCESS) {
+ /* Copy the data since this was an explicit rejection.
+ */
+ usdf_cm_report_failure(crp, reqp->creq_result, true);
return 0;
}
-
+
+ entry_len = sizeof(*entry) + reqp->creq_datalen;
+ entry = malloc(entry_len);
+ if (entry == NULL)
+ goto report_failure_skip_data;
+
udp = ep->ep_domain;
ep->e.msg.ep_lcl_peer_id = ntohs(reqp->creq_peer_id);
- ret = usd_create_dest_with_mac(udp->dom_dev, reqp->creq_ipaddr,
- reqp->creq_port, reqp->creq_mac,
- &ep->e.msg.ep_dest);
- if (ret != 0) {
- free(entry);
- usdf_cm_msg_connreq_failed(crp, ret);
- return 0;
- }
+ ret = usd_create_dest(udp->dom_dev, reqp->creq_ipaddr,
+ reqp->creq_port, &ep->e.msg.ep_dest);
+ if (ret != 0)
+ goto free_entry_and_report_failure;
ep->e.msg.ep_dest->ds_dest.ds_udp.u_hdr.uh_ip.frag_off |=
htons(IP_DF);
@@ -313,17 +380,23 @@ usdf_cm_msg_connect_cb_rd(void *v)
memcpy(entry->data, reqp->creq_data, reqp->creq_datalen);
ret = usdf_eq_write_internal(ep->ep_eq, FI_CONNECTED, entry,
entry_len, 0);
- free(entry);
if (ret != (int)entry_len) {
free(ep->e.msg.ep_dest);
ep->e.msg.ep_dest = NULL;
- usdf_cm_msg_connreq_failed(crp, ret);
- return 0;
+
+ goto free_entry_and_report_failure;
}
+ free(entry);
usdf_cm_msg_connreq_cleanup(crp);
}
return 0;
+
+free_entry_and_report_failure:
+ free(entry);
+report_failure_skip_data:
+ usdf_cm_report_failure(crp, ret, false);
+ return 0;
}
/*
@@ -346,24 +419,24 @@ usdf_cm_msg_connect_cb_wr(void *v)
ret = write(crp->cr_sockfd, crp->cr_ptr, crp->cr_resid);
if (ret == -1) {
- usdf_cm_msg_connreq_failed(crp, -errno);
+ usdf_cm_report_failure(crp, -errno, false);
return 0;
}
crp->cr_resid -= ret;
if (crp->cr_resid == 0) {
crp->cr_pollitem.pi_rtn = usdf_cm_msg_connect_cb_rd;
+ crp->cr_ptr = crp->cr_data;
+ crp->cr_resid = sizeof(struct usdf_connreq_msg);
+
ev.events = EPOLLIN;
ev.data.ptr = &crp->cr_pollitem;
ret = epoll_ctl(fp->fab_epollfd, EPOLL_CTL_MOD,
crp->cr_sockfd, &ev);
if (ret != 0) {
- usdf_cm_msg_connreq_failed(crp, -errno);
+ usdf_cm_report_failure(crp, -errno, false);
return 0;
}
-
- crp->cr_ptr = crp->cr_data;
- crp->cr_resid = sizeof(struct usdf_connreq_msg);
}
return 0;
}
@@ -381,6 +454,7 @@ usdf_cm_msg_connect(struct fid_ep *fep, const void *addr,
struct usdf_fabric *fp;
struct usdf_connreq_msg *reqp;
struct usd_qp_impl *qp;
+ size_t request_size;
int ret;
USDF_TRACE_SYS(EP_CTRL, "\n");
@@ -392,10 +466,16 @@ usdf_cm_msg_connect(struct fid_ep *fep, const void *addr,
udp = ep->ep_domain;
fp = udp->dom_fabric;
sin = addr;
- crp = NULL;
- crp = calloc(1, sizeof(*crp) + sizeof(struct usdf_connreq_msg) +
- paramlen);
+ /* Although paramlen may be less than USDF_MAX_CONN_DATA, the same crp
+ * struct is used for receiving the accept and reject payload. The
+ * structure has to be prepared to receive the maximum allowable amount
+ * of data per transfer. The maximum size includes the connection
+ * request structure, the connection request message, and the maximum
+ * amount of data per connection request message.
+ */
+ request_size = sizeof(*crp) + sizeof(*reqp) + USDF_MAX_CONN_DATA;
+ crp = calloc(1, request_size);
if (crp == NULL) {
ret = -errno;
goto fail;
@@ -415,13 +495,8 @@ usdf_cm_msg_connect(struct fid_ep *fep, const void *addr,
ep->e.msg.ep_cm_sock = -1;
}
- ret = fcntl(crp->cr_sockfd, F_GETFL, 0);
- if (ret == -1) {
- ret = -errno;
- goto fail;
- }
- ret = fcntl(crp->cr_sockfd, F_SETFL, ret | O_NONBLOCK);
- if (ret == -1) {
+ ret = fi_fd_nonblock(crp->cr_sockfd);
+ if (ret) {
ret = -errno;
goto fail;
}
@@ -445,18 +520,6 @@ usdf_cm_msg_connect(struct fid_ep *fep, const void *addr,
if (ret)
goto fail;
- /* register for notification when connect completes */
- crp->cr_pollitem.pi_rtn = usdf_cm_msg_connect_cb_wr;
- crp->cr_pollitem.pi_context = crp;
- ev.events = EPOLLOUT;
- ev.data.ptr = &crp->cr_pollitem;
- ret = epoll_ctl(fp->fab_epollfd, EPOLL_CTL_ADD, crp->cr_sockfd, &ev);
- if (ret != 0) {
- crp->cr_pollitem.pi_rtn = NULL;
- ret = -errno;
- goto fail;
- }
-
/* allocate remote peer ID */
ep->e.msg.ep_rem_peer_id = udp->dom_next_peer;
udp->dom_peer_tab[udp->dom_next_peer] = ep;
@@ -471,10 +534,21 @@ usdf_cm_msg_connect(struct fid_ep *fep, const void *addr,
reqp->creq_ipaddr = fp->fab_dev_attrs->uda_ipaddr_be;
reqp->creq_port =
qp->uq_attrs.uqa_local_addr.ul_addr.ul_udp.u_addr.sin_port;
- memcpy(reqp->creq_mac, fp->fab_dev_attrs->uda_mac_addr, ETH_ALEN);
reqp->creq_datalen = htonl(paramlen);
memcpy(reqp->creq_data, param, paramlen);
+ /* register for notification when connect completes */
+ crp->cr_pollitem.pi_rtn = usdf_cm_msg_connect_cb_wr;
+ crp->cr_pollitem.pi_context = crp;
+ ev.events = EPOLLOUT;
+ ev.data.ptr = &crp->cr_pollitem;
+ ret = epoll_ctl(fp->fab_epollfd, EPOLL_CTL_ADD, crp->cr_sockfd, &ev);
+ if (ret != 0) {
+ crp->cr_pollitem.pi_rtn = NULL;
+ ret = -errno;
+ goto fail;
+ }
+
return 0;
fail:
diff --git a/prov/usnic/src/usdf_cm.h b/prov/usnic/src/usdf_cm.h
index 03cac77..0359c6f 100644
--- a/prov/usnic/src/usdf_cm.h
+++ b/prov/usnic/src/usdf_cm.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2014-2016, Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -45,8 +45,6 @@ struct usdf_connreq_msg {
uint32_t creq_peer_id;
uint32_t creq_ipaddr;
uint32_t creq_port;
- uint8_t creq_mac[ETH_ALEN];
- uint8_t pad[8 - ETH_ALEN];
uint32_t creq_result;
uint32_t creq_reason;
uint32_t creq_datalen;
@@ -69,7 +67,9 @@ struct usdf_connreq {
uint8_t cr_data[0];
};
-void usdf_cm_msg_connreq_failed(struct usdf_connreq *crp, int error);
+void usdf_cm_report_failure(struct usdf_connreq *crp, int error,
+ bool skip_data);
+void usdf_cm_msg_connreq_cleanup(struct usdf_connreq *crp);
int usdf_cm_rdm_getname(fid_t fid, void *addr, size_t *addrlen);
int usdf_cm_dgram_getname(fid_t fid, void *addr, size_t *addrlen);
diff --git a/prov/usnic/src/usdf_cq.c b/prov/usnic/src/usdf_cq.c
index ba4cef4..4823e3d 100644
--- a/prov/usnic/src/usdf_cq.c
+++ b/prov/usnic/src/usdf_cq.c
@@ -51,7 +51,6 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
@@ -438,7 +437,7 @@ usdf_progress_hard_cq(struct usdf_cq_hard *hcq)
void
usdf_cq_post_soft(struct usdf_cq_hard *hcq, void *context, size_t len,
- int prov_errno)
+ int prov_errno, uint64_t flags)
{
int ret;
struct usdf_cq_soft_entry *entry;
@@ -461,6 +460,7 @@ usdf_cq_post_soft(struct usdf_cq_hard *hcq, void *context, size_t len,
entry->cse_context = context;
entry->cse_len = len;
entry->cse_prov_errno = prov_errno;
+ entry->cse_flags = flags;
/* update with wrap */
entry++;
diff --git a/prov/usnic/src/usdf_cq.h b/prov/usnic/src/usdf_cq.h
index 6ec5379..bad6d74 100644
--- a/prov/usnic/src/usdf_cq.h
+++ b/prov/usnic/src/usdf_cq.h
@@ -50,6 +50,6 @@ int usdf_cq_trywait(struct fid *fcq);
void usdf_progress_hard_cq(struct usdf_cq_hard *hcq);
void usdf_cq_post_soft(struct usdf_cq_hard *hcq, void *context,
- size_t len, int prov_errno);
+ size_t len, int prov_errno, uint64_t flags);
#endif /* _USDF_CQ_H_ */
diff --git a/prov/usnic/src/usdf_dgram.c b/prov/usnic/src/usdf_dgram.c
index e1a9422..ee9f719 100644
--- a/prov/usnic/src/usdf_dgram.c
+++ b/prov/usnic/src/usdf_dgram.c
@@ -49,7 +49,6 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
@@ -159,7 +158,7 @@ usdf_dgram_recvv(struct fid_ep *fep, const struct iovec *iov, void **desc,
qp = to_qpi(ep->e.dg.ep_qp);
rxd.urd_context = context;
- rxd.urd_iov[0].iov_base = ep->e.dg.ep_hdr_buf +
+ rxd.urd_iov[0].iov_base = ((uint8_t *)ep->e.dg.ep_hdr_buf) +
qp->uq_rq.urq_post_index * USDF_HDR_BUF_ENTRY;
rxd.urd_iov[0].iov_len = sizeof(struct usd_udp_hdr);
memcpy(&rxd.urd_iov[1], iov, sizeof(*iov) * count);
@@ -197,7 +196,8 @@ usdf_dgram_recvmsg(struct fid_ep *fep, const struct fi_msg *msg, uint64_t flags)
iovp = msg->msg_iov;
rq->urq_context[index] = msg->context;
- hdr_ptr = ep->e.dg.ep_hdr_buf + (index * USDF_HDR_BUF_ENTRY);
+ hdr_ptr = ((uint8_t *)ep->e.dg.ep_hdr_buf) +
+ (index * USDF_HDR_BUF_ENTRY);
rq_enet_desc_enc(desc, (dma_addr_t) hdr_ptr,
RQ_ENET_TYPE_ONLY_SOP, sizeof(struct usd_udp_hdr));
ep->e.dg.ep_hdr_ptr[index] = (struct usd_udp_hdr *) hdr_ptr;
@@ -427,7 +427,7 @@ usdf_dgram_inject(struct fid_ep *fep, const void *buf, size_t len,
ssize_t usdf_dgram_prefix_inject(struct fid_ep *fep, const void *buf,
size_t len, fi_addr_t dest_addr)
{
- return usdf_dgram_inject(fep, buf + USDF_HDR_BUF_ENTRY,
+ return usdf_dgram_inject(fep, ((uint8_t *)buf) + USDF_HDR_BUF_ENTRY,
len - USDF_HDR_BUF_ENTRY, dest_addr);
}
@@ -442,8 +442,8 @@ ssize_t usdf_dgram_rx_size_left(struct fid_ep *fep)
ep = ep_ftou(fep);
- if (ep->e.dg.ep_qp == NULL)
- return -FI_EOPBADSTATE; /* EP not enabled */
+ if (!(ep->flags & USDF_EP_ENABLED))
+ return -FI_EOPBADSTATE;
return usd_get_recv_credits(ep->e.dg.ep_qp) /
(ep->e.dg.rx_iov_limit + 1);
@@ -460,8 +460,8 @@ ssize_t usdf_dgram_tx_size_left(struct fid_ep *fep)
ep = ep_ftou(fep);
- if (ep->e.dg.ep_qp == NULL)
- return -FI_EOPBADSTATE; /* EP not enabled */
+ if (!(ep->flags & USDF_EP_ENABLED))
+ return -FI_EOPBADSTATE;
return usd_get_send_credits(ep->e.dg.ep_qp) /
(ep->e.dg.tx_iov_limit + 1);
@@ -550,7 +550,7 @@ usdf_dgram_prefix_recvmsg(struct fid_ep *fep, const struct fi_msg *msg, uint64_t
iovp = msg->msg_iov;
rq->urq_context[index] = msg->context;
- hdr_ptr = iovp[0].iov_base +
+ hdr_ptr = ((uint8_t *)iovp[0].iov_base) +
(USDF_HDR_BUF_ENTRY - sizeof(struct usd_udp_hdr));
rq_enet_desc_enc(desc, (dma_addr_t) hdr_ptr,
RQ_ENET_TYPE_ONLY_SOP,
@@ -614,7 +614,7 @@ usdf_dgram_prefix_send(struct fid_ep *fep, const void *buf, size_t len,
}
return usd_post_send_one_copy(ep->e.dg.ep_qp, &dest->ds_dest,
- buf + USDF_HDR_BUF_ENTRY, len -
+ ((uint8_t *)buf) + USDF_HDR_BUF_ENTRY, len -
USDF_HDR_BUF_ENTRY, flags,
context);
}
@@ -777,8 +777,8 @@ ssize_t usdf_dgram_prefix_rx_size_left(struct fid_ep *fep)
ep = ep_ftou(fep);
- if (ep->e.dg.ep_qp == NULL)
- return -FI_EOPBADSTATE; /* EP not enabled */
+ if (!(ep->flags & USDF_EP_ENABLED))
+ return -FI_EOPBADSTATE;
/* prefix_recvv can post up to iov_limit descriptors
*/
@@ -796,8 +796,8 @@ ssize_t usdf_dgram_prefix_tx_size_left(struct fid_ep *fep)
ep = ep_ftou(fep);
- if (ep->e.dg.ep_qp == NULL)
- return -FI_EOPBADSTATE; /* EP not enabled */
+ if (!(ep->flags & USDF_EP_ENABLED))
+ return -FI_EOPBADSTATE;
/* prefix_sendvcan post up to iov_limit descriptors
*/
diff --git a/prov/usnic/src/usdf_dgram.h b/prov/usnic/src/usdf_dgram.h
index ce94b0a..a82e2de 100644
--- a/prov/usnic/src/usdf_dgram.h
+++ b/prov/usnic/src/usdf_dgram.h
@@ -57,7 +57,8 @@ int usdf_dgram_fill_rx_attr(struct fi_info *hints,
struct fi_info *fi, struct usd_device_attrs *dap);
int usdf_dgram_fill_tx_attr(struct fi_info *hints, struct fi_info *fi,
struct usd_device_attrs *dap);
-int usdf_dgram_fill_dom_attr(struct fi_info *hints, struct fi_info *fi);
+int usdf_dgram_fill_dom_attr(uint32_t version, struct fi_info *hints,
+ struct fi_info *fi, struct usd_device_attrs *dap);
int usdf_dgram_fill_ep_attr(uint32_t version, struct fi_info *hints,
struct fi_info *fi, struct usd_device_attrs *dap);
diff --git a/prov/usnic/src/usdf_domain.c b/prov/usnic/src/usdf_domain.c
index 070de61..5f69d94 100644
--- a/prov/usnic/src/usdf_domain.c
+++ b/prov/usnic/src/usdf_domain.c
@@ -46,7 +46,6 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
@@ -238,7 +237,16 @@ usdf_domain_open(struct fid_fabric *fabric, struct fi_info *info,
USDF_TRACE_SYS(DOMAIN, "\n");
+ fp = fab_fidtou(fabric);
+
if (info->domain_attr != NULL) {
+ /* No versioning information available here. */
+ if (!usdf_domain_checkname(0, fp->fab_dev_attrs,
+ info->domain_attr->name)) {
+ USDF_WARN_SYS(DOMAIN, "domain name mismatch\n");
+ return -FI_ENODATA;
+ }
+
switch (info->domain_attr->mr_mode) {
case FI_MR_UNSPEC:
case FI_MR_BASIC:
@@ -258,8 +266,6 @@ usdf_domain_open(struct fid_fabric *fabric, struct fi_info *info,
goto fail;
}
- fp = fab_fidtou(fabric);
-
USDF_DBG("uda_devname=%s\n", fp->fab_dev_attrs->uda_devname);
/*
@@ -345,3 +351,84 @@ fail:
}
return ret;
}
+
+int usdf_domain_getname(uint32_t version, struct usd_device_attrs *dap,
+ char **name)
+{
+ int ret = FI_SUCCESS;
+ char *buf = NULL;
+
+ if (FI_VERSION_GE(version, FI_VERSION(1, 4))) {
+ buf = strdup(dap->uda_devname);
+ if (!buf) {
+ ret = -errno;
+ USDF_DBG("strdup failed while creating domain name\n");
+ }
+ }
+
+ *name = buf;
+ return ret;
+}
+
+/* In pre-1.4 the domain name was NULL. This is unfortunate as it makes it
+ * difficult to tell whether providing a name was intended. In this case, it can
+ * be broken into 4 cases:
+ *
+ * 1. Version is greater than or equal to 1.4 and a non-NULL hint is provided.
+ * Just do a string compare.
+ * 2. Version is greater than or equal to 1.4 and provided hint is NULL. Treat
+ * this as _valid_ as it could be an application requesting a 1.4 domain name
+ * but not providing an explicit hint.
+ * 3. Version is less than 1.4 and a name hint is provided. This should always
+ * be _invalid_.
+ * 4. Version is less than 1.4 and name hint is NULL. This will always be
+ * _valid_.
+ */
+bool usdf_domain_checkname(uint32_t version, struct usd_device_attrs *dap,
+ char *hint)
+{
+ char *reference;
+ bool valid;
+ int ret;
+
+ USDF_DBG("checking domain name: version=%d, domain name='%s'\n",
+ version, hint);
+
+ if (version) {
+ valid = false;
+
+ ret = usdf_domain_getname(version, dap, &reference);
+ if (ret < 0)
+ return false;
+
+ /* If the reference name exists, then this is version 1.4 or
+ * greater.
+ */
+ if (reference) {
+ if (hint) {
+ /* Case 1 */
+ valid = (strcmp(reference, hint) == 0);
+ } else {
+ /* Case 2 */
+ valid = true;
+ }
+ } else {
+ /* Case 3 & 4 */
+ valid = (hint == NULL);
+ }
+
+ if (!valid)
+ USDF_DBG("given hint %s does not match %s -- invalid\n",
+ hint, reference);
+
+ free(reference);
+ return valid;
+ }
+
+ /* If hint is non-NULL then assume the version is 1.4 if not provided.
+ */
+ if (hint)
+ return usdf_domain_checkname(FI_VERSION(1, 4), dap, hint);
+
+ return usdf_domain_checkname(FI_VERSION(1, 3), dap, hint);
+}
diff --git a/prov/usnic/src/usdf_endpoint.c b/prov/usnic/src/usdf_endpoint.c
index dd7cb68..2689665 100644
--- a/prov/usnic/src/usdf_endpoint.c
+++ b/prov/usnic/src/usdf_endpoint.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2014-2016, Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -49,7 +49,6 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
@@ -58,6 +57,7 @@
#include "usdf.h"
#include "usdf_endpoint.h"
+#include "usdf_cm.h"
int
usdf_endpoint_open(struct fid_domain *domain, struct fi_info *info,
@@ -76,3 +76,52 @@ usdf_endpoint_open(struct fid_domain *domain, struct fi_info *info,
return -FI_ENODEV;
}
}
+
+int usdf_ep_getopt_connected(fid_t fid, int level, int optname, void *optval,
+ size_t *optlen)
+{
+ size_t *cm_size;
+ size_t dest_size;
+
+ USDF_TRACE_SYS(EP_CTRL, "\n");
+
+ if (!optval || !optlen)
+ return -FI_EINVAL;
+
+ if (level != FI_OPT_ENDPOINT)
+ return -FI_ENOPROTOOPT;
+
+ switch (optname) {
+ case FI_OPT_CM_DATA_SIZE:
+ dest_size = *optlen;
+ *optlen = sizeof(*cm_size);
+
+ if (dest_size < sizeof(*cm_size))
+ return -FI_ETOOSMALL;
+
+ cm_size = optval;
+ *cm_size = USDF_MAX_CONN_DATA;
+ break;
+ default:
+ return -FI_ENOPROTOOPT;
+ }
+
+ return FI_SUCCESS;
+}
+
+int usdf_ep_getopt_unconnected(fid_t fid, int level, int optname, void *optval,
+ size_t *optlen)
+{
+ USDF_TRACE_SYS(EP_CTRL, "\n");
+
+ return -FI_ENOPROTOOPT;
+}
+
+
+int usdf_ep_setopt(fid_t fid, int level, int optname, const void *optval,
+ size_t optlen)
+{
+ USDF_TRACE_SYS(EP_CTRL, "\n");
+
+ return -FI_ENOPROTOOPT;
+}
diff --git a/prov/usnic/src/usdf_endpoint.h b/prov/usnic/src/usdf_endpoint.h
index debcf53..994f76c 100644
--- a/prov/usnic/src/usdf_endpoint.h
+++ b/prov/usnic/src/usdf_endpoint.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2014-2016, Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -47,6 +47,13 @@ int usdf_ep_msg_get_queues(struct usdf_ep *ep);
void usdf_ep_msg_release_queues(struct usdf_ep *ep);
int usdf_msg_upd_lcl_addr(struct usdf_ep *ep);
+int usdf_ep_getopt_connected(fid_t fid, int level, int optname, void *optval,
+ size_t *optlen);
+int usdf_ep_getopt_unconnected(fid_t fid, int level, int optname, void *optval,
+ size_t *optlen);
+int usdf_ep_setopt(fid_t fid, int level, int optname, const void *optval,
+ size_t optlen);
+
extern struct fi_ops usdf_ep_ops;
#endif /* _USDF_ENDPOINT_H_ */
diff --git a/prov/usnic/src/usdf_ep_dgram.c b/prov/usnic/src/usdf_ep_dgram.c
index 34f881d..325a950 100644
--- a/prov/usnic/src/usdf_ep_dgram.c
+++ b/prov/usnic/src/usdf_ep_dgram.c
@@ -50,7 +50,6 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
@@ -139,6 +138,8 @@ usdf_ep_dgram_enable(struct fid_ep *fep)
goto fail;
}
+ ep->flags |= USDF_EP_ENABLED;
+
return 0;
fail:
@@ -157,6 +158,7 @@ usdf_ep_dgram_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
{
struct usdf_ep *ep;
struct usdf_cq *cq;
+ struct usdf_av *av;
int ret;
USDF_TRACE_SYS(EP_CTRL, "\n");
@@ -169,7 +171,10 @@ usdf_ep_dgram_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
if (ep->e.dg.ep_av != NULL) {
return -FI_EINVAL;
}
- ep->e.dg.ep_av = av_fidtou(bfid);
+
+ av = av_fidtou(bfid);
+ ep->e.dg.ep_av = av;
+ atomic_inc(&av->av_refcnt);
break;
case FI_CLASS_CQ:
@@ -281,6 +286,10 @@ usdf_ep_dgram_close(fid_t fid)
if (ep->ep_eq != NULL) {
atomic_dec(&ep->ep_eq->eq_refcnt);
}
+
+ if (ep->e.dg.ep_av)
+ atomic_dec(&ep->e.dg.ep_av->av_refcnt);
+
usdf_ep_dgram_deref_cq(ep->e.dg.ep_wcq);
usdf_ep_dgram_deref_cq(ep->e.dg.ep_rcq);
@@ -455,11 +464,16 @@ out:
return FI_SUCCESS;
}
-int usdf_dgram_fill_dom_attr(struct fi_info *hints, struct fi_info *fi)
+int usdf_dgram_fill_dom_attr(uint32_t version, struct fi_info *hints,
+ struct fi_info *fi, struct usd_device_attrs *dap)
{
+ int ret;
struct fi_domain_attr defaults;
defaults = dgram_dflt_domain_attr;
+ ret = usdf_domain_getname(version, dap, &defaults.name);
+ if (ret < 0)
+ return -FI_ENODATA;
if (!hints || !hints->domain_attr)
goto out;
diff --git a/prov/usnic/src/usdf_ep_msg.c b/prov/usnic/src/usdf_ep_msg.c
index 1baa08c..daf7af1 100644
--- a/prov/usnic/src/usdf_ep_msg.c
+++ b/prov/usnic/src/usdf_ep_msg.c
@@ -50,7 +50,6 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
@@ -164,11 +163,16 @@ out:
return FI_SUCCESS;
}
-int usdf_msg_fill_dom_attr(struct fi_info *hints, struct fi_info *fi)
+int usdf_msg_fill_dom_attr(uint32_t version, struct fi_info *hints,
+ struct fi_info *fi, struct usd_device_attrs *dap)
{
+ int ret;
struct fi_domain_attr defaults;
defaults = msg_dflt_domain_attr;
+ ret = usdf_domain_getname(version, dap, &defaults.name);
+ if (ret < 0)
+ return -FI_ENODATA;
if (!hints || !hints->domain_attr)
goto out;
@@ -548,53 +552,16 @@ fail:
static int
usdf_ep_msg_enable(struct fid_ep *fep)
{
- return usdf_ep_msg_get_queues(ep_ftou(fep));
-}
-
-static int
-usdf_ep_msg_getopt(fid_t fid, int level, int optname,
- void *optval, size_t *optlen)
-{
- struct usdf_ep *ep;
-
- USDF_TRACE_SYS(EP_CTRL, "\n");
-
- ep = ep_fidtou(fid);
- (void)ep;
-
- switch (level) {
- case FI_OPT_CM_DATA_SIZE:
- if (*optlen < sizeof(size_t))
- return -FI_ETOOSMALL;
- *((size_t *) optval) = USDF_MAX_CONN_DATA;
- *optlen = sizeof(size_t);
- return 0;
- case FI_OPT_ENDPOINT:
- return -FI_ENOPROTOOPT;
- default:
- return -FI_ENOPROTOOPT;
- }
- return 0;
-}
-
-static int
-usdf_ep_msg_setopt(fid_t fid, int level, int optname,
- const void *optval, size_t optlen)
-{
struct usdf_ep *ep;
+ int ret;
- USDF_TRACE_SYS(EP_CTRL, "\n");
+ ep = ep_ftou(fep);
- ep = ep_fidtou(fid);
- (void)ep;
+ ret = usdf_ep_msg_get_queues(ep);
+ if (ret == FI_SUCCESS)
+ ep->flags |= USDF_EP_ENABLED;
- switch (level) {
- case FI_OPT_ENDPOINT:
- return -FI_ENOPROTOOPT;
- default:
- return -FI_ENOPROTOOPT;
- }
- return 0;
+ return ret;
}
static ssize_t
@@ -837,7 +804,7 @@ usdf_ep_msg_close(fid_t fid)
atomic_dec(&ep->ep_eq->eq_refcnt);
}
usdf_timer_free(ep->ep_domain->dom_fabric, ep->e.msg.ep_ack_timer);
-
+
free(ep);
return 0;
}
@@ -845,8 +812,8 @@ usdf_ep_msg_close(fid_t fid)
static struct fi_ops_ep usdf_base_msg_ops = {
.size = sizeof(struct fi_ops_ep),
.cancel = usdf_ep_msg_cancel,
- .getopt = usdf_ep_msg_getopt,
- .setopt = usdf_ep_msg_setopt,
+ .getopt = usdf_ep_getopt_connected,
+ .setopt = usdf_ep_setopt,
.tx_ctx = fi_no_tx_ctx,
.rx_ctx = fi_no_rx_ctx,
.rx_size_left = usdf_msg_rx_size_left,
diff --git a/prov/usnic/src/usdf_ep_rdm.c b/prov/usnic/src/usdf_ep_rdm.c
index 112f280..2f0db1e 100644
--- a/prov/usnic/src/usdf_ep_rdm.c
+++ b/prov/usnic/src/usdf_ep_rdm.c
@@ -51,7 +51,6 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
@@ -60,6 +59,7 @@
#include "usd.h"
#include "usdf.h"
+#include "usnic_direct.h"
#include "usdf_endpoint.h"
#include "fi_ext_usnic.h"
#include "usdf_rudp.h"
@@ -167,11 +167,16 @@ out:
}
-int usdf_rdm_fill_dom_attr(struct fi_info *hints, struct fi_info *fi)
+int usdf_rdm_fill_dom_attr(uint32_t version, struct fi_info *hints,
+ struct fi_info *fi, struct usd_device_attrs *dap)
{
+ int ret;
struct fi_domain_attr defaults;
defaults = rdm_dflt_domain_attr;
+ ret = usdf_domain_getname(version, dap, &defaults.name);
+ if (ret < 0)
+ return -FI_ENODATA;
if (!hints || !hints->domain_attr)
goto out;
@@ -539,47 +544,16 @@ fail:
static int
usdf_ep_rdm_enable(struct fid_ep *fep)
{
- return usdf_ep_rdm_get_queues(ep_ftou(fep));
-}
-
-static int
-usdf_ep_rdm_getopt(fid_t fid, int level, int optname,
- void *optval, size_t *optlen)
-{
- struct usdf_ep *ep;
-
- USDF_TRACE_SYS(EP_CTRL, "\n");
-
- ep = ep_fidtou(fid);
- (void)ep;
-
- switch (level) {
- case FI_OPT_ENDPOINT:
- return -FI_ENOPROTOOPT;
- default:
- return -FI_ENOPROTOOPT;
- }
- return 0;
-}
-
-static int
-usdf_ep_rdm_setopt(fid_t fid, int level, int optname,
- const void *optval, size_t optlen)
-{
struct usdf_ep *ep;
+ int ret;
- USDF_TRACE_SYS(EP_CTRL, "\n");
+ ep = ep_ftou(fep);
- ep = ep_fidtou(fid);
- (void)ep;
+ ret = usdf_ep_rdm_get_queues(ep);
+ if (ret == FI_SUCCESS)
+ ep->flags |= USDF_EP_ENABLED;
- switch (level) {
- case FI_OPT_ENDPOINT:
- return -FI_ENOPROTOOPT;
- default:
- return -FI_ENOPROTOOPT;
- }
- return 0;
+ return ret;
}
static ssize_t
@@ -691,6 +665,7 @@ usdf_ep_rdm_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
{
struct usdf_ep *ep;
struct usdf_cq *cq;
+ struct usdf_av *av;
USDF_TRACE_SYS(EP_CTRL, "\n");
@@ -702,9 +677,11 @@ usdf_ep_rdm_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
if (ep->e.rdm.ep_av != NULL) {
return -FI_EINVAL;
}
- ep->e.rdm.ep_av = av_fidtou(bfid);
- break;
+ av = av_fidtou(bfid);
+ ep->e.rdm.ep_av = av;
+ atomic_inc(&av->av_refcnt);
+ break;
case FI_CLASS_CQ:
if (flags & FI_SEND) {
@@ -834,7 +811,7 @@ usdf_rx_rdm_port_bind(struct usdf_rx *rx, struct fi_info *info)
sin->sin_addr.s_addr =
rx->rx_domain->dom_fabric->fab_dev_attrs->uda_ipaddr_be;
}
-
+
rx->r.rdm.rx_sock = socket(AF_INET, SOCK_DGRAM, 0);
if (rx->r.rdm.rx_sock == -1) {
return -errno;
@@ -884,7 +861,10 @@ usdf_ep_rdm_close(fid_t fid)
if (ep->ep_eq != NULL) {
atomic_dec(&ep->ep_eq->eq_refcnt);
}
-
+
+ if (ep->e.rdm.ep_av)
+ atomic_dec(&ep->e.rdm.ep_av->av_refcnt);
+
free(ep);
return 0;
}
@@ -892,8 +872,8 @@ usdf_ep_rdm_close(fid_t fid)
static struct fi_ops_ep usdf_base_rdm_ops = {
.size = sizeof(struct fi_ops_ep),
.cancel = usdf_ep_rdm_cancel,
- .getopt = usdf_ep_rdm_getopt,
- .setopt = usdf_ep_rdm_setopt,
+ .getopt = usdf_ep_getopt_unconnected,
+ .setopt = usdf_ep_setopt,
.tx_ctx = fi_no_tx_ctx,
.rx_ctx = fi_no_rx_ctx,
.rx_size_left = usdf_rdm_rx_size_left,
diff --git a/prov/usnic/src/usdf_eq.c b/prov/usnic/src/usdf_eq.c
index 52cb216..48a330e 100644
--- a/prov/usnic/src/usdf_eq.c
+++ b/prov/usnic/src/usdf_eq.c
@@ -52,7 +52,6 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
@@ -93,6 +92,13 @@ static inline ssize_t usdf_eq_read_event(struct usdf_eq *eq, uint32_t *event,
copylen = MIN(ev->ue_len, len);
+ if (copylen < ev->ue_len) {
+ USDF_WARN_SYS(EQ,
+ "buffer too small, got: %zu needed %zu\n",
+ copylen, ev->ue_len);
+ return -FI_ETOOSMALL;
+ }
+
/* copy out the event */
if (event)
*event = ev->ue_event;
@@ -167,6 +173,7 @@ usdf_eq_write_event(struct usdf_eq *eq, uint32_t event,
static ssize_t usdf_eq_readerr(struct fid_eq *feq,
struct fi_eq_err_entry *entry, uint64_t flags)
{
+ struct usdf_err_data_entry *err_data_entry;
struct usdf_eq *eq;
ssize_t ret;
@@ -189,11 +196,38 @@ static ssize_t usdf_eq_readerr(struct fid_eq *feq,
ret = usdf_eq_read_event(eq, NULL, entry, sizeof(*entry), flags);
+ /* Mark as seen so it can be cleaned on the next iteration of read. */
+ if (entry->err_data_size) {
+ err_data_entry = container_of(entry->err_data,
+ struct usdf_err_data_entry, err_data);
+ err_data_entry->seen = 1;
+ }
+
done:
pthread_spin_unlock(&eq->eq_lock);
return ret;
}
+static void usdf_eq_clean_err(struct usdf_eq *eq, uint8_t destroy)
+{
+ struct usdf_err_data_entry *err_data_entry;
+ struct slist_entry *entry;
+
+ while (!slist_empty(&eq->eq_err_data)) {
+ entry = slist_remove_head(&eq->eq_err_data);
+ err_data_entry = container_of(entry, struct usdf_err_data_entry,
+ entry);
+ if (err_data_entry->seen || destroy) {
+ free(err_data_entry);
+ } else {
+ /* Oops, the rest hasn't been seen yet. Put this back
+ * and exit.
+ */
+ slist_insert_head(entry, &eq->eq_err_data);
+ break;
+ }
+ }
+}
static ssize_t _usdf_eq_read(struct usdf_eq *eq, uint32_t *event, void *buf,
size_t len, uint64_t flags)
@@ -212,6 +246,9 @@ static ssize_t _usdf_eq_read(struct usdf_eq *eq, uint32_t *event, void *buf,
goto done;
}
+ if (!slist_empty(&eq->eq_err_data))
+ usdf_eq_clean_err(eq, 0);
+
ret = usdf_eq_read_event(eq, event, buf, len, flags);
done:
@@ -461,6 +498,9 @@ usdf_eq_close(fid_t fid)
break;
}
+ /* Set destroy flag to clear everything out */
+ usdf_eq_clean_err(eq, 1);
+
free(eq->eq_ev_ring);
free(eq->eq_ev_buf);
free(eq);
@@ -517,6 +557,8 @@ usdf_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr,
goto fail;
}
+ slist_init(&eq->eq_err_data);
+
/* get baseline routines */
eq->eq_ops_data = usdf_eq_ops;
diff --git a/prov/usnic/src/usdf_fabric.c b/prov/usnic/src/usdf_fabric.c
index 87f7000..65718b7 100644
--- a/prov/usnic/src/usdf_fabric.c
+++ b/prov/usnic/src/usdf_fabric.c
@@ -47,6 +47,7 @@
#include <sys/socket.h>
#include <stdio.h>
#include <stdlib.h>
+#include <stdbool.h>
#include <string.h>
#include <netdb.h>
#include <unistd.h>
@@ -54,7 +55,6 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
@@ -76,10 +76,77 @@
struct usdf_usnic_info *__usdf_devinfo;
-static int
-usdf_validate_hints(struct fi_info *hints, struct usd_device_attrs *dap)
+static int usdf_fabric_getname(uint32_t version, struct usd_device_attrs *dap,
+ char **name)
+{
+ int ret = FI_SUCCESS;
+ char *bufp = NULL;
+ struct in_addr in;
+ char *addrnetw;
+
+ if (FI_VERSION_GE(version, FI_VERSION(1, 4))) {
+ in.s_addr = dap->uda_ipaddr_be & dap->uda_netmask_be;
+ addrnetw = inet_ntoa(in);
+ ret = asprintf(&bufp, "%s/%d", addrnetw, dap->uda_prefixlen);
+ if (ret < 0) {
+ USDF_DBG(
+ "asprintf failed while creating fabric name\n");
+ ret = -ENOMEM;
+ }
+ } else {
+ bufp = strdup(dap->uda_devname);
+ if (!bufp) {
+ USDF_DBG("strdup failed while creating fabric name\n");
+ ret = -errno;
+ }
+ }
+
+ *name = bufp;
+
+ return ret;
+}
+
+static bool usdf_fabric_checkname(uint32_t version,
+ struct usd_device_attrs *dap, char *hint)
+{
+ int ret;
+ bool valid = false;
+ char *reference;
+
+ USDF_DBG("checking devname: version=%d, devname='%s'\n", version, hint);
+
+ if (version) {
+ ret = usdf_fabric_getname(version, dap, &reference);
+ if (ret < 0)
+ return false;
+
+ if (strcmp(reference, hint) == 0) {
+ valid = true;
+ } else {
+ USDF_DBG("hint %s failed to match %s\n", hint,
+ reference);
+ }
+
+ free(reference);
+ return valid;
+ }
+
+ /* The hint string itself is kind of a version check, in pre-1.4 the
+ * name was just the device name. In 1.4 and beyond, then name is
+ * actually CIDR
+ * notation.
+ */
+ if (strstr(hint, "/"))
+ return usdf_fabric_checkname(FI_VERSION(1, 4), dap, hint);
+
+ return usdf_fabric_checkname(FI_VERSION(1, 3), dap, hint);
+}
+
+static int usdf_validate_hints(uint32_t version, struct fi_info *hints,
+ struct usd_device_attrs *dap)
{
struct fi_fabric_attr *fattrp;
+ struct fi_domain_attr *dattrp;
size_t size;
switch (hints->addr_format) {
@@ -117,13 +184,20 @@ usdf_validate_hints(struct fi_info *hints, struct usd_device_attrs *dap)
fattrp->prov_version != USDF_PROV_VERSION) {
return -FI_ENODATA;
}
+
if (fattrp->name != NULL &&
- strcmp(fattrp->name, dap->uda_devname) != 0) {
+ !usdf_fabric_checkname(version, dap, fattrp->name)) {
return -FI_ENODATA;
}
}
- return 0;
+ dattrp = hints->domain_attr;
+ if (dattrp) {
+ if (!usdf_domain_checkname(version, dap, dattrp->name))
+ return -FI_ENODATA;
+ }
+
+ return FI_SUCCESS;
}
static int
@@ -190,8 +264,7 @@ fail:
return ret; // fi_freeinfo() in caller frees all
}
-static int
-usdf_fill_info_dgram(
+static int usdf_fill_info_dgram(
uint32_t version,
struct fi_info *hints,
struct sockaddr_in *src,
@@ -243,8 +316,8 @@ usdf_fill_info_dgram(
/* fabric attrs */
fattrp = fi->fabric_attr;
- fattrp->name = strdup(dap->uda_devname);
- if (fattrp->name == NULL) {
+ ret = usdf_fabric_getname(version, dap, &fattrp->name);
+ if (ret < 0 || fattrp->name == NULL) {
ret = -FI_ENOMEM;
goto fail;
}
@@ -260,7 +333,7 @@ usdf_fill_info_dgram(
if (ret)
goto fail;
- ret = usdf_dgram_fill_dom_attr(hints, fi);
+ ret = usdf_dgram_fill_dom_attr(version, hints, fi, dap);
if (ret)
goto fail;
@@ -289,8 +362,8 @@ fail:
return ret;
}
-static int
-usdf_fill_info_msg(
+static int usdf_fill_info_msg(
+ uint32_t version,
struct fi_info *hints,
struct sockaddr_in *src,
struct sockaddr_in *dest,
@@ -342,8 +415,8 @@ usdf_fill_info_msg(
/* fabric attrs */
fattrp = fi->fabric_attr;
- fattrp->name = strdup(dap->uda_devname);
- if (fattrp->name == NULL) {
+ ret = usdf_fabric_getname(version, dap, &fattrp->name);
+ if (ret < 0 || fattrp->name == NULL) {
ret = -FI_ENOMEM;
goto fail;
}
@@ -352,7 +425,7 @@ usdf_fill_info_msg(
if (ret)
goto fail;
- ret = usdf_msg_fill_dom_attr(hints, fi);
+ ret = usdf_msg_fill_dom_attr(version, hints, fi, dap);
if (ret)
goto fail;
@@ -381,8 +454,8 @@ fail:
return ret;
}
-static int
-usdf_fill_info_rdm(
+static int usdf_fill_info_rdm(
+ uint32_t version,
struct fi_info *hints,
struct sockaddr_in *src,
struct sockaddr_in *dest,
@@ -432,8 +505,8 @@ usdf_fill_info_rdm(
/* fabric attrs */
fattrp = fi->fabric_attr;
- fattrp->name = strdup(dap->uda_devname);
- if (fattrp->name == NULL) {
+ ret = usdf_fabric_getname(version, dap, &fattrp->name);
+ if (ret < 0 || fattrp->name == NULL) {
ret = -FI_ENOMEM;
goto fail;
}
@@ -442,7 +515,7 @@ usdf_fill_info_rdm(
if (ret)
goto fail;
- ret = usdf_rdm_fill_dom_attr(hints, fi);
+ ret = usdf_rdm_fill_dom_attr(version, hints, fi, dap);
if (ret)
goto fail;
@@ -641,7 +714,7 @@ usdf_getinfo(uint32_t version, const char *node, const char *service,
/* Does this device match requested attributes? */
if (hints != NULL) {
- ret = usdf_validate_hints(hints, dap);
+ ret = usdf_validate_hints(version, hints, dap);
if (ret != 0) {
USDF_DBG("hints do not match for %s/%s, skipping\n",
dap->uda_devname, dap->uda_ifname);
@@ -663,16 +736,16 @@ usdf_getinfo(uint32_t version, const char *node, const char *service,
}
if (ep_type == FI_EP_MSG || ep_type == FI_EP_UNSPEC) {
- ret = usdf_fill_info_msg(hints, src, dest, dap,
- &fi_first, &fi_last);
+ ret = usdf_fill_info_msg(version, hints, src, dest,
+ dap, &fi_first, &fi_last);
if (ret != 0 && ret != -FI_ENODATA) {
goto fail;
}
}
if (ep_type == FI_EP_RDM || ep_type == FI_EP_UNSPEC) {
- ret = usdf_fill_info_rdm(hints, src, dest, dap,
- &fi_first, &fi_last);
+ ret = usdf_fill_info_rdm(version, hints, src, dest,
+ dap, &fi_first, &fi_last);
if (ret != 0 && ret != -FI_ENODATA) {
goto fail;
}
@@ -777,7 +850,7 @@ usdf_fabric_open(struct fi_fabric_attr *fattrp, struct fid_fabric **fabric,
for (d = 0; d < dp->uu_num_devs; ++d) {
dep = &dp->uu_info[d];
if (dep->ue_dev_ok &&
- strcmp(fattrp->name, dep->ue_dattr.uda_devname) == 0) {
+ usdf_fabric_checkname(0, &(dep->ue_dattr), fattrp->name)) {
break;
}
}
@@ -894,7 +967,7 @@ static void usdf_fini(void)
struct fi_provider usdf_ops = {
.name = USDF_PROV_NAME,
.version = USDF_PROV_VERSION,
- .fi_version = FI_VERSION(1, 3),
+ .fi_version = FI_VERSION(1, 4),
.getinfo = usdf_getinfo,
.fabric = usdf_fabric_open,
.cleanup = usdf_fini
diff --git a/prov/usnic/src/usdf_mem.c b/prov/usnic/src/usdf_mem.c
index 7bbb111..f48fc4d 100644
--- a/prov/usnic/src/usdf_mem.c
+++ b/prov/usnic/src/usdf_mem.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2014,2016, Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -49,7 +49,6 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
diff --git a/prov/usnic/src/usdf_msg.c b/prov/usnic/src/usdf_msg.c
index e662f81..1ae728d 100644
--- a/prov/usnic/src/usdf_msg.c
+++ b/prov/usnic/src/usdf_msg.c
@@ -50,7 +50,6 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
@@ -155,7 +154,7 @@ usdf_msg_rewind_qe(struct usdf_msg_qe *qe, size_t rewind, size_t mtu)
}
qe->ms_cur_iov = cur_iov;
- qe->ms_cur_ptr = qe->ms_iov[cur_iov].iov_base +
+ qe->ms_cur_ptr = ((uint8_t *)qe->ms_iov[cur_iov].iov_base) +
qe->ms_iov[cur_iov].iov_len - cur_resid;
qe->ms_iov_resid = cur_resid;
}
@@ -278,7 +277,7 @@ usdf_msg_recvv(struct fid_ep *fep, const struct iovec *iov, void **desc,
rqe->ms_cur_ptr = iov[0].iov_base;
rqe->ms_iov_resid = iov[0].iov_len;
rqe->ms_resid = tot_len;
- rqe->ms_length = tot_len;
+ rqe->ms_length = 0;
op_flags = ep->ep_rx->rx_attr.op_flags;
rqe->ms_signal_comp = ep->ep_rx_dflt_signal_comp ||
@@ -576,7 +575,7 @@ usdf_msg_recvmsg(struct fid_ep *fep, const struct fi_msg *msg, uint64_t flags)
rqe->ms_cur_iov = 0;
rqe->ms_resid = tot_len;
- rqe->ms_length = tot_len;
+ rqe->ms_length = 0;
rqe->ms_cur_ptr = iov[0].iov_base;
rqe->ms_iov_resid = iov[0].iov_len;
@@ -897,7 +896,8 @@ usdf_msg_recv_complete(struct usdf_ep *ep, struct usdf_msg_qe *rqe, int status)
rx = ep->ep_rx;
hcq = rx->r.msg.rx_hcq;
- hcq->cqh_post(hcq, rqe->ms_context, rqe->ms_length, status);
+ hcq->cqh_post(hcq, rqe->ms_context, rqe->ms_length, status,
+ FI_MSG | FI_RECV);
usdf_msg_put_rx_rqe(rx, rqe);
}
@@ -968,7 +968,8 @@ usdf_msg_process_ack(struct usdf_ep *ep, uint16_t seq)
USDF_DBG_SYS(EP_DATA, "send complete, signal_comp=%u\n", wqe->ms_signal_comp);
if (wqe->ms_signal_comp)
hcq->cqh_post(hcq, wqe->ms_context,
- wqe->ms_length, FI_SUCCESS);
+ wqe->ms_length, FI_SUCCESS,
+ FI_MSG | FI_SEND);
usdf_msg_put_tx_wqe(tx, wqe);
} else {
@@ -1230,8 +1231,9 @@ ssize_t usdf_msg_rx_size_left(struct fid_ep *fep)
ep = ep_ftou(fep);
rx = ep->ep_rx;
- if (rx == NULL)
- return -FI_EOPBADSTATE; /* EP not enabled */
+
+ if (!(ep->flags & USDF_EP_ENABLED))
+ return -FI_EOPBADSTATE;
return rx->r.msg.rx_num_free_rqe;
}
@@ -1245,8 +1247,9 @@ ssize_t usdf_msg_tx_size_left(struct fid_ep *fep)
ep = ep_ftou(fep);
tx = ep->ep_tx;
- if (tx == NULL)
- return -FI_EOPBADSTATE; /* EP not enabled */
+
+ if (!(ep->flags & USDF_EP_ENABLED))
+ return -FI_EOPBADSTATE;
return tx->t.msg.tx_num_free_wqe;
}
diff --git a/prov/usnic/src/usdf_msg.h b/prov/usnic/src/usdf_msg.h
index abc6690..3e9fb1f 100644
--- a/prov/usnic/src/usdf_msg.h
+++ b/prov/usnic/src/usdf_msg.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2014-2016, Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -93,7 +93,8 @@ int usdf_msg_fill_tx_attr(struct fi_info *hints, struct fi_info *fi);
int usdf_msg_fill_rx_attr(struct fi_info *hints, struct fi_info *fi);
int usdf_msg_fill_ep_attr(struct fi_info *hints, struct fi_info *fi,
struct usd_device_attrs *dap);
-int usdf_msg_fill_dom_attr(struct fi_info *hints, struct fi_info *fi);
+int usdf_msg_fill_dom_attr(uint32_t version, struct fi_info *hints,
+ struct fi_info *fi, struct usd_device_attrs *dap);
void usdf_msg_ep_timeout(void *vep);
diff --git a/prov/usnic/src/usdf_pep.c b/prov/usnic/src/usdf_pep.c
index 03d9c4b..031e754 100644
--- a/prov/usnic/src/usdf_pep.c
+++ b/prov/usnic/src/usdf_pep.c
@@ -52,17 +52,18 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
#include "fi.h"
#include "fi_enosys.h"
+#include "fi_file.h"
#include "fi_ext_usnic.h"
#include "usnic_direct.h"
#include "usd.h"
#include "usdf.h"
+#include "usdf_endpoint.h"
#include "usdf_cm.h"
#include "usdf_msg.h"
@@ -168,8 +169,8 @@ usdf_pep_read_connreq(void *v)
n = read(crp->cr_sockfd, crp->cr_ptr, crp->cr_resid);
if (n == -1) {
- usdf_cm_msg_connreq_failed(crp, -errno);
- return 0;
+ ret = -errno;
+ goto report_failure_skip_data;
}
crp->cr_ptr += n;
@@ -185,37 +186,41 @@ usdf_pep_read_connreq(void *v)
/* if resid is 0 now, completely done */
if (crp->cr_resid == 0) {
ret = usdf_pep_creq_epoll_del(crp);
- if (ret != 0) {
- usdf_cm_msg_connreq_failed(crp, ret);
- return 0;
- }
+ if (ret != 0)
+ goto report_failure_skip_data;
/* create CONNREQ EQ entry */
entry_len = sizeof(*entry) + reqp->creq_datalen;
entry = malloc(entry_len);
if (entry == NULL) {
- usdf_cm_msg_connreq_failed(crp, -errno);
- return 0;
+ ret = -errno;
+ goto report_failure_skip_data;
}
entry->fid = &pep->pep_fid.fid;
entry->info = usdf_pep_conn_info(crp);
if (entry->info == NULL) {
- free(entry);
- usdf_cm_msg_connreq_failed(crp, -FI_ENOMEM);
- return 0;
+ ret = -FI_ENOMEM;
+ goto free_entry_and_report_failure;
}
+
memcpy(entry->data, reqp->creq_data, reqp->creq_datalen);
ret = usdf_eq_write_internal(pep->pep_eq, FI_CONNREQ, entry,
entry_len, 0);
+
+ if (ret != (int)entry_len)
+ goto free_entry_and_report_failure;
+
free(entry);
- if (ret != (int)entry_len) {
- usdf_cm_msg_connreq_failed(crp, ret);
- return 0;
- }
}
return 0;
+
+free_entry_and_report_failure:
+ free(entry);
+report_failure_skip_data:
+ usdf_cm_report_failure(crp, ret, false);
+ return 0;
}
static int
@@ -266,8 +271,7 @@ usdf_pep_listen_cb(void *v)
ret = epoll_ctl(pep->pep_fabric->fab_epollfd, EPOLL_CTL_ADD,
crp->cr_sockfd, &ev);
if (ret == -1) {
- crp->cr_pollitem.pi_rtn = NULL;
- usdf_cm_msg_connreq_failed(crp, -errno);
+ usdf_cm_report_failure(crp, -errno, false);
return 0;
}
@@ -282,6 +286,7 @@ usdf_pep_listen(struct fid_pep *fpep)
struct usdf_pep *pep;
struct epoll_event ev;
struct usdf_fabric *fp;
+ socklen_t socklen;
int ret;
USDF_TRACE_SYS(EP_CTRL, "\n");
@@ -314,6 +319,15 @@ usdf_pep_listen(struct fid_pep *fpep)
if (ret == -1) {
return -errno;
}
+
+ /* Get the actual port (since we may have requested
+ * port 0)
+ */
+ socklen = sizeof(pep->pep_src_addr);
+ ret = getsockname(pep->pep_sock, &pep->pep_src_addr,
+ &socklen);
+ if (ret == -1)
+ return -errno;
pep->pep_state = USDF_PEP_BOUND;
}
@@ -335,6 +349,99 @@ usdf_pep_listen(struct fid_pep *fpep)
return 0;
}
+/* Register as a callback triggered by the socket becoming writeable. Write as
+ * much data as can be written in a single write, and keep track of how much
+ * data is left. If the data is not fully written, it will finish getting
+ * written in another iteration of the progression.
+ */
+static int usdf_pep_reject_async(void *vreq)
+{
+ struct usdf_connreq *crp;
+ int ret;
+
+ crp = vreq;
+
+ do {
+ ret = write(crp->cr_sockfd, crp->cr_ptr, crp->cr_resid);
+ } while ((ret < 0) && (errno == EINTR));
+
+ if ((ret <= 0) && (errno != EAGAIN)) {
+ USDF_DBG_SYS(EP_CTRL, "write failed: %s\n",
+ strerror(errno));
+ usdf_cm_report_failure(crp, -errno, false);
+ return -errno;
+ }
+
+ crp->cr_resid -= ret;
+ crp->cr_ptr += ret;
+
+ if (crp->cr_resid == 0)
+ usdf_cm_msg_connreq_cleanup(crp);
+
+ return FI_SUCCESS;
+}
+
+static int usdf_pep_reject(struct fid_pep *fpep, fid_t handle, const void *param,
+ size_t paramlen)
+{
+ struct usdf_pep *pep;
+ struct usdf_connreq *crp;
+ struct epoll_event event;
+ struct usdf_connreq_msg *reqp;
+ int ret;
+
+ if (paramlen > USDF_MAX_CONN_DATA) {
+ USDF_WARN_SYS(EP_CTRL,
+ "reject payload size %zu exceeds max %zu\n",
+ paramlen, USDF_MAX_CONN_DATA);
+ return -FI_EINVAL;
+ }
+
+ if (!fpep || !handle) {
+ USDF_WARN_SYS(EP_CTRL,
+ "handle and passive ep needed for reject\n");
+ return -FI_EINVAL;
+ }
+
+ if (!param && paramlen > 0) {
+ USDF_WARN_SYS(EP_CTRL,
+ "NULL data pointer with non-zero data length\n");
+ return -FI_EINVAL;
+ }
+
+ /* usdf_pep_conn_info stashed the pep pointer into the handle field of
+ * the info struct previously returned
+ */
+ crp = (struct usdf_connreq *) handle;
+ pep = pep_ftou(fpep);
+
+ crp->cr_ptr = crp->cr_data;
+ crp->cr_resid = sizeof(*reqp) + paramlen;
+
+ reqp = (struct usdf_connreq_msg *) crp->cr_data;
+
+ /* The result field is used on the remote end to detect whether the
+ * connection succeeded or failed.
+ */
+ reqp->creq_result = htonl(-FI_ECONNREFUSED);
+ reqp->creq_datalen = htonl(paramlen);
+ memcpy(reqp->creq_data, param, paramlen);
+
+ crp->cr_pollitem.pi_rtn = usdf_pep_reject_async;
+ crp->cr_pollitem.pi_context = crp;
+
+ event.events = EPOLLOUT;
+ event.data.ptr = &crp->cr_pollitem;
+
+ ret = epoll_ctl(pep->pep_fabric->fab_epollfd, EPOLL_CTL_ADD,
+ crp->cr_sockfd, &event);
+
+ if (ret)
+ return -errno;
+
+ return FI_SUCCESS;
+}
+
static void
usdf_pep_free_cr_lists(struct usdf_pep *pep)
{
@@ -496,8 +603,8 @@ struct fi_ops usdf_pep_ops = {
static struct fi_ops_ep usdf_pep_base_ops = {
.size = sizeof(struct fi_ops_ep),
.cancel = fi_no_cancel,
- .getopt = fi_no_getopt,
- .setopt = fi_no_setopt,
+ .getopt = usdf_ep_getopt_connected,
+ .setopt = usdf_ep_setopt,
.tx_ctx = fi_no_tx_ctx,
.rx_ctx = fi_no_rx_ctx,
.rx_size_left = fi_no_rx_size_left,
@@ -512,7 +619,7 @@ static struct fi_ops_cm usdf_pep_cm_ops = {
.connect = fi_no_connect,
.listen = usdf_pep_listen,
.accept = fi_no_accept,
- .reject = fi_no_reject,
+ .reject = usdf_pep_reject,
.shutdown = fi_no_shutdown,
};
@@ -581,16 +688,11 @@ usdf_pep_open(struct fid_fabric *fabric, struct fi_info *info,
ret = -errno;
goto fail;
}
- ret = fcntl(pep->pep_sock, F_GETFL, 0);
- if (ret == -1) {
- ret = -errno;
- goto fail;
- }
- ret = fcntl(pep->pep_sock, F_SETFL, ret | O_NONBLOCK);
- if (ret == -1) {
- ret = -errno;
- goto fail;
- }
+ ret = fi_fd_nonblock(pep->pep_sock);
+ if (ret) {
+ ret = -errno;
+ goto fail;
+ }
/* set SO_REUSEADDR to prevent annoying "Address already in use" errors
* on successive runs of programs listening on a well known port */
@@ -637,6 +739,8 @@ usdf_pep_open(struct fid_fabric *fabric, struct fi_info *info,
TAILQ_INIT(&pep->pep_cr_free);
TAILQ_INIT(&pep->pep_cr_pending);
pep->pep_backlog = 10;
+ pep->pep_cr_max_data = USDF_MAX_CONN_DATA;
+
ret = usdf_pep_grow_backlog(pep);
if (ret != 0) {
goto fail;
diff --git a/prov/usnic/src/usdf_progress.c b/prov/usnic/src/usdf_progress.c
index 257273f..5356f36 100644
--- a/prov/usnic/src/usdf_progress.c
+++ b/prov/usnic/src/usdf_progress.c
@@ -45,7 +45,6 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
diff --git a/prov/usnic/src/usdf_rdm.c b/prov/usnic/src/usdf_rdm.c
index 2c429e0..5c7f1ce 100644
--- a/prov/usnic/src/usdf_rdm.c
+++ b/prov/usnic/src/usdf_rdm.c
@@ -50,7 +50,6 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
@@ -404,7 +403,7 @@ usdf_rdm_rewind_qe(struct usdf_rdm_qe *qe, size_t rewind, size_t mtu)
}
qe->rd_cur_iov = cur_iov;
- qe->rd_cur_ptr = qe->rd_iov[cur_iov].iov_base +
+ qe->rd_cur_ptr = ((uint8_t *)qe->rd_iov[cur_iov].iov_base) +
qe->rd_iov[cur_iov].iov_len - cur_resid;
qe->rd_iov_resid = cur_resid;
}
@@ -534,7 +533,7 @@ static inline ssize_t _usdf_rdm_recv_vector(struct fid_ep *fep,
rqe->rd_last_iov = count - 1;
rqe->rd_cur_ptr = iov[0].iov_base;
rqe->rd_resid = tot_len;
- rqe->rd_length = tot_len;
+ rqe->rd_length = 0;
rqe->rd_signal_comp = ep->ep_rx_dflt_signal_comp ||
(flags & FI_COMPLETION) ? 1 : 0;
@@ -1145,7 +1144,8 @@ static inline void usdf_rdm_recv_complete(struct usdf_rx *rx,
USDF_DBG_SYS(EP_DATA, "RECV complete ID=%u len=%lu with status %d\n",
rdc->dc_rx_msg_id, rqe->rd_length, status);
hcq = rx->r.rdm.rx_hcq;
- hcq->cqh_post(hcq, rqe->rd_context, rqe->rd_length, status);
+ hcq->cqh_post(hcq, rqe->rd_context, rqe->rd_length, status,
+ FI_MSG | FI_RECV);
usdf_rdm_put_rx_rqe(rx, rqe);
@@ -1319,7 +1319,9 @@ usdf_rdm_process_ack(struct usdf_rdm_connection *rdc,
USDF_DBG_SYS(EP_DATA, "send ID=%u complete\n", msg_id);
if (wqe->rd_signal_comp)
hcq->cqh_post(hcq, wqe->rd_context,
- wqe->rd_length, FI_SUCCESS);
+ wqe->rd_length,
+ FI_SUCCESS,
+ FI_MSG | FI_SEND);
usdf_rdm_put_tx_wqe(tx, wqe);
@@ -1613,8 +1615,9 @@ ssize_t usdf_rdm_rx_size_left(struct fid_ep *fep)
ep = ep_ftou(fep);
rx = ep->ep_rx;
- if (rx == NULL)
- return -FI_EOPBADSTATE; /* EP not enabled */
+
+ if (!(ep->flags & USDF_EP_ENABLED))
+ return -FI_EOPBADSTATE;
return rx->r.rdm.rx_num_free_rqe;
}
@@ -1628,8 +1631,9 @@ ssize_t usdf_rdm_tx_size_left(struct fid_ep *fep)
ep = ep_ftou(fep);
tx = ep->ep_tx;
- if (tx == NULL)
- return -FI_EOPBADSTATE; /* EP not enabled */
+
+ if (!(ep->flags & USDF_EP_ENABLED))
+ return -FI_EOPBADSTATE;
return tx->t.rdm.tx_num_free_wqe;
}
diff --git a/prov/usnic/src/usdf_rdm.h b/prov/usnic/src/usdf_rdm.h
index e9d5e42..ab1dfd6 100644
--- a/prov/usnic/src/usdf_rdm.h
+++ b/prov/usnic/src/usdf_rdm.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2014-2016, Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -138,10 +138,10 @@ struct usdf_rdm_connection {
struct usdf_rdm_connection *dc_hash_next;
};
-
int usdf_rdm_fill_ep_attr(struct fi_info *hints, struct fi_info *fi,
struct usd_device_attrs *dap);
-int usdf_rdm_fill_dom_attr(struct fi_info *hints, struct fi_info *fi);
+int usdf_rdm_fill_dom_attr(uint32_t version, struct fi_info *hints,
+ struct fi_info *fi, struct usd_device_attrs *dap);
int usdf_rdm_fill_tx_attr(struct fi_info *hints, struct fi_info *fi);
int usdf_rdm_fill_rx_attr(struct fi_info *hints, struct fi_info *fi);
diff --git a/prov/usnic/src/usdf_socket.c b/prov/usnic/src/usdf_socket.c
deleted file mode 100644
index e3ea5de..0000000
--- a/prov/usnic/src/usdf_socket.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright (c) 2014, Cisco Systems, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include <errno.h>
-#include <stdint.h>
-#include <sys/socket.h>
-#include <sys/epoll.h>
-
-#include <rdma/fi_errno.h>
-
-#include "usdf_socket.h"
-
-int
-usdf_check_sock_error(int sock, uint32_t events)
-{
- socklen_t len;
- int status;
- int ret;
-
- if (events & EPOLLBAD) {
- return -FI_ECONNRESET;
- }
-
- len = sizeof(status);
- ret = getsockopt(sock, SOL_SOCKET, SO_ERROR, &status, &len);
- if (ret == -1) {
- return -errno;
- }
- if (status != 0) {
- return -status;
- }
- return 0;
-}
diff --git a/prov/usnic/src/usdf_socket.h b/prov/usnic/src/usdf_socket.h
deleted file mode 100644
index 790a0fe..0000000
--- a/prov/usnic/src/usdf_socket.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright (c) 2014, Cisco Systems, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-#ifndef _USDF_SOCKET_H_
-#define _USDF_SOCKET_H_
-
-#define EPOLLBAD (EPOLLHUP | EPOLLRDHUP | EPOLLERR)
-
-int usdf_check_sock_error(int sock, uint32_t events);
-
-#endif /* _USDF_SOCKET_H_ */
diff --git a/prov/usnic/src/usnic_direct/vnic_dev.c b/prov/usnic/src/usnic_direct/vnic_dev.c
index 4418d66..d4ff0a0 100644
--- a/prov/usnic/src/usnic_direct/vnic_dev.c
+++ b/prov/usnic/src/usnic_direct/vnic_dev.c
@@ -251,7 +251,11 @@ void vnic_dev_upd_res_vaddr(struct vnic_dev *vdev,
continue;
if (vdev->res[i].bus_addr >= map->bus_addr &&
vdev->res[i].bus_addr < map->bus_addr + map->len)
- vdev->res[i].vaddr = map->vaddr +
+ /*
+ * ptr cast is a libfabric-local modification, port to
+ * Cisco-internal repository is in progress
+ */
+ vdev->res[i].vaddr = ((uint8_t *)map->vaddr) +
(vdev->res[i].bus_addr - map->bus_addr);
}
}
diff --git a/prov/util/src/util_attr.c b/prov/util/src/util_attr.c
index acbf736..76015b0 100644
--- a/prov/util/src/util_attr.c
+++ b/prov/util/src/util_attr.c
@@ -30,8 +30,23 @@
* SOFTWARE.
*/
+#include <stdio.h>
+
#include <fi_util.h>
+#define FI_INFO_FIELD(provider, prov, user, prov_str, user_str, field, type) \
+ do { \
+ FI_INFO(provider, FI_LOG_CORE, prov_str ": %s\n", \
+ fi_tostr(&prov->field, type)); \
+ FI_INFO(provider, FI_LOG_CORE, user_str ": %s\n", \
+ fi_tostr(&user->field, type)); \
+ } while (0)
+
+#define FI_INFO_CAPS(provider, prov, user, field, type) \
+ FI_INFO_FIELD(provider, prov, user, "Supported", "Requested", field, type)
+
+#define FI_INFO_MODE(provider, prov, user) \
+ FI_INFO_FIELD(provider, prov, user, "Expected", "Given", mode, FI_TYPE_MODE)
static int fi_valid_addr_format(uint32_t prov_format, uint32_t user_format)
{
@@ -56,11 +71,186 @@ static int fi_valid_addr_format(uint32_t prov_format, uint32_t user_format)
}
}
+char *ofi_strdup_less_prefix(char *name, char *prefix)
+{
+ return strdup(name + strlen(prefix) + 1);
+}
+
+char *ofi_strdup_add_prefix(char *name, char *prefix)
+{
+ char *prefix_name;
+ char *base = "";
+ ssize_t size;
+ int ret;
+
+ if (name)
+ base = name;
+
+ size = snprintf(NULL, 0, "%s_%s", prefix, base) + 1;
+ if (size < 0)
+ return NULL;
+
+ prefix_name = calloc(size, sizeof(*prefix_name));
+ if (!prefix_name)
+ return NULL;
+
+ ret = snprintf(prefix_name, size, "%s_%s", prefix, base);
+ if (ret < 0 || ret > size)
+ goto err;
+
+ return prefix_name;
+
+err:
+ free(prefix_name);
+ return NULL;
+}
+
+static int ofix_dup_addr(struct fi_info *info, struct fi_info *dup)
+{
+ dup->addr_format = info->addr_format;
+ if (info->src_addr) {
+ dup->src_addrlen = info->src_addrlen;
+ dup->src_addr = mem_dup(info->src_addr, info->src_addrlen);
+ if (dup->src_addr == NULL)
+ return -FI_ENOMEM;
+ }
+ if (info->dest_addr) {
+ dup->dest_addrlen = info->dest_addrlen;
+ dup->dest_addr = mem_dup(info->dest_addr, info->dest_addrlen);
+ if (dup->dest_addr == NULL) {
+ free(dup->src_addr);
+ dup->src_addr = NULL;
+ return -FI_ENOMEM;
+ }
+ }
+ return 0;
+}
+
+static int ofix_alter_layer_info(const struct fi_provider *prov,
+ const struct fi_info *prov_info,
+ struct fi_info *layer_info, ofi_alter_info_t alter_layer_info,
+ struct fi_info **base_info)
+{
+ if (!(*base_info = fi_allocinfo()))
+ return -FI_ENOMEM;
+
+ if (alter_layer_info(layer_info, *base_info))
+ goto err;
+
+ if (!layer_info)
+ return 0;
+
+ if (ofix_dup_addr(layer_info, *base_info))
+ goto err;
+
+ if (layer_info->domain_attr && layer_info->domain_attr->name &&
+ !((*base_info)->domain_attr->name =
+ ofi_strdup_less_prefix(layer_info->domain_attr->name,
+ prov_info->domain_attr->name))) {
+ FI_WARN(prov, FI_LOG_FABRIC,
+ "Unable to alter layer_info domain name\n");
+ goto err;
+ }
+ return 0;
+err:
+ fi_freeinfo(*base_info);
+ return -FI_ENOMEM;
+}
+
+static int ofix_alter_base_info(const struct fi_provider *prov,
+ const struct fi_info *prov_info,
+ struct fi_info *base_info, ofi_alter_info_t alter_base_info,
+ struct fi_info **layer_info)
+{
+ if (!(*layer_info = fi_allocinfo()))
+ return -FI_ENOMEM;
+
+ if (alter_base_info(base_info, *layer_info))
+ goto err;
+
+ if (ofix_dup_addr(base_info, *layer_info))
+ goto err;
+
+ if (!((*layer_info)->domain_attr->name =
+ ofi_strdup_add_prefix(base_info->domain_attr->name,
+ prov_info->domain_attr->name))) {
+ FI_WARN(prov, FI_LOG_FABRIC,
+ "Unable to alter base prov domain name\n");
+ goto err;
+ }
+ (*layer_info)->fabric_attr->prov_version = prov->version;
+ if (!((*layer_info)->fabric_attr->name = strdup(base_info->fabric_attr->name)))
+ goto err;
+
+ return 0;
+err:
+ fi_freeinfo(*layer_info);
+ return -FI_ENOMEM;
+}
+
+int ofix_getinfo(uint32_t version, const char *node, const char *service,
+ uint64_t flags, const struct util_prov *util_prov,
+ struct fi_info *hints,
+ ofi_alter_info_t alter_layer_info,
+ ofi_alter_info_t alter_base_info,
+ int get_base_info, struct fi_info **info)
+{
+ struct fi_info *base_hints = NULL, *base_info;
+ struct fi_info *temp = NULL, *fi, *tail = NULL;
+ int ret;
+
+ ret = fi_check_info(util_prov, hints, FI_MATCH_PREFIX);
+ if (ret)
+ goto err1;
+
+ ret = ofix_alter_layer_info(util_prov->prov, util_prov->info, hints,
+ alter_layer_info, &base_hints);
+ if (ret)
+ goto err1;
+
+ ret = fi_getinfo(version, node, service, flags, base_hints, &base_info);
+ if (ret)
+ goto err2;
+
+ if (get_base_info) {
+ *info = base_info;
+ } else {
+ for (fi = base_info; fi; fi = fi->next) {
+ ret = ofix_alter_base_info(util_prov->prov, util_prov->info,
+ fi, alter_base_info, &temp);
+ if (ret)
+ goto err3;
+ if (!tail)
+ *info = temp;
+ else
+ tail->next = temp;
+ tail = temp;
+ }
+ fi_freeinfo(base_info);
+ }
+ fi_freeinfo(base_hints);
+ return 0;
+err3:
+ fi_freeinfo(*info);
+err2:
+ fi_freeinfo(base_hints);
+err1:
+ return -FI_ENODATA;
+}
+
+static int fi_check_name(char *user_name, char *prov_name, enum fi_match_type type)
+{
+ return (type == FI_MATCH_PREFIX) ?
+ strncasecmp(user_name, prov_name, strlen(prov_name)) :
+ strcasecmp(prov_name, user_name);
+}
+
int fi_check_fabric_attr(const struct fi_provider *prov,
const struct fi_fabric_attr *prov_attr,
- const struct fi_fabric_attr *user_attr)
+ const struct fi_fabric_attr *user_attr,
+ enum fi_match_type type)
{
- if (user_attr->name && strcasecmp(user_attr->name, prov_attr->name)) {
+ if (user_attr->name && fi_check_name(user_attr->name, prov_attr->name, type)) {
FI_INFO(prov, FI_LOG_CORE, "Unknown fabric name\n");
return -FI_ENODATA;
}
@@ -132,9 +322,10 @@ static int fi_resource_mgmt_level(enum fi_resource_mgmt rm_model)
int fi_check_domain_attr(const struct fi_provider *prov,
const struct fi_domain_attr *prov_attr,
- const struct fi_domain_attr *user_attr)
+ const struct fi_domain_attr *user_attr,
+ enum fi_match_type type)
{
- if (user_attr->name && strcasecmp(user_attr->name, prov_attr->name)) {
+ if (user_attr->name && fi_check_name(user_attr->name, prov_attr->name, type)) {
FI_INFO(prov, FI_LOG_CORE, "Unknown domain name\n");
return -FI_ENODATA;
}
@@ -183,17 +374,21 @@ int fi_check_domain_attr(const struct fi_provider *prov,
return 0;
}
-int fi_check_ep_attr(const struct fi_provider *prov,
- const struct fi_ep_attr *prov_attr,
+int fi_check_ep_attr(const struct util_prov *util_prov,
const struct fi_ep_attr *user_attr)
{
+ const struct fi_provider *prov = util_prov->prov;
+ const struct fi_ep_attr *prov_attr = util_prov->info->ep_attr;
+
if (user_attr->type && (user_attr->type != prov_attr->type)) {
FI_INFO(prov, FI_LOG_CORE, "Unsupported endpoint type\n");
+ FI_INFO_CAPS(prov, prov_attr, user_attr, type, FI_TYPE_EP_TYPE);
return -FI_ENODATA;
}
if (user_attr->protocol && (user_attr->protocol != prov_attr->protocol)) {
FI_INFO(prov, FI_LOG_CORE, "Unsupported protocol\n");
+ FI_INFO_CAPS(prov, prov_attr, user_attr, protocol, FI_TYPE_PROTOCOL);
return -FI_ENODATA;
}
@@ -208,6 +403,34 @@ int fi_check_ep_attr(const struct fi_provider *prov,
return -FI_ENODATA;
}
+ if (user_attr->tx_ctx_cnt > util_prov->info->domain_attr->max_ep_tx_ctx) {
+ if (user_attr->tx_ctx_cnt == FI_SHARED_CONTEXT) {
+ if (!(util_prov->flags & UTIL_TX_SHARED_CTX)) {
+ FI_INFO(prov, FI_LOG_CORE,
+ "Shared tx context not supported\n");
+ return -FI_ENODATA;
+ }
+ } else {
+ FI_INFO(prov, FI_LOG_CORE,
+ "Requested tx_ctx_cnt exceeds supported\n");
+ return -FI_ENODATA;
+ }
+ }
+
+ if (user_attr->rx_ctx_cnt > util_prov->info->domain_attr->max_ep_rx_ctx) {
+ if (user_attr->rx_ctx_cnt == FI_SHARED_CONTEXT) {
+ if (!(util_prov->flags & UTIL_RX_SHARED_CTX)) {
+ FI_INFO(prov, FI_LOG_CORE,
+ "Shared rx context not supported\n");
+ return -FI_ENODATA;
+ }
+ } else {
+ FI_INFO(prov, FI_LOG_CORE,
+ "Requested rx_ctx_cnt exceeds supported\n");
+ return -FI_ENODATA;
+ }
+ }
+
return 0;
}
@@ -217,26 +440,31 @@ int fi_check_rx_attr(const struct fi_provider *prov,
{
if (user_attr->caps & ~(prov_attr->caps)) {
FI_INFO(prov, FI_LOG_CORE, "caps not supported\n");
+ FI_INFO_CAPS(prov, prov_attr, user_attr, caps, FI_TYPE_CAPS);
return -FI_ENODATA;
}
if ((user_attr->mode & prov_attr->mode) != prov_attr->mode) {
FI_INFO(prov, FI_LOG_CORE, "needed mode not set\n");
+ FI_INFO_MODE(prov, prov_attr, user_attr);
return -FI_ENODATA;
}
if (prov_attr->op_flags & ~(prov_attr->op_flags)) {
FI_INFO(prov, FI_LOG_CORE, "op_flags not supported\n");
+ FI_INFO_CAPS(prov, prov_attr, user_attr, op_flags, FI_TYPE_OP_FLAGS);
return -FI_ENODATA;
}
if (user_attr->msg_order & ~(prov_attr->msg_order)) {
FI_INFO(prov, FI_LOG_CORE, "msg_order not supported\n");
+ FI_INFO_CAPS(prov, prov_attr, user_attr, msg_order, FI_TYPE_MSG_ORDER);
return -FI_ENODATA;
}
if (user_attr->comp_order & ~(prov_attr->comp_order)) {
FI_INFO(prov, FI_LOG_CORE, "comp_order not supported\n");
+ FI_INFO_CAPS(prov, prov_attr, user_attr, comp_order, FI_TYPE_MSG_ORDER);
return -FI_ENODATA;
}
@@ -264,26 +492,31 @@ int fi_check_tx_attr(const struct fi_provider *prov,
{
if (user_attr->caps & ~(prov_attr->caps)) {
FI_INFO(prov, FI_LOG_CORE, "caps not supported\n");
+ FI_INFO_CAPS(prov, prov_attr, user_attr, caps, FI_TYPE_CAPS);
return -FI_ENODATA;
}
if ((user_attr->mode & prov_attr->mode) != prov_attr->mode) {
FI_INFO(prov, FI_LOG_CORE, "needed mode not set\n");
+ FI_INFO_MODE(prov, prov_attr, user_attr);
return -FI_ENODATA;
}
if (prov_attr->op_flags & ~(prov_attr->op_flags)) {
FI_INFO(prov, FI_LOG_CORE, "op_flags not supported\n");
+ FI_INFO_CAPS(prov, prov_attr, user_attr, op_flags, FI_TYPE_OP_FLAGS);
return -FI_ENODATA;
}
if (user_attr->msg_order & ~(prov_attr->msg_order)) {
FI_INFO(prov, FI_LOG_CORE, "msg_order not supported\n");
+ FI_INFO_CAPS(prov, prov_attr, user_attr, msg_order, FI_TYPE_MSG_ORDER);
return -FI_ENODATA;
}
if (user_attr->comp_order & ~(prov_attr->comp_order)) {
FI_INFO(prov, FI_LOG_CORE, "comp_order not supported\n");
+ FI_INFO_CAPS(prov, prov_attr, user_attr, comp_order, FI_TYPE_MSG_ORDER);
return -FI_ENODATA;
}
@@ -310,10 +543,12 @@ int fi_check_tx_attr(const struct fi_provider *prov,
return 0;
}
-int fi_check_info(const struct fi_provider *prov,
- const struct fi_info *prov_info,
- const struct fi_info *user_info)
+int fi_check_info(const struct util_prov *util_prov,
+ const struct fi_info *user_info,
+ enum fi_match_type type)
{
+ const struct fi_info *prov_info = util_prov->info;
+ const struct fi_provider *prov = util_prov->prov;
int ret;
if (!user_info)
@@ -321,11 +556,13 @@ int fi_check_info(const struct fi_provider *prov,
if (user_info->caps & ~(prov_info->caps)) {
FI_INFO(prov, FI_LOG_CORE, "Unsupported capabilities\n");
+ FI_INFO_CAPS(prov, prov_info, user_info, caps, FI_TYPE_CAPS);
return -FI_ENODATA;
}
if ((user_info->mode & prov_info->mode) != prov_info->mode) {
FI_INFO(prov, FI_LOG_CORE, "needed mode not set\n");
+ FI_INFO_MODE(prov, prov_info, user_info);
return -FI_ENODATA;
}
@@ -337,35 +574,36 @@ int fi_check_info(const struct fi_provider *prov,
if (user_info->fabric_attr) {
ret = fi_check_fabric_attr(prov, prov_info->fabric_attr,
- user_info->fabric_attr);
+ user_info->fabric_attr,
+ type);
if (ret)
return ret;
}
if (user_info->domain_attr) {
ret = fi_check_domain_attr(prov, prov_info->domain_attr,
- user_info->domain_attr);
+ user_info->domain_attr,
+ type);
if (ret)
return ret;
}
if (user_info->ep_attr) {
- ret = fi_check_ep_attr(prov, prov_info->ep_attr,
- user_info->ep_attr);
+ ret = fi_check_ep_attr(util_prov, user_info->ep_attr);
if (ret)
return ret;
}
if (user_info->rx_attr) {
ret = fi_check_rx_attr(prov, prov_info->rx_attr,
- user_info->rx_attr);
+ user_info->rx_attr);
if (ret)
return ret;
}
if (user_info->tx_attr) {
ret = fi_check_tx_attr(prov, prov_info->tx_attr,
- user_info->tx_attr);
+ user_info->tx_attr);
if (ret)
return ret;
}
@@ -433,16 +671,18 @@ static void fi_alter_tx_attr(struct fi_tx_attr *attr,
* the hints have been validated and the starting fi_info is properly
* configured by the provider.
*/
-void fi_alter_info(struct fi_info *info,
+void ofi_alter_info(struct fi_info *info,
const struct fi_info *hints)
{
if (!hints)
return;
- info->caps = (hints->caps & FI_PRIMARY_CAPS) |
- (info->caps & FI_SECONDARY_CAPS);
+ for (; info; info = info->next) {
+ info->caps = (hints->caps & FI_PRIMARY_CAPS) |
+ (info->caps & FI_SECONDARY_CAPS);
- fi_alter_ep_attr(info->ep_attr, hints->ep_attr);
- fi_alter_rx_attr(info->rx_attr, hints->rx_attr, info->caps);
- fi_alter_tx_attr(info->tx_attr, hints->tx_attr, info->caps);
+ fi_alter_ep_attr(info->ep_attr, hints->ep_attr);
+ fi_alter_rx_attr(info->rx_attr, hints->rx_attr, info->caps);
+ fi_alter_tx_attr(info->tx_attr, hints->tx_attr, info->caps);
+ }
}
diff --git a/prov/util/src/util_av.c b/prov/util/src/util_av.c
index 1f232a3..8972858 100644
--- a/prov/util/src/util_av.c
+++ b/prov/util/src/util_av.c
@@ -30,6 +30,8 @@
* SOFTWARE.
*/
+#include "config.h"
+
#include <arpa/inet.h>
#include <ctype.h>
#include <stdlib.h>
@@ -39,6 +41,11 @@
#include <netdb.h>
#include <netinet/in.h>
+#if HAVE_GETIFADDRS
+#include <net/if.h>
+#include <ifaddrs.h>
+#endif
+
#include <fi_util.h>
@@ -94,7 +101,46 @@ out:
}
-int fi_get_src_addr(uint32_t addr_format,
+void ofi_getnodename(char *buf, int buflen)
+{
+ int ret;
+ struct addrinfo ai, *rai = NULL;
+ struct ifaddrs *ifaddrs, *ifa;
+
+ ret = gethostname(buf, buflen);
+ if (ret == 0) {
+ memset(&ai, 0, sizeof(ai));
+ ai.ai_family = AF_INET;
+ ret = getaddrinfo(buf, NULL, &ai, &rai);
+ if (!ret) {
+ freeaddrinfo(rai);
+ return;
+ }
+ }
+
+#if HAVE_GETIFADDRS
+ ret = getifaddrs(&ifaddrs);
+ if (!ret) {
+ for (ifa = ifaddrs; ifa != NULL; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr == NULL || !(ifa->ifa_flags & IFF_UP) ||
+ (ifa->ifa_addr->sa_family != AF_INET))
+ continue;
+
+ ret = getnameinfo(ifa->ifa_addr, sizeof(struct sockaddr_in),
+ buf, buflen, NULL, 0, NI_NUMERICHOST);
+ if (ret == 0) {
+ freeifaddrs(ifaddrs);
+ return;
+ }
+ }
+ freeifaddrs(ifaddrs);
+ }
+#endif
+ /* no reasonable address found, try loopback */
+ strncpy(buf, "127.0.0.1", buflen);
+}
+
+int ofi_get_src_addr(uint32_t addr_format,
const void *dest_addr, size_t dest_addrlen,
void **src_addr, size_t *src_addrlen)
{
@@ -139,7 +185,7 @@ out:
return ret;
}
-int fi_get_addr(uint32_t addr_format, uint64_t flags,
+int ofi_get_addr(uint32_t addr_format, uint64_t flags,
const char *node, const char *service,
void **addr, size_t *addrlen)
{
@@ -163,7 +209,7 @@ static void *util_av_get_data(struct util_av *av, int index)
return (char *) av->data + (index * av->addrlen);
}
-void *fi_av_get_addr(struct util_av *av, int index)
+void *ofi_av_get_addr(struct util_av *av, int index)
{
return util_av_get_data(av, index);
}
@@ -219,8 +265,7 @@ static int util_av_hash_insert(struct util_av_hash *hash, int slot, int index)
return 0;
}
-static int fi_av_insert_addr(struct util_av *av, const void *addr, int slot,
- int *index)
+int ofi_av_insert_addr(struct util_av *av, const void *addr, int slot, int *index)
{
int ret = 0;
@@ -307,7 +352,7 @@ static int fi_av_remove_addr(struct util_av *av, int slot, int index)
return 0;
}
-static int fi_av_lookup_index(struct util_av *av, const void *addr, int slot)
+int ofi_av_lookup_index(struct util_av *av, const void *addr, int slot)
{
int i, ret = -FI_ENODATA;
@@ -323,7 +368,7 @@ static int fi_av_lookup_index(struct util_av *av, const void *addr, int slot)
}
for (i = slot; i != UTIL_NO_ENTRY; i = av->hash.table[i].next) {
- if (!memcmp(fi_av_get_addr(av, av->hash.table[i].index), addr,
+ if (!memcmp(ofi_av_get_addr(av, av->hash.table[i].index), addr,
av->addrlen)) {
ret = av->hash.table[i].index;
FI_DBG(av->prov, FI_LOG_AV, "entry at index (%d)\n", ret);
@@ -336,7 +381,7 @@ out:
return ret;
}
-static int util_av_bind(struct fid *av_fid, struct fid *eq_fid, uint64_t flags)
+int ofi_av_bind(struct fid *av_fid, struct fid *eq_fid, uint64_t flags)
{
struct util_av *av;
struct util_eq *eq;
@@ -358,11 +403,8 @@ static int util_av_bind(struct fid *av_fid, struct fid *eq_fid, uint64_t flags)
return 0;
}
-static int fi_av_free(struct fid *av_fid)
+int ofi_av_close(struct util_av *av)
{
- struct util_av *av;
-
- av = container_of(av_fid, struct util_av, av_fid.fid);
if (atomic_get(&av->ref)) {
FI_WARN(av->prov, FI_LOG_AV, "AV is busy\n");
return -FI_EBUSY;
@@ -375,7 +417,6 @@ static int fi_av_free(struct fid *av_fid)
fastlock_destroy(&av->lock);
/* TODO: unmap data? */
free(av->data);
- free(av);
return 0;
}
@@ -476,27 +517,20 @@ static int util_verify_av_attr(struct util_domain *domain,
return 0;
}
-int fi_av_create(struct util_domain *domain, const struct fi_av_attr *attr,
- const struct util_av_attr *util_attr,
- struct fid_av **av_fid, void *context)
+int ofi_av_init(struct util_domain *domain, const struct fi_av_attr *attr,
+ const struct util_av_attr *util_attr,
+ struct util_av *av, void *context)
{
- struct util_av *av;
int ret;
ret = util_verify_av_attr(domain, attr, util_attr);
if (ret)
return ret;
- av = calloc(1, sizeof(*av));
- if (!av)
- return -FI_ENOMEM;
-
av->prov = domain->prov;
ret = util_av_init(av, attr, util_attr);
- if (ret) {
- free(av);
+ if (ret)
return ret;
- }
av->av_fid.fid.fclass = FI_CLASS_AV;
/*
@@ -505,11 +539,8 @@ int fi_av_create(struct util_domain *domain, const struct fi_av_attr *attr,
* av->av_fid.ops = &prov_av_ops;
*/
av->context = context;
-
av->domain = domain;
atomic_inc(&domain->ref);
-
- *av_fid = &av->av_fid;
return 0;
}
@@ -522,7 +553,7 @@ int fi_av_create(struct util_domain *domain, const struct fi_av_attr *attr,
static int ip_av_slot(struct util_av *av, const struct sockaddr *sa)
{
- uint16_t host;
+ uint32_t host;
uint16_t port;
if (!sa)
@@ -552,11 +583,11 @@ static int ip_av_slot(struct util_av *av, const struct sockaddr *sa)
int ip_av_get_index(struct util_av *av, const void *addr)
{
- return fi_av_lookup_index(av, addr, ip_av_slot(av, addr));
+ return ofi_av_lookup_index(av, addr, ip_av_slot(av, addr));
}
-static void ip_av_write_event(struct util_av *av, uint64_t data,
- int err, void *context)
+void ofi_av_write_event(struct util_av *av, uint64_t data,
+ int err, void *context)
{
struct fi_eq_err_entry entry;
size_t size;
@@ -606,7 +637,7 @@ static int ip_av_insert_addr(struct util_av *av, const void *addr,
int ret, index = -1;
if (ip_av_valid_addr(av, addr)) {
- ret = fi_av_insert_addr(av, addr, ip_av_slot(av, addr), &index);
+ ret = ofi_av_insert_addr(av, addr, ip_av_slot(av, addr), &index);
} else {
ret = -FI_EADDRNOTAVAIL;
FI_WARN(av->prov, FI_LOG_AV, "invalid address\n");
@@ -638,12 +669,12 @@ static int ip_av_insert(struct fid_av *av_fid, const void *addr, size_t count,
if (!ret)
success_cnt++;
else if (av->eq)
- ip_av_write_event(av, i, -ret, context);
+ ofi_av_write_event(av, i, -ret, context);
}
FI_DBG(av->prov, FI_LOG_AV, "%d addresses successful\n", success_cnt);
if (av->eq) {
- ip_av_write_event(av, success_cnt, 0, context);
+ ofi_av_write_event(av, success_cnt, 0, context);
ret = 0;
} else {
ret = success_cnt;
@@ -711,7 +742,7 @@ static int ip_av_insert_ip4sym(struct util_av *av,
if (!ret)
success_cnt++;
else if (av->eq)
- ip_av_write_event(av, fi, -ret, context);
+ ofi_av_write_event(av, fi, -ret, context);
}
}
@@ -738,7 +769,7 @@ static int ip_av_insert_ip6sym(struct util_av *av,
if (!ret)
success_cnt++;
else if (av->eq)
- ip_av_write_event(av, fi, -ret, context);
+ ofi_av_write_event(av, fi, -ret, context);
}
/* TODO: should we skip addresses x::0 and x::255? */
@@ -791,7 +822,7 @@ static int ip_av_insert_nodesym(struct util_av *av,
if (!ret)
success_cnt++;
else if (av->eq)
- ip_av_write_event(av, fi, -ret, context);
+ ofi_av_write_event(av, fi, -ret, context);
}
}
@@ -842,7 +873,7 @@ static int ip_av_insertsym(struct fid_av *av_fid, const char *node, size_t nodec
out:
if (av->eq) {
- ip_av_write_event(av, ret, 0, context);
+ ofi_av_write_event(av, ret, 0, context);
ret = 0;
}
return ret;
@@ -925,10 +956,22 @@ static struct fi_ops_av ip_av_ops = {
.straddr = ip_av_straddr,
};
+static int ip_av_close(struct fid *av_fid)
+{
+ int ret;
+ struct util_av *av;
+ av = container_of(av_fid, struct util_av, av_fid.fid);
+ ret = ofi_av_close(av);
+ if (ret)
+ return ret;
+ free(av);
+ return 0;
+}
+
static struct fi_ops ip_av_fi_ops = {
.size = sizeof(struct fi_ops),
- .close = fi_av_free,
- .bind = util_av_bind,
+ .close = ip_av_close,
+ .bind = ofi_av_bind,
.control = fi_no_control,
.ops_open = fi_no_ops_open,
};
@@ -938,6 +981,7 @@ int ip_av_create(struct fid_domain *domain_fid, struct fi_av_attr *attr,
{
struct util_domain *domain;
struct util_av_attr util_attr;
+ struct util_av *util_av;
int ret;
domain = container_of(domain_fid, struct util_domain, domain_fid);
@@ -946,17 +990,213 @@ int ip_av_create(struct fid_domain *domain_fid, struct fi_av_attr *attr,
else
util_attr.addrlen = sizeof(struct sockaddr_in6);
- util_attr.overhead = attr->count >> 2;
+ util_attr.overhead = attr->count >> 1;
util_attr.flags = domain->caps & FI_SOURCE ? FI_SOURCE : 0;
if (attr->type == FI_AV_UNSPEC)
attr->type = FI_AV_MAP;
- ret = fi_av_create(domain, attr, &util_attr, av, context);
- if (ret)
+ util_av = calloc(1, sizeof(*util_av));
+ if (!util_av)
+ return -FI_ENOMEM;
+
+ ret = ofi_av_init(domain, attr, &util_attr, util_av, context);
+ if (ret) {
+ free(util_av);
return ret;
+ }
+ *av = &util_av->av_fid;
(*av)->fid.ops = &ip_av_fi_ops;
(*av)->ops = &ip_av_ops;
return 0;
}
+
+/*
+ * Connection Map
+ */
+static void ofi_cmap_init_handle(struct util_cmap_handle *handle,
+ struct util_cmap *cmap,
+ enum util_cmap_state state,
+ fi_addr_t fi_addr,
+ struct util_cmap_peer *peer)
+{
+ handle->cmap = cmap;
+ handle->state = state;
+ handle->key = freestack_pop(cmap->keypool);
+ handle->key->handle = handle;
+ handle->key_index = util_cmap_keypool_index(cmap->keypool, handle->key);
+ handle->fi_addr = fi_addr;
+ handle->peer = peer;
+}
+
+void ofi_cmap_update_state(struct util_cmap_handle *handle,
+ enum util_cmap_state state)
+{
+ fastlock_acquire(&handle->cmap->lock);
+ handle->state = state;
+ fastlock_release(&handle->cmap->lock);
+}
+
+static int ofi_cmap_match_peer(struct dlist_entry *entry, const void *addr)
+{
+ struct util_cmap_peer *peer;
+
+ peer = container_of(entry, struct util_cmap_peer, entry);
+ return !memcmp(peer->addr, addr, peer->addrlen);
+}
+
+static int ofi_cmap_add_peer(struct util_cmap *cmap, struct util_cmap_handle *handle,
+ enum util_cmap_state state, void *addr, size_t addrlen)
+{
+ struct util_cmap_peer *peer;
+ int ret = 0;
+
+ fastlock_acquire(&cmap->lock);
+ if (dlist_find_first_match(&cmap->peer_list, ofi_cmap_match_peer, addr)) {
+ FI_WARN(cmap->av->prov, FI_LOG_EP_CTRL,
+ "Peer already present\n");
+ goto out;
+ }
+
+ // TODO Use util_buf_pool
+ peer = calloc(1, sizeof(*peer) + addrlen);
+ if (!peer) {
+ ret = -FI_ENOMEM;
+ goto out;
+ }
+
+ ofi_cmap_init_handle(handle, cmap, state, FI_ADDR_UNSPEC, peer);
+ peer->handle = handle;
+ peer->addrlen = addrlen;
+ memcpy(peer->addr, addr, addrlen);
+ dlist_insert_tail(&peer->entry, &cmap->peer_list);
+out:
+ fastlock_release(&cmap->lock);
+ return ret;
+}
+
+/*
+ * Caller must hold cmap->lock. Either fi_addr or
+ * addr and addrlen args should be present.
+ */
+int ofi_cmap_add_handle(struct util_cmap *cmap, struct util_cmap_handle *handle,
+ enum util_cmap_state state, fi_addr_t fi_addr, void *addr,
+ size_t addrlen)
+{
+ int index;
+ if (fi_addr == FI_ADDR_UNSPEC) {
+ index = ip_av_get_index(cmap->av, addr);
+ if (index < 0)
+ return ofi_cmap_add_peer(cmap, handle, state, addr, addrlen);
+ fi_addr = index;
+ }
+
+ if (cmap->handles[fi_addr]) {
+ FI_WARN(cmap->av->prov, FI_LOG_EP_CTRL,
+ "Handle already present\n");
+ } else {
+ ofi_cmap_init_handle(handle, cmap, state, fi_addr, NULL);
+ cmap->handles[fi_addr] = handle;
+ }
+ return 0;
+}
+
+/* Caller must hold cmap->lock */
+struct util_cmap_handle *ofi_cmap_get_handle(struct util_cmap *cmap, fi_addr_t fi_addr)
+{
+ struct util_cmap_peer *peer;
+ struct dlist_entry *entry;
+
+ if (cmap->handles[fi_addr])
+ return cmap->handles[fi_addr];
+
+ /* Search in peer list */
+ entry = dlist_remove_first_match(&cmap->peer_list, ofi_cmap_match_peer,
+ ip_av_get_addr(cmap->av, fi_addr));
+ if (!entry)
+ return NULL;
+ peer = container_of(entry, struct util_cmap_peer, entry);
+
+ /* Move handle to cmap */
+ peer->handle->peer = NULL;
+ peer->handle->fi_addr = fi_addr;
+
+ cmap->handles[fi_addr] = peer->handle;
+ free(peer);
+ return cmap->handles[fi_addr];
+}
+
+void ofi_cmap_del_handle(struct util_cmap_handle *handle)
+{
+ struct util_cmap *cmap = handle->cmap;
+
+ fastlock_acquire(&cmap->lock);
+ if (handle->peer) {
+ dlist_remove(&handle->peer->entry);
+ free(handle->peer);
+ } else {
+ cmap->handles[handle->fi_addr] = 0;
+ }
+ handle->key->handle = NULL;
+ freestack_push(cmap->keypool, handle->key);
+ cmap->free_handle(handle);
+ fastlock_release(&cmap->lock);
+}
+
+void ofi_cmap_del_handles(struct util_cmap *cmap)
+{
+ struct util_cmap_peer *peer;
+ struct dlist_entry *entry;
+ int i;
+
+ for (i = 0; i < cmap->av->count; i++) {
+ if (cmap->handles[i])
+ ofi_cmap_del_handle(cmap->handles[i]);
+ }
+ dlist_foreach(&cmap->peer_list, entry) {
+ peer = container_of(entry, struct util_cmap_peer, entry);
+ ofi_cmap_del_handle(peer->handle);
+ }
+}
+
+void ofi_cmap_free(struct util_cmap *cmap)
+{
+ ofi_cmap_del_handles(cmap);
+ fastlock_acquire(&cmap->lock);
+ util_cmap_keypool_free(cmap->keypool);
+ free(cmap->handles);
+ fastlock_release(&cmap->lock);
+ free(cmap);
+}
+
+struct util_cmap *ofi_cmap_alloc(struct util_av *av,
+ ofi_cmap_free_handle_func free_handle)
+{
+ struct util_cmap *cmap;
+
+ cmap = calloc(1, sizeof *cmap);
+ if (!cmap)
+ return NULL;
+
+ cmap->av = av;
+
+ cmap->handles = calloc(cmap->av->count, sizeof(*cmap->handles));
+ if (!cmap->handles)
+ goto err1;
+
+ cmap->keypool = util_cmap_keypool_create(cmap->av->count);
+ if (!cmap->keypool)
+ goto err2;
+
+ dlist_init(&cmap->peer_list);
+ cmap->free_handle = free_handle;
+ fastlock_init(&cmap->lock);
+
+ return cmap;
+err2:
+ free(cmap->handles);
+err1:
+ free(cmap);
+ return NULL;
+}
diff --git a/prov/util/src/util_buf.c b/prov/util/src/util_buf.c
index 47eb20a..204991b 100644
--- a/prov/util/src/util_buf.c
+++ b/prov/util/src/util_buf.c
@@ -36,6 +36,7 @@
#include <fi_enosys.h>
#include <fi_mem.h>
#include <fi.h>
+#include <fi_osd.h>
#if ENABLE_DEBUG
static inline int util_buf_use_ftr(struct util_buf_pool *pool)
@@ -75,7 +76,7 @@ int util_buf_grow(struct util_buf_pool *pool)
if (!buf_region)
return -1;
- ret = posix_memalign((void **)&buf_region->mem_region, pool->alignment,
+ ret = ofi_memalign((void **)&buf_region->mem_region, pool->alignment,
pool->chunk_cnt * pool->entry_sz);
if (ret)
goto err;
@@ -174,7 +175,7 @@ void util_buf_pool_destroy(struct util_buf_pool *pool)
#endif
if (pool->free_hndlr)
pool->free_hndlr(pool->ctx, buf_region->context);
- free(buf_region->mem_region);
+ ofi_freealign(buf_region->mem_region);
free(buf_region);
}
free(pool);
diff --git a/prov/util/src/util_cq.c b/prov/util/src/util_cq.c
index cd78056..816fc41 100644
--- a/prov/util/src/util_cq.c
+++ b/prov/util/src/util_cq.c
@@ -36,6 +36,7 @@
#include <fi_enosys.h>
#include <fi_util.h>
+#define UTIL_DEF_CQ_SIZE (1024)
int fi_check_cq_attr(const struct fi_provider *prov,
const struct fi_cq_attr *attr)
@@ -77,71 +78,53 @@ int fi_check_cq_attr(const struct fi_provider *prov,
return -FI_EINVAL;
}
- if (attr->flags) {
+ if (attr->flags & ~(FI_AFFINITY)) {
FI_WARN(prov, FI_LOG_CQ, "invalid flags\n");
return -FI_EINVAL;
}
- if (attr->signaling_vector) {
- FI_WARN(prov, FI_LOG_CQ, "signaling vectors not supported\n");
- return -FI_ENOSYS;
+ if (attr->flags & FI_AFFINITY) {
+ FI_WARN(prov, FI_LOG_CQ, "signaling vector ignored\n");
}
return 0;
}
-static void util_cq_progress(struct util_cq *cq)
-{
- struct util_ep *ep;
- struct fid_list_entry *fid_entry;
- struct dlist_entry *item;
-
- fastlock_acquire(&cq->list_lock);
- dlist_foreach(&cq->list, item) {
- fid_entry = container_of(item, struct fid_list_entry, entry);
- ep = container_of(fid_entry->fid, struct util_ep, ep_fid.fid);
- ep->progress(ep);
-
- }
- fastlock_release(&cq->list_lock);
-}
-
static void util_cq_read_ctx(void **dst, void *src)
{
*(struct fi_cq_entry *) *dst = *(struct fi_cq_entry *) src;
- *dst += sizeof(struct fi_cq_entry);
+ *(char**)dst += sizeof(struct fi_cq_entry);
}
static void util_cq_read_msg(void **dst, void *src)
{
*(struct fi_cq_msg_entry *) *dst = *(struct fi_cq_msg_entry *) src;
- *dst += sizeof(struct fi_cq_msg_entry);
+ *(char**)dst += sizeof(struct fi_cq_msg_entry);
}
static void util_cq_read_data(void **dst, void *src)
{
*(struct fi_cq_data_entry *) *dst = *(struct fi_cq_data_entry *) src;
- *dst += sizeof(struct fi_cq_data_entry);
+ *(char**)dst += sizeof(struct fi_cq_data_entry);
}
static void util_cq_read_tagged(void **dst, void *src)
{
- util_cq_read_data(dst, src);
- ((struct fi_cq_tagged_entry *) *dst)->tag = 0;
- *dst += sizeof(struct fi_cq_tagged_entry);
+ *(struct fi_cq_tagged_entry *) *dst = *(struct fi_cq_tagged_entry *) src;
+ *(char **)dst += sizeof(struct fi_cq_tagged_entry);
}
static ssize_t util_cq_read(struct fid_cq *cq_fid, void *buf, size_t count)
{
struct util_cq *cq;
- struct fi_cq_data_entry *entry;
+ struct fi_cq_tagged_entry *entry;
ssize_t i;
cq = container_of(cq_fid, struct util_cq, cq_fid);
fastlock_acquire(&cq->cq_lock);
if (cirque_isempty(cq->cirq)) {
fastlock_release(&cq->cq_lock);
- util_cq_progress(cq);
+ cq->progress(cq);
fastlock_acquire(&cq->cq_lock);
if (cirque_isempty(cq->cirq)) {
i = -FI_EAGAIN;
@@ -171,7 +154,7 @@ static ssize_t util_cq_readfrom(struct fid_cq *cq_fid, void *buf,
size_t count, fi_addr_t *src_addr)
{
struct util_cq *cq;
- struct fi_cq_data_entry *entry;
+ struct fi_cq_tagged_entry *entry;
ssize_t i;
cq = container_of(cq_fid, struct util_cq, cq_fid);
@@ -187,7 +170,7 @@ static ssize_t util_cq_readfrom(struct fid_cq *cq_fid, void *buf,
fastlock_acquire(&cq->cq_lock);
if (cirque_isempty(cq->cirq)) {
fastlock_release(&cq->cq_lock);
- util_cq_progress(cq);
+ cq->progress(cq);
fastlock_acquire(&cq->cq_lock);
if (cirque_isempty(cq->cirq)) {
i = -FI_EAGAIN;
@@ -289,7 +272,7 @@ static struct fi_ops_cq util_cq_ops = {
.strerror = util_cq_strerror,
};
-static int fi_cq_cleanup(struct util_cq *cq)
+int ofi_cq_cleanup(struct util_cq *cq)
{
struct util_cq_err_entry *err;
struct slist_entry *entry;
@@ -314,6 +297,8 @@ static int fi_cq_cleanup(struct util_cq *cq)
}
atomic_dec(&cq->domain->ref);
+ util_comp_cirq_free(cq->cirq);
+ free(cq->src);
return 0;
}
@@ -323,13 +308,9 @@ static int util_cq_close(struct fid *fid)
int ret;
cq = container_of(fid, struct util_cq, cq_fid.fid);
- ret = fi_cq_cleanup(cq);
+ ret = ofi_cq_cleanup(cq);
if (ret)
return ret;
-
- util_comp_cirq_free(cq->cirq);
- free(cq->src);
- free(cq);
return 0;
}
@@ -390,12 +371,38 @@ static int fi_cq_init(struct fid_domain *domain, struct fi_cq_attr *attr,
return 0;
}
-static int util_cq_init(struct fid_domain *domain, struct fi_cq_attr *attr,
- struct util_cq *cq, void *context)
+void ofi_cq_progress(struct util_cq *cq)
+{
+ struct util_ep *ep;
+ struct fid_list_entry *fid_entry;
+ struct dlist_entry *item;
+
+ fastlock_acquire(&cq->list_lock);
+ dlist_foreach(&cq->list, item) {
+ fid_entry = container_of(item, struct fid_list_entry, entry);
+ ep = container_of(fid_entry->fid, struct util_ep, ep_fid.fid);
+ ep->progress(ep);
+
+ }
+ fastlock_release(&cq->list_lock);
+}
+
+int ofi_cq_init(const struct fi_provider *prov, struct fid_domain *domain,
+ struct fi_cq_attr *attr, struct util_cq *cq,
+ fi_cq_progress_func progress, void *context)
{
fi_cq_read_func read_func;
int ret;
+ assert(progress);
+ ret = fi_check_cq_attr(prov, attr);
+ if (ret)
+ return ret;
+
+ cq->cq_fid.fid.ops = &util_cq_fi_ops;
+ cq->cq_fid.ops = &util_cq_ops;
+ cq->progress = progress;
+
switch (attr->format) {
case FI_CQ_FORMAT_UNSPEC:
case FI_CQ_FORMAT_CONTEXT:
@@ -419,7 +426,17 @@ static int util_cq_init(struct fid_domain *domain, struct fi_cq_attr *attr,
if (ret)
return ret;
- cq->cirq = util_comp_cirq_create(attr->size);
+ /* CQ must be fully operational before adding to wait set */
+ if (cq->wait) {
+ ret = fi_poll_add(&cq->wait->pollset->poll_fid,
+ &cq->cq_fid.fid, 0);
+ if (ret) {
+ ofi_cq_cleanup(cq);
+ return ret;
+ }
+ }
+
+ cq->cirq = util_comp_cirq_create(attr->size == 0 ? UTIL_DEF_CQ_SIZE : attr->size);
if (!cq->cirq) {
ret = -FI_ENOMEM;
goto err1;
@@ -437,44 +454,6 @@ static int util_cq_init(struct fid_domain *domain, struct fi_cq_attr *attr,
err2:
util_comp_cirq_free(cq->cirq);
err1:
- fi_cq_cleanup(cq);
+ ofi_cq_cleanup(cq);
return ret;
}
-
-int util_cq_open(const struct fi_provider *prov,
- struct fid_domain *domain, struct fi_cq_attr *attr,
- struct fid_cq **cq_fid, void *context)
-{
- struct util_cq *cq;
- int ret;
-
- ret = fi_check_cq_attr(prov, attr);
- if (ret)
- return ret;
-
- cq = calloc(1, sizeof(*cq));
- if (!cq)
- return -FI_ENOMEM;
-
- ret = util_cq_init(domain, attr, cq, context);
- if (ret) {
- free(cq);
- return ret;
- }
-
- cq->cq_fid.fid.ops = &util_cq_fi_ops;
- cq->cq_fid.ops = &util_cq_ops;
-
- /* CQ must be fully operational before adding to wait set */
- if (cq->wait) {
- ret = fi_poll_add(&cq->wait->pollset->poll_fid,
- &cq->cq_fid.fid, 0);
- if (ret) {
- util_cq_close(&cq->cq_fid.fid);
- return ret;
- }
- }
-
- *cq_fid = &cq->cq_fid;
- return 0;
-}
diff --git a/prov/util/src/util_domain.c b/prov/util/src/util_domain.c
index 4d718e3..4c1f9d4 100644
--- a/prov/util/src/util_domain.c
+++ b/prov/util/src/util_domain.c
@@ -37,7 +37,7 @@
#include <fi_util.h>
-int util_domain_close(struct util_domain *domain)
+int ofi_domain_close(struct util_domain *domain)
{
if (atomic_get(&domain->ref))
return -FI_EBUSY;
@@ -46,6 +46,7 @@ int util_domain_close(struct util_domain *domain)
dlist_remove(&domain->list_entry);
fastlock_release(&domain->fabric->lock);
+ free(domain->name);
fastlock_destroy(&domain->lock);
atomic_dec(&domain->fabric->ref);
return 0;
@@ -71,7 +72,7 @@ static int util_domain_init(struct util_domain *domain,
return domain->name ? 0 : -FI_ENOMEM;
}
-int fi_domain_init(struct fid_fabric *fabric_fid, const struct fi_info *info,
+int ofi_domain_init(struct fid_fabric *fabric_fid, const struct fi_info *info,
struct util_domain *domain, void *context)
{
struct util_fabric *fabric;
diff --git a/prov/sockets/include/rdma/fi_direct_cm.h b/prov/util/src/util_ep.c
similarity index 64%
rename from prov/sockets/include/rdma/fi_direct_cm.h
rename to prov/util/src/util_ep.c
index 971eba0..b6d0f2b 100644
--- a/prov/sockets/include/rdma/fi_direct_cm.h
+++ b/prov/util/src/util_ep.c
@@ -30,4 +30,37 @@
* SOFTWARE.
*/
-/* Do not remove this file. See fi_direct.h */
+#include <stdlib.h>
+#include <string.h>
+
+#include <fi_enosys.h>
+#include <fi_util.h>
+
+int ofi_endpoint_init(struct fid_domain *domain, const struct util_prov *util_prov,
+ struct fi_info *info, struct util_ep *ep, void *context,
+ enum fi_match_type type)
+{
+ struct util_domain *util_domain;
+ int ret;
+
+ util_domain = container_of(domain, struct util_domain, domain_fid);
+
+ if (!info || !info->ep_attr || !info->rx_attr || !info->tx_attr)
+ return -FI_EINVAL;
+
+ ret = fi_check_info(util_prov, info, type);
+ if (ret)
+ return ret;
+
+ ep->ep_fid.fid.fclass = FI_CLASS_EP;
+ ep->ep_fid.fid.context = context;
+ ep->domain = util_domain;
+ atomic_inc(&util_domain->ref);
+ return 0;
+}
+
+int ofi_endpoint_close(struct util_ep *util_ep)
+{
+ atomic_dec(&util_ep->domain->ref);
+ return 0;
+}
diff --git a/prov/util/src/util_eq.c b/prov/util/src/util_eq.c
index 5bc1d37..1cf2daf 100644
--- a/prov/util/src/util_eq.c
+++ b/prov/util/src/util_eq.c
@@ -262,15 +262,19 @@ static int util_verify_eq_attr(const struct fi_provider *prov,
return -FI_EINVAL;
}
- if (attr->signaling_vector) {
- FI_WARN(prov, FI_LOG_EQ, "signaling vectors not supported\n");
- return -FI_ENOSYS;
+ if (attr->flags & ~(FI_AFFINITY | FI_WRITE)) {
+ FI_WARN(prov, FI_LOG_EQ, "invalid flags\n");
+ return -FI_EINVAL;
+ }
+
+ if (attr->flags & FI_AFFINITY) {
+ FI_WARN(prov, FI_LOG_EQ, "signaling vector ignored\n");
}
return 0;
}
-int fi_eq_create(struct fid_fabric *fabric_fid, struct fi_eq_attr *attr,
+int ofi_eq_create(struct fid_fabric *fabric_fid, struct fi_eq_attr *attr,
struct fid_eq **eq_fid, void *context)
{
struct util_fabric *fabric;
diff --git a/prov/util/src/util_fabric.c b/prov/util/src/util_fabric.c
index 8237161..6dfe3d4 100644
--- a/prov/util/src/util_fabric.c
+++ b/prov/util/src/util_fabric.c
@@ -36,7 +36,7 @@
#include <fi_enosys.h>
#include <fi_util.h>
-int util_fabric_close(struct util_fabric *fabric)
+int ofi_fabric_close(struct util_fabric *fabric)
{
if (atomic_get(&fabric->ref))
return -FI_EBUSY;
@@ -54,14 +54,15 @@ static void util_fabric_init(struct util_fabric *fabric, const char *name)
fabric->name = name;
}
-int fi_fabric_init(const struct fi_provider *prov,
+int ofi_fabric_init(const struct fi_provider *prov,
struct fi_fabric_attr *prov_attr,
struct fi_fabric_attr *user_attr,
- struct util_fabric *fabric, void *context)
+ struct util_fabric *fabric, void *context,
+ enum fi_match_type type)
{
int ret;
- ret = fi_check_fabric_attr(prov, prov_attr, user_attr);
+ ret = fi_check_fabric_attr(prov, prov_attr, user_attr, type);
if (ret)
return ret;
diff --git a/prov/util/src/util_main.c b/prov/util/src/util_main.c
index 6c868f4..b59a53d 100644
--- a/prov/util/src/util_main.c
+++ b/prov/util/src/util_main.c
@@ -86,21 +86,6 @@ void fi_fabric_remove(struct util_fabric *fabric)
fastlock_release(&lock);
}
-int fid_list_insert(struct dlist_entry *fid_list, fastlock_t *lock,
- struct fid *fid)
-{
- struct fid_list_entry *entry;
-
- entry = calloc(1, sizeof(*entry));
- if (!entry)
- return -FI_ENOMEM;
-
- fastlock_acquire(lock);
- entry->fid = fid;
- dlist_insert_tail(&entry->entry, fid_list);
- fastlock_release(lock);
- return 0;
-}
static int fi_fid_match(struct dlist_entry *entry, const void *fid)
{
@@ -109,6 +94,31 @@ static int fi_fid_match(struct dlist_entry *entry, const void *fid)
return (item->fid == fid);
}
+int fid_list_insert(struct dlist_entry *fid_list, fastlock_t *lock,
+ struct fid *fid)
+{
+ int ret = 0;
+ struct dlist_entry *entry;
+ struct fid_list_entry *item;
+
+ fastlock_acquire(lock);
+ entry = dlist_find_first_match(fid_list, fi_fid_match, fid);
+ if (entry)
+ goto out;
+
+ item = calloc(1, sizeof(*item));
+ if (!item) {
+ ret = -FI_ENOMEM;
+ goto out;
+ }
+
+ item->fid = fid;
+ dlist_insert_tail(&item->entry, fid_list);
+out:
+ fastlock_release(lock);
+ return ret;
+}
+
void fid_list_remove(struct dlist_entry *fid_list, fastlock_t *lock,
struct fid *fid)
{
@@ -137,14 +147,14 @@ int util_find_domain(struct dlist_entry *item, const void *arg)
((info->mode & domain->mode) == domain->mode);
}
-int util_getinfo(const struct fi_provider *prov, uint32_t version,
+int util_getinfo(const struct util_prov *util_prov, uint32_t version,
const char *node, const char *service, uint64_t flags,
- const struct fi_info *prov_info, struct fi_info *hints,
- struct fi_info **info)
+ struct fi_info *hints, struct fi_info **info)
{
struct util_fabric *fabric;
struct util_domain *domain;
struct dlist_entry *item;
+ const struct fi_provider *prov = util_prov->prov;
int ret, copy_dest;
FI_DBG(prov, FI_LOG_CORE, "checking info\n");
@@ -155,17 +165,17 @@ int util_getinfo(const struct fi_provider *prov, uint32_t version,
return -FI_EINVAL;
}
- ret = fi_check_info(prov, prov_info, hints);
+ ret = fi_check_info(util_prov, hints, FI_MATCH_EXACT);
if (ret)
return ret;
- *info = fi_dupinfo(prov_info);
+ *info = fi_dupinfo(util_prov->info);
if (!*info) {
FI_INFO(prov, FI_LOG_CORE, "cannot copy info\n");
return -FI_ENOMEM;
}
- fi_alter_info(*info, hints);
+ ofi_alter_info(*info, hints);
fabric = fi_fabric_find((*info)->fabric_attr->name);
if (fabric) {
@@ -186,7 +196,7 @@ int util_getinfo(const struct fi_provider *prov, uint32_t version,
}
if (flags & FI_SOURCE) {
- ret = fi_get_addr((*info)->addr_format, flags,
+ ret = ofi_get_addr((*info)->addr_format, flags,
node, service, &(*info)->src_addr,
&(*info)->src_addrlen);
if (ret) {
@@ -198,7 +208,7 @@ int util_getinfo(const struct fi_provider *prov, uint32_t version,
} else {
if (node || service) {
copy_dest = 0;
- ret = fi_get_addr((*info)->addr_format, flags,
+ ret = ofi_get_addr((*info)->addr_format, flags,
node, service, &(*info)->dest_addr,
&(*info)->dest_addrlen);
if (ret) {
@@ -232,7 +242,7 @@ int util_getinfo(const struct fi_provider *prov, uint32_t version,
}
if ((*info)->dest_addr && !(*info)->src_addr) {
- ret = fi_get_src_addr((*info)->addr_format, (*info)->dest_addr,
+ ret = ofi_get_src_addr((*info)->addr_format, (*info)->dest_addr,
(*info)->dest_addrlen, &(*info)->src_addr,
&(*info)->src_addrlen);
if (ret) {
diff --git a/prov/util/src/util_mr.c b/prov/util/src/util_mr.c
new file mode 100644
index 0000000..4116eda
--- /dev/null
+++ b/prov/util/src/util_mr.c
@@ -0,0 +1,256 @@
+/*
+ * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include <config.h>
+#include <stdlib.h>
+#include <fi_enosys.h>
+#include <fi_util.h>
+#include <assert.h>
+#include <rbtree.h>
+
+/* deep copy: make seperate copy of mr_attr and use context for prov_mr */
+static struct fi_mr_attr * create_mr_attr_copy(
+ const struct fi_mr_attr *in_attr, void * prov_mr)
+{
+ struct fi_mr_attr * item;
+ struct iovec *mr_iov;
+ int i = 0;
+
+ if (!prov_mr || !in_attr)
+ return NULL;
+
+ item = malloc(sizeof(struct fi_mr_attr));
+ if (!item)
+ return NULL;
+
+ *item = *in_attr;
+ item->context = prov_mr;
+ mr_iov = malloc(sizeof(struct iovec) * in_attr->iov_count);
+ if (!mr_iov) {
+ free(item);
+ return NULL;
+ }
+
+ for(i = 0; i < in_attr->iov_count; i++)
+ mr_iov[i] = in_attr->mr_iov[i];
+
+ item->mr_iov = mr_iov;
+
+ return item;
+}
+
+static uint64_t get_mr_key(struct ofi_util_mr *mr_h)
+{
+ assert(mr_h->b_key != UINT64_MAX);
+ return mr_h->b_key++;
+}
+
+static int verify_addr(struct ofi_util_mr * in_mr, struct fi_mr_attr * item, uint64_t in_access,
+ uint64_t in_addr, ssize_t in_len)
+{
+ int i = 0;
+ uint64_t start = (uintptr_t) item->mr_iov[i].iov_base;
+ uint64_t end = start + item->mr_iov[i].iov_len;
+
+ if (!in_addr) {
+ FI_DBG(in_mr->prov, FI_LOG_MR, "verify_addr: input address to is zero\n");
+ return -FI_EINVAL;
+ }
+
+ if ((in_access & item->access) != in_access) {
+ FI_DBG(in_mr->prov, FI_LOG_MR, "verify_addr: requested access is not valid\n");
+ return -FI_EACCES;
+ }
+
+ for (i = 0; i < item->iov_count; i++) {
+ if (start <= in_addr && end >= (in_addr + in_len))
+ return 0;
+ }
+
+ return -FI_EACCES;
+}
+
+int ofi_mr_insert(struct ofi_util_mr * in_mr_h, const struct fi_mr_attr *in_attr,
+ uint64_t * out_key, void * in_prov_mr)
+{
+ struct fi_mr_attr * item;
+
+ if (!in_attr || in_attr->iov_count <= 0 || !in_prov_mr) {
+ return -FI_EINVAL;
+ }
+
+ item = create_mr_attr_copy(in_attr, in_prov_mr);
+ if (!item)
+ return -FI_ENOMEM;
+
+ /* Scalable MR handling: use requested key and offset */
+ if (in_mr_h->mr_type == FI_MR_SCALABLE) {
+ item->offset = (uintptr_t) in_attr->mr_iov[0].iov_base + in_attr->offset;
+ /* verify key doesn't already exist */
+ if (rbtFind(in_mr_h->map_handle, &item->requested_key)) {
+ free((void *)item->mr_iov);
+ free(item);
+ return -FI_EINVAL;
+ }
+ } else {
+ item->requested_key = get_mr_key(in_mr_h);
+ item->offset = (uintptr_t) in_attr->mr_iov[0].iov_base;
+ }
+
+ rbtInsert(in_mr_h->map_handle, &item->requested_key, item);
+ *out_key = item->requested_key;
+
+ return 0;
+}
+
+void * ofi_mr_retrieve(struct ofi_util_mr * in_mr_h, uint64_t in_key)
+{
+ void * itr;
+ struct fi_mr_attr * item;
+ void * key = &in_key;
+
+ itr = rbtFind(in_mr_h->map_handle, key);
+
+ if (!itr)
+ return NULL;
+
+ rbtKeyValue(in_mr_h->map_handle, itr, (void **)&key,
+ (void **) &item);
+ return item->context;
+}
+
+
+/* io_addr is address of buff (&buf) */
+int ofi_mr_retrieve_and_verify(struct ofi_util_mr * in_mr_h, ssize_t in_len,
+ uintptr_t *io_addr, uint64_t in_key,
+ uint64_t in_access, void **out_prov_mr)
+{
+ int ret = 0;
+ void * itr;
+ struct fi_mr_attr * item;
+ void * key = &in_key;
+
+ itr = rbtFind(in_mr_h->map_handle, key);
+
+ if (!itr)
+ return -FI_EINVAL;
+
+ rbtKeyValue(in_mr_h->map_handle, itr, &key, (void **) &item);
+
+ /*return providers MR struct */
+ if (!item || !io_addr)
+ return -FI_EINVAL;
+
+ if (out_prov_mr)
+ (*out_prov_mr) = item->context;
+
+ /*offset for scalable */
+ if (in_mr_h->mr_type == FI_MR_SCALABLE)
+ *io_addr = (*io_addr) + item->offset;
+
+ ret = verify_addr(in_mr_h, item, in_access, *io_addr, in_len);
+
+ return ret;
+}
+
+int ofi_mr_erase(struct ofi_util_mr * in_mr_h, uint64_t in_key)
+{
+ void * itr;
+ struct fi_mr_attr * item;
+
+ if (!in_mr_h)
+ return -FI_EINVAL;
+
+ itr = rbtFind(in_mr_h->map_handle, &in_key);
+
+ if (!itr)
+ return -FI_ENOKEY;
+
+ /*release memory */
+ rbtKeyValue(in_mr_h->map_handle, itr, (void **)&in_key,
+ (void **) &item);
+
+ assert(item);
+
+ free((void *)item->mr_iov);
+ free(item);
+
+ rbtErase(in_mr_h->map_handle, itr);
+
+ return 0;
+}
+
+/*assumes uint64_t keys */
+static int compare_mr_keys(void *key1, void *key2)
+{
+ uint64_t k1 = *((uint64_t *) key1);
+ uint64_t k2 = *((uint64_t *) key2);
+ return (k1 < k2) ? -1 : (k1 > k2);
+}
+
+
+int ofi_mr_init(const struct fi_provider *in_prov, enum fi_mr_mode mode,
+ struct ofi_util_mr ** out_new_mr)
+{
+ struct ofi_util_mr * new_mr = malloc(sizeof(struct ofi_util_mr));
+ if (!new_mr)
+ return -FI_ENOMEM;
+
+ assert((mode == FI_MR_SCALABLE) || (mode == FI_MR_BASIC));
+
+ new_mr->mr_type = mode;
+
+ new_mr->map_handle = rbtNew(compare_mr_keys);
+ if (!new_mr->map_handle) {
+ free(new_mr);
+ return -FI_ENOMEM;
+ }
+
+ new_mr->b_key = 0;
+
+ new_mr->prov = in_prov;
+
+ (*out_new_mr) = new_mr;
+
+ return 0;
+}
+
+
+void ofi_mr_close(struct ofi_util_mr *in_mr_h)
+{
+ if (!in_mr_h) {
+ FI_WARN(&core_prov, FI_LOG_MR, "util mr_close: received NULL input\n");
+ return;
+ }
+
+ rbtDelete(in_mr_h->map_handle);
+ free(in_mr_h);
+}
diff --git a/prov/util/src/util_poll.c b/prov/util/src/util_poll.c
index e2b284c..2ec0ca3 100644
--- a/prov/util/src/util_poll.c
+++ b/prov/util/src/util_poll.c
@@ -99,8 +99,13 @@ static int util_poll_run(struct fid_poll *poll_fid, void **context, int count)
cntr = container_of(fid_entry->fid, struct util_cntr,
cntr_fid.fid);
val = fi_cntr_read(&cntr->cntr_fid);
- if ((ret = (val != fid_entry->last_cntr_val)))
- fid_entry->last_cntr_val = val;
+ if ((ret = (val != cntr->checkpoint_cnt))) {
+ cntr->checkpoint_cnt = val;
+ } else {
+ val = fi_cntr_readerr(&cntr->cntr_fid);
+ if ((ret = (val != cntr->checkpoint_err)))
+ cntr->checkpoint_err = val;
+ }
break;
case FI_CLASS_EQ:
eq = container_of(fid_entry->fid, struct util_eq,
diff --git a/prov/util/src/util_wait.c b/prov/util/src/util_wait.c
index 3be155b..287768d 100644
--- a/prov/util/src/util_wait.c
+++ b/prov/util/src/util_wait.c
@@ -38,7 +38,7 @@
#include <fi_util.h>
-int util_trywait(struct fid_fabric *fabric, struct fid **fids, int count)
+int ofi_trywait(struct fid_fabric *fabric, struct fid **fids, int count)
{
struct util_cq *cq;
struct util_eq *eq;
@@ -260,7 +260,7 @@ static int util_verify_wait_fd_attr(const struct fi_provider *prov,
return 0;
}
-int fi_wait_fd_open(struct fid_fabric *fabric_fid, struct fi_wait_attr *attr,
+int ofi_wait_fd_open(struct fid_fabric *fabric_fid, struct fi_wait_attr *attr,
struct fid_wait **waitset)
{
struct util_fabric *fabric;
diff --git a/prov/verbs/Makefile.include b/prov/verbs/Makefile.include
index 39e930a..219f0e0 100644
--- a/prov/verbs/Makefile.include
+++ b/prov/verbs/Makefile.include
@@ -7,6 +7,7 @@ _verbs_files = \
prov/verbs/src/verbs_av.c \
prov/verbs/src/verbs_cm.c \
prov/verbs/src/verbs_cq.c \
+ prov/verbs/src/verbs_srq.c \
prov/verbs/src/verbs_domain.c \
prov/verbs/src/verbs_eq.c \
prov/verbs/src/verbs_info.c \
@@ -18,6 +19,8 @@ _verbs_files = \
prov/verbs/src/ep_rdm/verbs_ep_rdm.c \
prov/verbs/src/ep_rdm/verbs_queuing.h \
prov/verbs/src/ep_rdm/verbs_rdm_cm.c \
+ prov/verbs/src/ep_rdm/verbs_rdm_cntr.c \
+ prov/verbs/src/ep_rdm/verbs_rdm_msg.c \
prov/verbs/src/ep_rdm/verbs_rdm.h \
prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.h \
prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c \
diff --git a/prov/verbs/configure.m4 b/prov/verbs/configure.m4
index 87665eb..7b04c7d 100644
--- a/prov/verbs/configure.m4
+++ b/prov/verbs/configure.m4
@@ -19,7 +19,7 @@ AC_DEFUN([FI_VERBS_CONFIGURE],[
[],
[$verbs_PREFIX],
[$verbs_LIBDIR],
- [verbs_ibverbs_happy=1],
+ [FI_VERBS_DOUBLE_CHECK_LIBIBVERBS],
[verbs_ibverbs_happy=0])
FI_CHECK_PACKAGE([verbs_rdmacm],
@@ -48,3 +48,37 @@ AC_DEFUN([FI_VERBS_CONFIGURE],[
AC_SUBST(verbs_LDFLAGS)
AC_SUBST(verbs_LIBS)
])
+
+dnl
+dnl Per https://github.com/ofiwg/libfabric/issues/2070, it is possible
+dnl that the AC_CHECK_LIB test for libibverbs is not sufficient --
+dnl i.e., AC_CHECK_LIB may succeed, but then linking with libtool may
+dnl fail. This test therefore double checks that we can successfully
+dnl use libtool to link against libibverbs. NOTE: this test is
+dnl contingent upon LT_OUTPUT having already been invoked (i.e., so that
+dnl the libtool script exists).
+dnl
+AC_DEFUN([FI_VERBS_DOUBLE_CHECK_LIBIBVERBS],[
+ AC_MSG_CHECKING(if libibverbs is linkable by libtool)
+ file=conftemp.$$.c
+ rm -f $file conftemp
+ cat > $file <<-EOF
+char ibv_open_device ();
+int main ()
+{ return ibv_open_device (); }
+EOF
+
+ cmd="./libtool --mode=link --tag=CC $CC $CPPFLAGS $CFLAGS $file -o conftemp $LDFLAGS -libverbs"
+ echo "configure:$LINENO: $cmd" >> config.log 2>&1
+ eval $cmd >> config.log 2>&1
+ status=$?
+ AS_IF([test $status -eq 0 && test -x conftemp],
+ [AC_MSG_RESULT(yes)
+ verbs_ibverbs_happy=1],
+ [AC_MSG_RESULT(no)
+ echo "configure: failed program was" >> config.log
+ cat $file >> config.log
+ verbs_ibverbs_happy=0])
+
+ rm -f $file conftemp
+])
diff --git a/prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c b/prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c
index 62238d7..ae584c6 100644
--- a/prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c
+++ b/prov/verbs/src/ep_rdm/verbs_av_ep_rdm.c
@@ -36,29 +36,31 @@
extern struct fi_provider fi_ibv_prov;
-extern struct fi_ibv_rdm_tagged_conn *fi_ibv_rdm_tagged_conn_hash;
-
-int fi_ibv_rdm_start_connection(struct fi_ibv_rdm_ep *ep,
- struct fi_ibv_rdm_tagged_conn *conn)
+ssize_t
+fi_ibv_rdm_start_connection(struct fi_ibv_rdm_ep *ep,
+ struct fi_ibv_rdm_conn *conn)
{
struct rdma_cm_id *id = NULL;
- assert(ep->cm.listener);
+ assert(ep->domain->rdm_cm->listener);
- if (conn->state != FI_VERBS_CONN_ALLOCATED)
- return 0;
+ if (conn->state != FI_VERBS_CONN_ALLOCATED) {
+ return FI_SUCCESS;
+ }
if (ep->is_closing) {
VERBS_INFO(FI_LOG_AV, "Attempt to start connection with addr %s:%u when ep is closing\n",
inet_ntoa(conn->addr.sin_addr),
ntohs(conn->addr.sin_port));
- return -1;
+ return -FI_EOTHER;
}
conn->state = FI_VERBS_CONN_STARTED;
- if (rdma_create_id(ep->cm.ec, &id, conn, RDMA_PS_TCP))
- return -1;
+ if (rdma_create_id(ep->domain->rdm_cm->ec, &id, conn, RDMA_PS_TCP)) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_create_id\n", errno);
+ return -errno;
+ }
if (conn->cm_role == FI_VERBS_CM_ACTIVE ||
conn->cm_role == FI_VERBS_CM_SELF)
@@ -66,25 +68,33 @@ int fi_ibv_rdm_start_connection(struct fi_ibv_rdm_ep *ep,
conn->id[0] = id;
}
- return rdma_resolve_addr(id, NULL, (struct sockaddr *)&conn->addr, 30000);
+ if (rdma_resolve_addr(id, NULL, (struct sockaddr *)&conn->addr, 30000)) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_resolve_addr\n", errno);
+ return -errno;
+ }
+ return FI_SUCCESS;
}
-int fi_ibv_rdm_start_disconnection(struct fi_ibv_rdm_tagged_conn *conn)
+ssize_t fi_ibv_rdm_start_disconnection(struct fi_ibv_rdm_conn *conn)
{
- int ret = 0;
+ ssize_t ret = FI_SUCCESS;
+ ssize_t err = FI_SUCCESS;
FI_INFO(&fi_ibv_prov, FI_LOG_AV,
"Closing connection %p, state %d\n", conn, conn->state);
if (conn->id[0]) {
- ret = rdma_disconnect(conn->id[0]);
- assert(ret == 0);
+ if (rdma_disconnect(conn->id[0])) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_disconnect\n", errno);
+ ret = -errno;
+ }
}
switch (conn->state) {
case FI_VERBS_CONN_ALLOCATED:
case FI_VERBS_CONN_REMOTE_DISCONNECT:
- fi_ibv_rdm_tagged_conn_cleanup(conn);
+ err = fi_ibv_rdm_conn_cleanup(conn);
+ ret = (ret == FI_SUCCESS) ? err : ret;
break;
case FI_VERBS_CONN_ESTABLISHED:
conn->state = FI_VERBS_CONN_LOCAL_DISCONNECT;
@@ -93,10 +103,9 @@ int fi_ibv_rdm_start_disconnection(struct fi_ibv_rdm_tagged_conn *conn)
conn->state = FI_VERBS_CONN_CLOSED;
break;
default:
- ret = -1;
+ ret = -FI_EOTHER;
}
- assert(ret == 0);
return ret;
}
@@ -106,21 +115,38 @@ static inline int fi_ibv_rdm_av_is_valid_address(struct sockaddr_in *addr)
}
-static int fi_ibv_rdm_av_insert(struct fid_av *av, const void *addr,
+static int fi_ibv_rdm_av_insert(struct fid_av *av_fid, const void *addr,
size_t count, fi_addr_t * fi_addr,
uint64_t flags, void *context)
{
- struct fi_ibv_av *fid_av = container_of(av, struct fi_ibv_av, av);
- struct fi_ibv_rdm_ep *ep = fid_av->ep;
- int i, ret = 0;
+ struct fi_ibv_av *av = container_of(av_fid, struct fi_ibv_av, av_fid);
+ struct fi_ibv_rdm_ep *ep = av->ep;
+ size_t i;
+ int ret = 0;
if (ep) {
pthread_mutex_lock(&ep->cm_lock);
}
+ if (av->used + count > av->count) {
+ const size_t new_av_count = av->used + count;
+ if (av->type == FI_AV_TABLE) {
+ void *p = realloc(av->domain->rdm_cm->conn_table,
+ (new_av_count *
+ sizeof(*av->domain->rdm_cm->conn_table)));
+ if (p) {
+ av->domain->rdm_cm->conn_table = p;
+ } else {
+ ret = -FI_ENOMEM;
+ goto out;
+ }
+ }
+ av->count = new_av_count;
+ }
+
for (i = 0; i < count; i++) {
- struct fi_ibv_rdm_tagged_conn *conn = NULL;
- void *addr_i = (char *) addr +
+ struct fi_ibv_rdm_conn *conn = NULL;
+ void *addr_i = (uint8_t *) addr +
i * (ep ? ep->addrlen : FI_IBV_RDM_DFLT_ADDRLEN);
if (!fi_ibv_rdm_av_is_valid_address(addr_i)) {
@@ -134,15 +160,15 @@ static int fi_ibv_rdm_av_insert(struct fid_av *av, const void *addr,
continue;
}
- HASH_FIND(hh, fi_ibv_rdm_tagged_conn_hash, addr_i,
- FI_IBV_RDM_DFLT_ADDRLEN, conn);
+ HASH_FIND(hh, av->domain->rdm_cm->conn_hash, addr_i,
+ FI_IBV_RDM_DFLT_ADDRLEN, conn);
if (!conn) {
/* If addr_i is not found in HASH then we malloc it.
* It could be found if the connection was initiated by the remote
* side.
*/
- conn = memalign(64, sizeof *conn);
+ conn = memalign(FI_IBV_RDM_MEM_ALIGNMENT, sizeof *conn);
if (!conn) {
ret = -FI_ENOMEM;
goto out;
@@ -152,19 +178,37 @@ static int fi_ibv_rdm_av_insert(struct fid_av *av, const void *addr,
dlist_init(&conn->postponed_requests_head);
conn->state = FI_VERBS_CONN_ALLOCATED;
memcpy(&conn->addr, addr_i, FI_IBV_RDM_DFLT_ADDRLEN);
- HASH_ADD(hh, fi_ibv_rdm_tagged_conn_hash, addr,
- FI_IBV_RDM_DFLT_ADDRLEN, conn);
+ HASH_ADD(hh, av->domain->rdm_cm->conn_hash, addr,
+ FI_IBV_RDM_DFLT_ADDRLEN, conn);
}
if (ep) {
fi_ibv_rdm_conn_init_cm_role(conn, ep);
}
- fi_addr[i] = (uintptr_t) (void *) conn;
+
+ switch (av->type) {
+ case FI_AV_MAP:
+ if (fi_addr) {
+ fi_addr[i] = (uintptr_t) (void *) conn;
+ }
+ break;
+ case FI_AV_TABLE:
+ if (fi_addr) {
+ fi_addr[i] = av->used;
+ }
+ av->domain->rdm_cm->conn_table[av->used] = conn;
+ break;
+ default:
+ assert(0);
+ break;
+ }
+
FI_INFO(&fi_ibv_prov, FI_LOG_AV, "fi_av_insert: addr %s:%u conn %p %d\n",
inet_ntoa(conn->addr.sin_addr),
ntohs(conn->addr.sin_port), conn, conn->cm_role);
+ av->used++;
ret++;
}
@@ -175,22 +219,47 @@ out:
return ret;
}
-static int fi_ibv_rdm_av_remove(struct fid_av *av, fi_addr_t * fi_addr,
+static int fi_ibv_rdm_av_remove(struct fid_av *av_fid, fi_addr_t * fi_addr,
size_t count, uint64_t flags)
{
- struct fi_ibv_rdm_tagged_conn *conn;
+ struct fi_ibv_av *av = container_of(av_fid, struct fi_ibv_av, av_fid);
+ struct fi_ibv_rdm_conn *conn = NULL;
+ int ret = FI_SUCCESS;
+ int err = FI_SUCCESS;
int i;
+ if (!fi_addr || (av->type != FI_AV_MAP && av->type != FI_AV_TABLE)) {
+ return -FI_EINVAL;
+ }
+
+ if (av->ep) {
+ pthread_mutex_lock(&av->ep->cm_lock);
+ }
+
for (i = 0; i < count; i++) {
- conn = (struct fi_ibv_rdm_tagged_conn *) fi_addr[i];
+
+ if (fi_addr[i] == FI_ADDR_NOTAVAIL) {
+ continue;
+ }
+
+ if (av->type == FI_AV_MAP) {
+ conn = (struct fi_ibv_rdm_conn *) fi_addr[i];
+ } else { /* (av->type == FI_AV_TABLE) */
+ conn = av->domain->rdm_cm->conn_table[fi_addr[i]];
+ }
+
FI_INFO(&fi_ibv_prov, FI_LOG_AV, "av_remove conn %p, addr %s:%u\n",
conn, inet_ntoa(conn->addr.sin_addr),
ntohs(conn->addr.sin_port));
- HASH_DEL(fi_ibv_rdm_tagged_conn_hash, conn);
- fi_ibv_rdm_start_disconnection(conn);
+ HASH_DEL(av->domain->rdm_cm->conn_hash, conn);
+ err = fi_ibv_rdm_start_disconnection(conn);
+ ret = (ret == FI_SUCCESS) ? err : ret;
}
- return 0;
+ if (av->ep) {
+ pthread_mutex_unlock(&av->ep->cm_lock);
+ }
+ return ret;
}
static struct fi_ops_av fi_ibv_rdm_av_ops = {
diff --git a/prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c b/prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c
index d9568ad..455a478 100644
--- a/prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c
+++ b/prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c
@@ -38,61 +38,70 @@
#include "verbs_queuing.h"
-extern struct util_buf_pool* fi_ibv_rdm_tagged_request_pool;
+struct util_buf_pool *fi_ibv_rdm_request_pool;
+struct util_buf_pool *fi_ibv_rdm_postponed_pool;
+
+/*
+ * extra buffer size equal eager buffer size, it is used for any intermediate
+ * needs like unexpected recv, pack/unpack noncontig messages, etc
+ */
+struct util_buf_pool *fi_ibv_rdm_extra_buffers_pool;
static ssize_t fi_ibv_rdm_tagged_cq_readfrom(struct fid_cq *cq, void *buf,
size_t count, fi_addr_t * src_addr)
{
- struct fi_ibv_cq *_cq = container_of(cq, struct fi_ibv_cq, cq_fid);
+ struct fi_ibv_rdm_cq *_cq =
+ container_of(cq, struct fi_ibv_rdm_cq, cq_fid);
struct fi_cq_tagged_entry *entry = buf;
- size_t i;
+ struct fi_ibv_rdm_request *cq_entry;
+ size_t ret = 0;
- for (i = 0;
- i < count && !dlist_empty(&fi_ibv_rdm_tagged_request_ready_queue);
- ++i)
+ for (cq_entry = count ? fi_ibv_rdm_take_first_from_cq(_cq) : NULL;
+ cq_entry;
+ cq_entry = (ret < count) ? fi_ibv_rdm_take_first_from_cq(_cq) : NULL)
{
- struct fi_ibv_rdm_tagged_request *completed_req =
- container_of(fi_ibv_rdm_tagged_request_ready_queue.next,
- struct fi_ibv_rdm_tagged_request, queue_entry);
+ FI_DBG(&fi_ibv_prov, FI_LOG_CQ,
+ "\t\t-> found in ready: %p op_ctx %p, len %lu, tag 0x%llx\n",
+ cq_entry, cq_entry->context, cq_entry->len,
+ cq_entry->minfo.tag);
- fi_ibv_rdm_tagged_remove_from_ready_queue(completed_req);
+ src_addr[ret] =
+ _cq->ep->av->conn_to_addr(_cq->ep, cq_entry->minfo.conn);
+ entry[ret].op_context = cq_entry->context;
+ entry[ret].flags = (cq_entry->comp_flags & ~FI_COMPLETION);
+ entry[ret].len = cq_entry->len;
+ entry[ret].data = cq_entry->imm;
+ entry[ret].tag = cq_entry->minfo.tag;
- FI_DBG(&fi_ibv_prov, FI_LOG_CQ,
- "\t\t-> found match in ready: op_ctx %p, len %d, tag 0x%llx\n",
- completed_req->context, completed_req->len,
- (long long unsigned int) completed_req->tag);
-
- src_addr[i] = (fi_addr_t) (uintptr_t) completed_req->conn;
- entry[i].op_context = completed_req->context;
- entry[i].flags = 0;
- entry[i].len = completed_req->len;
- entry[i].data = completed_req->imm;
- entry[i].tag = completed_req->tag;
-
- if (completed_req->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE) {
- FI_IBV_RDM_TAGGED_DBG_REQUEST("to_pool: ", completed_req,
- FI_LOG_DEBUG);
- util_buf_release(fi_ibv_rdm_tagged_request_pool, completed_req);
+ if (cq_entry->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE) {
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", cq_entry,
+ FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, cq_entry);
} else {
- completed_req->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
+ cq_entry->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
}
- }
+ ret++;
+ };
- if (i == 0) {
- int err = fi_ibv_rdm_tagged_poll(_cq->ep);
- if (err < 0) {
+ if (ret == 0) {
+ if (fi_ibv_rdm_tagged_poll(_cq->ep) < 0) {
VERBS_INFO(FI_LOG_CQ, "fi_ibv_rdm_tagged_poll failed\n");
}
+
+ if (!dlist_empty(&_cq->request_errcq)) {
+ ret = -FI_EAVAIL;
+ }
}
- return i;
+ return !ret ? -FI_EAGAIN : ret;
}
static ssize_t fi_ibv_rdm_tagged_cq_read(struct fid_cq *cq, void *buf,
size_t count)
{
- struct fi_ibv_cq *_cq = container_of(cq, struct fi_ibv_cq, cq_fid);
- const size_t _count = _cq->ep->n_buffs;
+ struct fi_ibv_rdm_cq *_cq =
+ container_of(cq, struct fi_ibv_rdm_cq, cq_fid);
+ const size_t _count = _cq->read_bunch_size;
fi_addr_t addr[_count];
return fi_ibv_rdm_tagged_cq_readfrom(cq, buf, MIN(_count, count), addr);
@@ -105,8 +114,10 @@ ssize_t fi_ibv_rdm_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count,
size_t threshold = count;
uint64_t time_limit = fi_gettime_ms() + timeout;
size_t counter = 0;
+ ssize_t ret = 0;
+ struct fi_cq_tagged_entry *cqe_buf = buf;
- struct fi_ibv_cq *_cq = container_of(cq, struct fi_ibv_cq, cq_fid);
+ struct fi_ibv_rdm_cq *_cq = container_of(cq, struct fi_ibv_rdm_cq, cq_fid);
switch (_cq->wait_cond) {
case FI_CQ_COND_THRESHOLD:
threshold = MIN((uintptr_t) cond, threshold);
@@ -118,19 +129,28 @@ ssize_t fi_ibv_rdm_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count,
}
do {
- counter += fi_ibv_rdm_tagged_cq_readfrom(cq, buf, threshold,
- src_addr);
- } while (counter < threshold ||
- (timeout >= 0 && fi_gettime_ms() < time_limit));
+ ret = fi_ibv_rdm_tagged_cq_readfrom(cq, &cqe_buf[counter],
+ threshold - counter,
+ src_addr);
+ counter += (ret > 0) ? ret : 0;
+ } while ((ret >= 0) && (counter < threshold ||
+ (timeout >= 0 && fi_gettime_ms() < time_limit)));
- return (counter != 0) ? counter : -FI_EAGAIN;
+ if (counter != 0 && ret >= 0) {
+ ret = counter;
+ } else if (ret == 0) {
+ ret = -FI_EAGAIN;
+ }
+
+ return ret;
}
ssize_t fi_ibv_rdm_cq_sread(struct fid_cq *cq, void *buf, size_t count,
const void *cond, int timeout)
{
- struct fi_ibv_cq *_cq = container_of(cq, struct fi_ibv_cq, cq_fid);
- size_t chunk = MIN(_cq->ep->n_buffs, count);
+ struct fi_ibv_rdm_cq *_cq =
+ container_of(cq, struct fi_ibv_rdm_cq, cq_fid);
+ size_t chunk = MIN(_cq->read_bunch_size, count);
uint64_t time_limit = fi_gettime_ms() + timeout;
size_t rest = count;
fi_addr_t addr[chunk];
@@ -149,72 +169,155 @@ ssize_t fi_ibv_rdm_cq_sread(struct fid_cq *cq, void *buf, size_t count,
return (count != rest) ? (count - rest) : ret;
}
-#if 0
-static const char *fi_ibv_cq_strerror(struct fid_cq *eq, int prov_errno,
- const void *err_data, char *buf,
- size_t len)
+static const char *
+fi_ibv_rdm_cq_strerror(struct fid_cq *eq, int prov_errno, const void *err_data,
+ char *buf, size_t len)
{
+ /* TODO: */
if (buf && len)
strncpy(buf, ibv_wc_status_str(prov_errno), len);
return ibv_wc_status_str(prov_errno);
}
-#endif /* 0 */
static ssize_t
-fi_ibv_rdm_tagged_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *entry,
+fi_ibv_rdm_cq_readerr(struct fid_cq *cq_fid, struct fi_cq_err_entry *entry,
uint64_t flags)
{
-#if 0
- struct fi_ibv_cq *_cq;
- _cq = container_of(cq, struct fi_ibv_cq, cq_fid);
-#endif /* 0 */
- /* TODO */
- return sizeof(*entry);
+ ssize_t ret = 0;
+ struct fi_ibv_rdm_cq *cq =
+ container_of(cq_fid, struct fi_ibv_rdm_cq, cq_fid.fid);
+
+ struct fi_ibv_rdm_request *err_request =
+ fi_ibv_rdm_take_first_from_errcq(cq);
+
+ if (err_request) {
+ entry->op_context = err_request->context;
+ entry->flags = (err_request->comp_flags & ~FI_COMPLETION);
+ entry->len = err_request->len;
+ entry->buf = err_request->unexp_rbuf;
+ entry->data = err_request->imm;
+ entry->tag = err_request->minfo.tag;
+ entry->olen = -1; /* TODO: */
+ entry->err = err_request->state.err;
+ entry->prov_errno = -err_request->state.err;
+ entry->err_data = NULL;
+
+ if (err_request->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE) {
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", err_request,
+ FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, err_request);
+ } else {
+ err_request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
+ }
+
+ ret++;
+ }
+
+ return ret;
}
-static struct fi_ops_cq fi_ibv_rdm_tagged_cq_ops = {
+static struct fi_ops_cq fi_ibv_rdm_cq_ops = {
.size = sizeof(struct fi_ops_cq),
.read = fi_ibv_rdm_tagged_cq_read,
.readfrom = fi_ibv_rdm_tagged_cq_readfrom,
- .readerr = fi_ibv_rdm_tagged_cq_readerr,
+ .readerr = fi_ibv_rdm_cq_readerr,
.sread = fi_ibv_rdm_cq_sread,
.sreadfrom = fi_ibv_rdm_cq_sreadfrom,
- .strerror = fi_cq_strerror
+ .strerror = fi_ibv_rdm_cq_strerror
};
-#if 0
-static int fi_ibv_cq_close(fid_t fid)
+static int fi_ibv_rdm_cq_close(fid_t fid)
{
- struct fi_ibv_cq *cq;
+ struct fi_ibv_rdm_cq *cq =
+ container_of(fid, struct fi_ibv_rdm_cq, cq_fid.fid);
+
+ if(cq->ep) {
+ return -FI_EBUSY;
+ }
+
+ /* TODO: move queues & related pools cleanup from close EP */
+ /* fi_ibv_rdm_clean_queues(); */
+
+ free(cq);
+
+ return FI_SUCCESS;
+}
+
+static struct fi_ops fi_ibv_rdm_cq_fi_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = fi_ibv_rdm_cq_close,
+ .bind = fi_no_bind,
+ .control = fi_no_control,
+ .ops_open = fi_no_ops_open,
+};
+
+
+int fi_ibv_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
+ struct fid_cq **cq, void *context)
+{
+ struct fi_ibv_rdm_cq *_cq;
int ret;
+ int param;
+
+ _cq = calloc(1, sizeof *_cq);
+ if (!_cq)
+ return -FI_ENOMEM;
- cq = container_of(fid, struct fi_ibv_cq, cq_fid.fid);
- assert(cq->ep->type == FI_EP_RDM);
- while (cq->ep->ep_rdm.conn_active) {
- if (0 != (ret = fi_ibv_rdm_tagged_cm_progress(cq->ep)))
- return ret;
+ _cq->domain = container_of(domain, struct fi_ibv_domain, domain_fid);
+ assert(_cq->domain->rdm);
+ switch (attr->wait_obj) {
+ case FI_WAIT_NONE:
+ case FI_WAIT_UNSPEC:
+ break;
+ case FI_WAIT_SET:
+ case FI_WAIT_FD:
+ case FI_WAIT_MUTEX_COND:
+ default:
+ assert(0);
+ ret = -FI_ENOSYS;
+ goto err;
+ }
+
+ _cq->flags |= attr->flags;
+ _cq->wait_cond = attr->wait_cond;
+ _cq->cq_fid.fid.fclass = FI_CLASS_CQ;
+ _cq->cq_fid.fid.context = context;
+ _cq->cq_fid.fid.ops = &fi_ibv_rdm_cq_fi_ops;
+ _cq->cq_fid.ops = &fi_ibv_rdm_cq_ops;
+
+ switch (attr->format) {
+ case FI_CQ_FORMAT_UNSPEC:
+ case FI_CQ_FORMAT_CONTEXT:
+ case FI_CQ_FORMAT_MSG:
+ case FI_CQ_FORMAT_DATA:
+ case FI_CQ_FORMAT_TAGGED:
+ _cq->entry_size = sizeof(struct fi_cq_tagged_entry);
+ break;
+ default:
+ ret = -FI_ENOSYS;
+ goto err;
}
- while (cq->ep->ep_rdm.pend_send || cq->ep->ep_rdm.pend_recv)
- fi_ibv_rdm_tagged_poll(cq->ep);
+ dlist_init(&_cq->request_cq);
+ dlist_init(&_cq->request_errcq);
- if (cq->cq) {
- ret = ibv_destroy_cq(cq->cq);
- if (ret) {
- FI_IBV_ERROR("ibv_destroy_cq returned: "
- "ret %d, errno %d, errstr %s", ret,
- errno, strerror(errno));
- return -ret;
+ _cq->read_bunch_size = FI_IBV_RDM_DFLT_CQREAD_BUNCH_SIZE;
+ if (!fi_param_get_int(&fi_ibv_prov, "rdm_cqread_bunch_size", ¶m)) {
+ if (param > 0) {
+ _cq->read_bunch_size = param;
+ } else {
+ FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
+ "invalid value of rdm_cqread_bunch_size\n");
+ ret = -FI_EINVAL;
+ goto err;
}
}
- free(cq);
+ *cq = &_cq->cq_fid;
return 0;
-}
-#endif
-struct fi_ops_cq *fi_ibv_cq_ops_tagged(struct fi_ibv_cq *cq)
-{
- return &fi_ibv_rdm_tagged_cq_ops;
+err:
+ free(_cq);
+ return ret;
}
diff --git a/prov/verbs/src/ep_rdm/verbs_ep_rdm.c b/prov/verbs/src/ep_rdm/verbs_ep_rdm.c
index f93d264..8e31055 100644
--- a/prov/verbs/src/ep_rdm/verbs_ep_rdm.c
+++ b/prov/verbs/src/ep_rdm/verbs_ep_rdm.c
@@ -36,6 +36,7 @@
#include <rdma/rdma_cma.h>
#include <fi_list.h>
+#include <fi_file.h>
#include <fi_enosys.h>
#include "../fi_verbs.h"
#include "verbs_queuing.h"
@@ -44,17 +45,13 @@
extern struct fi_ops_tagged fi_ibv_rdm_tagged_ops;
extern struct fi_ops_cm fi_ibv_rdm_tagged_ep_cm_ops;
-extern struct dlist_entry fi_ibv_rdm_tagged_recv_posted_queue;
-extern struct util_buf_pool* fi_ibv_rdm_tagged_request_pool;
-extern struct util_buf_pool* fi_ibv_rdm_tagged_extra_buffers_pool;
+extern struct dlist_entry fi_ibv_rdm_posted_queue;
+extern struct util_buf_pool* fi_ibv_rdm_request_pool;
+extern struct util_buf_pool* fi_ibv_rdm_extra_buffers_pool;
extern struct fi_provider fi_ibv_prov;
-struct fi_ibv_rdm_tagged_conn *fi_ibv_rdm_tagged_conn_hash = NULL;
-
-
static int
-fi_ibv_rdm_tagged_find_max_inline_size(struct ibv_pd *pd,
- struct ibv_context *context)
+fi_ibv_rdm_find_max_inline(struct ibv_pd *pd, struct ibv_context *context)
{
struct ibv_qp_init_attr qp_attr;
struct ibv_qp *qp = NULL;
@@ -81,10 +78,20 @@ fi_ibv_rdm_tagged_find_max_inline_size(struct ibv_pd *pd,
/*
* truescale returns max_inline_data 0
*/
- if (0 == qp_attr.cap.max_inline_data) break;
+ if (qp_attr.cap.max_inline_data == 0)
+ break;
+
+ /*
+ * iWarp is able to create qp with unsupported
+ * max_inline, lets take first returned value.
+ */
+ if (context->device->transport_type == IBV_TRANSPORT_IWARP) {
+ max_inline = rst = qp_attr.cap.max_inline_data;
+ break;
+ }
rst = max_inline;
}
- } while (qp && (max_inline *= 2));
+ } while (qp && (max_inline < INT_MAX / 2) && (max_inline *= 2));
if (rst != 0) {
int pos = rst, neg = max_inline;
@@ -116,40 +123,84 @@ fi_ibv_rdm_tagged_find_max_inline_size(struct ibv_pd *pd,
return rst;
}
-static int fi_ibv_rdm_tagged_ep_bind(struct fid *fid, struct fid *bfid,
- uint64_t flags)
+static int fi_ibv_rdm_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
{
struct fi_ibv_rdm_ep *ep;
- struct fi_ibv_cq *cq;
+ struct fi_ibv_rdm_cq *cq;
struct fi_ibv_av *av;
+ struct fi_ibv_rdm_cntr *cntr;
+ int ret;
ep = container_of(fid, struct fi_ibv_rdm_ep, ep_fid.fid);
+ ret = ofi_ep_bind_valid(&fi_ibv_prov, bfid, flags);
+ if (ret)
+ return ret;
switch (bfid->fclass) {
case FI_CLASS_CQ:
- cq = container_of(bfid, struct fi_ibv_cq, cq_fid);
+ cq = container_of(bfid, struct fi_ibv_rdm_cq, cq_fid);
+ if (ep->domain != cq->domain) {
+ return -FI_EINVAL;
+ }
if (flags & FI_RECV) {
if (ep->fi_rcq)
return -EINVAL;
ep->fi_rcq = cq;
+ ep->rx_selective_completion =
+ (flags & FI_SELECTIVE_COMPLETION) ? 1 : 0;
}
+
if (flags & FI_SEND) {
if (ep->fi_scq)
return -EINVAL;
ep->fi_scq = cq;
+ ep->tx_selective_completion =
+ (flags & FI_SELECTIVE_COMPLETION) ? 1 : 0;
}
/* TODO: this is wrong. CQ to EP is 1:n */
cq->ep = ep;
break;
case FI_CLASS_AV:
- av = container_of(bfid, struct fi_ibv_av, av.fid);
+ av = container_of(bfid, struct fi_ibv_av, av_fid.fid);
+ if (ep->domain != av->domain) {
+ return -FI_EINVAL;
+ }
+
ep->av = av;
/* TODO: this is wrong, AV to EP is 1:n */
ep->av->ep = ep;
break;
+ case FI_CLASS_CNTR:
+ cntr = container_of(bfid, struct fi_ibv_rdm_cntr, fid.fid);
+ if (ep->domain != cntr->domain) {
+ return -FI_EINVAL;
+ }
+
+ if ((flags & FI_REMOTE_READ) || (flags & FI_REMOTE_WRITE)) {
+ return -FI_ENOSYS;
+ }
+
+ if (flags & FI_SEND) {
+ ep->send_cntr = cntr;
+ atomic_inc(&ep->send_cntr->ep_ref);
+ }
+ if (flags & FI_RECV) {
+ ep->recv_cntr = cntr;
+ atomic_inc(&ep->recv_cntr->ep_ref);
+ }
+ if (flags & FI_READ) {
+ ep->read_cntr = cntr;
+ atomic_inc(&ep->read_cntr->ep_ref);
+ }
+ if (flags & FI_WRITE) {
+ ep->write_cntr = cntr;
+ atomic_inc(&ep->write_cntr->ep_ref);
+ }
+
+ break;
default:
return -EINVAL;
}
@@ -159,12 +210,12 @@ static int fi_ibv_rdm_tagged_ep_bind(struct fid *fid, struct fid *bfid,
static ssize_t fi_ibv_rdm_tagged_ep_cancel(fid_t fid, void *ctx)
{
- struct fi_ibv_rdm_ep *fid_ep;
struct fi_context *context = (struct fi_context *)ctx;
+ struct fi_ibv_rdm_ep *ep_rdm =
+ container_of(fid, struct fi_ibv_rdm_ep, ep_fid);
int err = 1;
- fid_ep = container_of(fid, struct fi_ibv_rdm_ep, ep_fid);
- if (!fid_ep->domain)
+ if (!ep_rdm->domain)
return -EBADF;
if (!context)
@@ -173,32 +224,38 @@ static ssize_t fi_ibv_rdm_tagged_ep_cancel(fid_t fid, void *ctx)
if (context->internal[0] == NULL)
return 0;
- struct fi_ibv_rdm_tagged_request *request = context->internal[0];
+ struct fi_ibv_rdm_request *request = context->internal[0];
VERBS_DBG(FI_LOG_EP_DATA,
"ep_cancel, match %p, tag 0x%llx, len %d, ctx %p\n",
- request, (long long unsigned)request->tag,
- request->len, request->context);
+ request, request->minfo.tag, request->len, request->context);
struct dlist_entry *found =
- dlist_find_first_match(&fi_ibv_rdm_tagged_recv_posted_queue,
- fi_ibv_rdm_tagged_req_match, request);
-
+ dlist_find_first_match(&fi_ibv_rdm_posted_queue,
+ fi_ibv_rdm_req_match, request);
if (found) {
- assert(container_of(found, struct fi_ibv_rdm_tagged_request,
+ assert(container_of(found, struct fi_ibv_rdm_request,
queue_entry) == request);
+ fi_ibv_rdm_remove_from_posted_queue(request, ep_rdm);
+ VERBS_DBG(FI_LOG_EP_DATA, "\t\t-> SUCCESS, post recv %d\n",
+ ep_rdm->posted_recvs);
+ err = 0;
+ } else {
+ request = fi_ibv_rdm_take_first_match_from_postponed_queue
+ (fi_ibv_rdm_req_match, request);
+ if (request) {
+ fi_ibv_rdm_remove_from_postponed_queue(request);
+ err = 0;
+ }
+ }
- fi_ibv_rdm_tagged_remove_from_posted_queue(request, fid_ep);
-
- FI_IBV_RDM_TAGGED_DBG_REQUEST("to_pool: ", request,
- FI_LOG_DEBUG);
-
- util_buf_release(fi_ibv_rdm_tagged_request_pool, request);
-
- VERBS_DBG(FI_LOG_EP_DATA,
- "\t\t-> SUCCESS, post recv %d\n", fid_ep->posted_recvs);
+ if (!err) {
+ fi_ibv_rdm_cntr_inc_err(ep_rdm->recv_cntr);
- err = 0;
+ if (request->comp_flags & FI_COMPLETION) {
+ fi_ibv_rdm_move_to_errcq(ep_rdm->fi_rcq, request,
+ FI_ECANCELED);
+ }
}
return err;
@@ -269,22 +326,30 @@ static void *fi_ibv_rdm_tagged_cm_progress_thread(void *ctx)
{
struct fi_ibv_rdm_ep *ep = (struct fi_ibv_rdm_ep *)ctx;
while (_fi_ibv_rdm_tagged_cm_progress_running) {
- if (fi_ibv_rdm_tagged_cm_progress(ep)) {
+ if (fi_ibv_rdm_cm_progress(ep)) {
VERBS_INFO (FI_LOG_EP_DATA,
"fi_ibv_rdm_cm_progress error\n");
abort();
}
- usleep(FI_IBV_RDM_CM_THREAD_TIMEOUT);
+ usleep(ep->cm_progress_timeout);
}
return NULL;
}
-static int fi_ibv_rdm_tagged_ep_close(fid_t fid)
+static int fi_ibv_rdm_ep_close(fid_t fid)
{
- int ret = 0;
- struct fi_ibv_rdm_ep *ep;
- void *status;
- ep = container_of(fid, struct fi_ibv_rdm_ep, ep_fid.fid);
+ int ret = FI_SUCCESS;
+ int err = FI_SUCCESS;
+ void *status = NULL;
+ struct fi_ibv_rdm_ep *ep =
+ container_of(fid, struct fi_ibv_rdm_ep, ep_fid.fid);
+
+ if (ep->fi_scq) {
+ ep->fi_scq->ep = NULL;
+ }
+ if (ep->fi_rcq) {
+ ep->fi_rcq->ep = NULL;
+ }
ep->is_closing = 1;
_fi_ibv_rdm_tagged_cm_progress_running = 0;
@@ -296,26 +361,44 @@ static int fi_ibv_rdm_tagged_ep_close(fid_t fid)
fi_ibv_rdm_tagged_poll(ep);
}
- assert(ep->posted_recvs == 0);
+ if (ep->send_cntr) {
+ atomic_dec(&ep->send_cntr->ep_ref);
+ ep->send_cntr = 0;
+ }
+
+ if (ep->recv_cntr) {
+ atomic_dec(&ep->recv_cntr->ep_ref);
+ ep->recv_cntr = 0;
+ }
+
+ if (ep->read_cntr) {
+ atomic_dec(&ep->read_cntr->ep_ref);
+ ep->read_cntr = 0;
+ }
+
+ if (ep->write_cntr) {
+ atomic_dec(&ep->write_cntr->ep_ref);
+ ep->write_cntr = 0;
+ }
- struct fi_ibv_rdm_tagged_conn *conn = NULL, *tmp = NULL;
+ struct fi_ibv_rdm_conn *conn = NULL, *tmp = NULL;
- HASH_ITER(hh, fi_ibv_rdm_tagged_conn_hash, conn, tmp) {
- HASH_DEL(fi_ibv_rdm_tagged_conn_hash, conn);
+ HASH_ITER(hh, ep->domain->rdm_cm->conn_hash, conn, tmp) {
+ HASH_DEL(ep->domain->rdm_cm->conn_hash, conn);
switch (conn->state) {
case FI_VERBS_CONN_ALLOCATED:
case FI_VERBS_CONN_REMOTE_DISCONNECT:
case FI_VERBS_CONN_ESTABLISHED:
- fi_ibv_rdm_start_disconnection(conn);
+ ret = fi_ibv_rdm_start_disconnection(conn);
break;
case FI_VERBS_CONN_STARTED:
while (conn->state != FI_VERBS_CONN_ESTABLISHED &&
conn->state != FI_VERBS_CONN_REJECTED) {
- ret = fi_ibv_rdm_tagged_cm_progress(ep);
+ ret = fi_ibv_rdm_cm_progress(ep);
if (ret) {
- VERBS_INFO(FI_LOG_AV,
- "cm progress failed\n");
- return ret;
+ VERBS_INFO(FI_LOG_AV,
+ "cm progress failed\n");
+ break;
}
}
break;
@@ -324,29 +407,41 @@ static int fi_ibv_rdm_tagged_ep_close(fid_t fid)
}
}
while (ep->num_active_conns) {
- ret = fi_ibv_rdm_tagged_cm_progress(ep);
- if (ret) {
+ err = fi_ibv_rdm_cm_progress(ep);
+ if (err) {
VERBS_INFO(FI_LOG_AV, "cm progress failed\n");
- return ret;
+ ret = (ret == FI_SUCCESS) ? err : ret;
}
}
- assert(0 == HASH_COUNT(fi_ibv_rdm_tagged_conn_hash) &&
- NULL == fi_ibv_rdm_tagged_conn_hash);
+ assert(HASH_COUNT(ep->domain->rdm_cm->conn_hash) == 0 &&
+ ep->domain->rdm_cm->conn_hash == NULL);
+ free(ep->domain->rdm_cm->conn_table);
VERBS_INFO(FI_LOG_AV, "DISCONNECT complete\n");
- ibv_destroy_cq(ep->scq);
- ibv_destroy_cq(ep->rcq);
+ assert(ep->scq && ep->rcq);
+ if (ibv_destroy_cq(ep->scq) || ibv_destroy_cq(ep->rcq)) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "ibv_destroy_cq failed\n", errno);
+ ret = (ret == FI_SUCCESS) ? -errno : ret;
+ }
+
+ rdma_freeaddrinfo(ep->rai);
+ errno = 0;
+ if (errno) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_freeaddrinfo failed\n", errno);
+ ret = (ret == FI_SUCCESS) ? -ret : ret;
+ }
- fi_ibv_destroy_ep(FI_EP_RDM, ep->cm.rai, &(ep->cm.listener));
+ /* TODO: move queues & related pools cleanup to close CQ*/
+ fi_ibv_rdm_clean_queues(ep);
- util_buf_pool_destroy(fi_ibv_rdm_tagged_request_pool);
- util_buf_pool_destroy(fi_ibv_rdm_tagged_postponed_pool);
- util_buf_pool_destroy(fi_ibv_rdm_tagged_extra_buffers_pool);
+ util_buf_pool_destroy(fi_ibv_rdm_request_pool);
+ util_buf_pool_destroy(fi_ibv_rdm_extra_buffers_pool);
+ util_buf_pool_destroy(fi_ibv_rdm_postponed_pool);
free(ep);
- return 0;
+ return ret;
}
#if 0
@@ -381,23 +476,27 @@ static int fi_ibv_ep_sync(fid_t fid, uint64_t flags, void *context)
}
#endif /* 0 */
-struct fi_ops fi_ibv_rdm_tagged_ep_ops = {
+struct fi_ops fi_ibv_rdm_ep_ops = {
.size = sizeof(struct fi_ops),
- .close = fi_ibv_rdm_tagged_ep_close,
- .bind = fi_ibv_rdm_tagged_ep_bind,
+ .close = fi_ibv_rdm_ep_close,
+ .bind = fi_ibv_rdm_ep_bind,
.control = fi_ibv_rdm_tagged_control,
.ops_open = fi_no_ops_open,
};
-int fi_ibv_open_rdm_ep(struct fid_domain *domain, struct fi_info *info,
+int fi_ibv_rdm_open_ep(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
struct fi_ibv_domain *_domain =
container_of(domain, struct fi_ibv_domain, domain_fid);
int ret = 0;
+ int param = 0;
+ char *str_param = NULL;
- if (strncmp(_domain->verbs->device->name, info->domain_attr->name,
- strlen(_domain->verbs->device->name))) {
+ if (!info || !info->ep_attr || !info->domain_attr ||
+ strncmp(_domain->verbs->device->name, info->domain_attr->name,
+ strlen(_domain->verbs->device->name)))
+ {
return -FI_EINVAL;
}
@@ -410,53 +509,132 @@ int fi_ibv_open_rdm_ep(struct fid_domain *domain, struct fi_info *info,
_ep->domain = _domain;
_ep->ep_fid.fid.fclass = FI_CLASS_EP;
_ep->ep_fid.fid.context = context;
- _ep->ep_fid.fid.ops = &fi_ibv_rdm_tagged_ep_ops;
+ _ep->ep_fid.fid.ops = &fi_ibv_rdm_ep_ops;
_ep->ep_fid.ops = &fi_ibv_rdm_tagged_ep_base_ops;
- _ep->ep_fid.tagged = &fi_ibv_rdm_tagged_ops;
- _ep->ep_fid.rma = fi_ibv_rdm_ep_ops_rma(_ep);
_ep->ep_fid.cm = &fi_ibv_rdm_tagged_ep_cm_ops;
+ _ep->ep_fid.msg = fi_ibv_rdm_ep_ops_msg();
+ _ep->ep_fid.rma = fi_ibv_rdm_ep_ops_rma();
+ _ep->ep_fid.tagged = &fi_ibv_rdm_tagged_ops;
+ _ep->tx_selective_completion = 0;
+ _ep->rx_selective_completion = 0;
- ret = fi_ibv_create_ep(NULL, NULL, 0, info, &_ep->cm.rai, &_ep->cm.listener);
+ _ep->n_buffs = fi_param_get_int(&fi_ibv_prov, "rdm_buffer_num", ¶m) ?
+ FI_IBV_RDM_TAGGED_DFLT_BUFFER_NUM : param;
- if (rdma_listen(_ep->cm.listener, 1024)) {
- VERBS_INFO(FI_LOG_EP_CTRL, "rdma_listen failed: %s\n",
- strerror(errno));
- ret = -FI_EOTHER;
+ if (_ep->n_buffs & (_ep->n_buffs - 1)) {
+ FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
+ "invalid value of rdm_buffer_num\n");
+ ret = -FI_EINVAL;
goto err;
}
- _ep->n_buffs = FI_IBV_RDM_TAGGED_DFLT_BUFFER_NUM;
- _ep->buff_len = FI_IBV_RDM_DFLT_BUFFER_SIZE;
- _ep->rndv_threshold = FI_IBV_RDM_DFLT_BUFFERED_SSIZE;
+ FI_INFO(&fi_ibv_prov, FI_LOG_EP_CTRL, "inject_size: %d\n",
+ info->tx_attr->inject_size);
+
+ _ep->rndv_threshold = info->tx_attr->inject_size;
+ FI_INFO(&fi_ibv_prov, FI_LOG_EP_CTRL, "rndv_threshold: %d\n",
+ _ep->rndv_threshold);
+
+ _ep->buff_len = rdm_buffer_size(info->tx_attr->inject_size);
+ FI_INFO(&fi_ibv_prov, FI_LOG_EP_CTRL, "buff_len: %d\n", _ep->buff_len);
+
+ _ep->rndv_seg_size = FI_IBV_RDM_SEG_MAXSIZE;
+ if (!fi_param_get_int(&fi_ibv_prov, "rdm_rndv_seg_size", ¶m)) {
+ if (param > 0) {
+ _ep->rndv_seg_size = param;
+ } else {
+ FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
+ "invalid value of rdm_rndv_seg_size\n");
+ ret = -FI_EINVAL;
+ goto err;
+ }
+ }
- _ep->rq_wr_depth = FI_IBV_RDM_TAGGED_DFLT_RQ_SIZE;
+ _ep->cm_progress_timeout = FI_IBV_RDM_CM_THREAD_TIMEOUT;
+ if (!fi_param_get_int(&fi_ibv_prov, "rdm_thread_timeout", ¶m)) {
+ if (param < 0) {
+ FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
+ "invalid value of rdm_thread_timeout\n");
+ ret = -FI_EINVAL;
+ goto err;
+ } else {
+ _ep->cm_progress_timeout = param;
+ }
+ }
- /*
- * max number of WRs in SQ is n_buffer for send and
- * the same amount for buffer releasing from recv
- */
- _ep->sq_wr_depth = 2 * (_ep->n_buffs + 1);
+ _ep->rq_wr_depth = info->rx_attr->size;
+ /* one more outstanding slot for releasing eager buffers */
+ _ep->sq_wr_depth = _ep->n_buffs + 1;
+ if (!fi_param_get_str(&fi_ibv_prov, "rdm_eager_send_opcode", &str_param)) {
+ if (!strncmp(str_param, "IBV_WR_RDMA_WRITE_WITH_IMM",
+ strlen("IBV_WR_RDMA_WRITE_WITH_IMM"))) {
+ _ep->eopcode = IBV_WR_RDMA_WRITE_WITH_IMM;
+ } else if (!strncmp(str_param, "IBV_WR_SEND",
+ strlen("IBV_WR_SEND"))) {
+ _ep->eopcode = IBV_WR_SEND;
+ } else {
+ FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
+ "invalid value of rdm_eager_send_opcode\n");
+ ret = -FI_EINVAL;
+ goto err;
+ }
+ } else {
+ _ep->eopcode = IBV_WR_SEND;
+ }
+
+ switch (info->ep_attr->protocol) {
+ case FI_PROTO_IB_RDM:
+ if (_ep->eopcode != IBV_WR_RDMA_WRITE_WITH_IMM &&
+ _ep->eopcode != IBV_WR_SEND) {
+ FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
+ "Unsupported eager operation code\n");
+ ret = -FI_ENODATA;
+ goto err;
+ }
+ break;
+ case FI_PROTO_IWARP_RDM:
+ if (_ep->eopcode != IBV_WR_SEND) {
+ FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
+ "Unsupported eager operation code\n");
+ ret = -FI_ENODATA;
+ goto err;
+ }
+ break;
+ default:
+ FI_INFO(&fi_ibv_prov, FI_LOG_CORE, "Unsupported protocol\n");
+ ret = -FI_ENODATA;
+ goto err;
+ }
+
+ ret = fi_ibv_get_rdma_rai(NULL, NULL, 0, info, &_ep->rai);
+ if (ret) {
+ goto err;
+ }
+ ret = fi_ibv_rdm_cm_bind_ep(_ep->domain->rdm_cm, _ep);
+ if (ret) {
+ goto err;
+ }
_ep->posted_sends = 0;
_ep->posted_recvs = 0;
- _ep->recv_preposted_threshold = MAX(0.2 * _ep->rq_wr_depth, 5);
+ _ep->recv_preposted_threshold = MAX(0.2 * _ep->rq_wr_depth, _ep->n_buffs);
VERBS_INFO(FI_LOG_EP_CTRL, "recv preposted threshold: %d\n",
_ep->recv_preposted_threshold);
- fi_ibv_rdm_tagged_request_pool = util_buf_pool_create(
- sizeof(struct fi_ibv_rdm_tagged_request),
+ fi_ibv_rdm_request_pool = util_buf_pool_create(
+ sizeof(struct fi_ibv_rdm_request),
FI_IBV_RDM_MEM_ALIGNMENT, 0, 100);
- fi_ibv_rdm_tagged_postponed_pool = util_buf_pool_create(
- sizeof(struct fi_ibv_rdm_tagged_postponed_entry),
+ fi_ibv_rdm_postponed_pool = util_buf_pool_create(
+ sizeof(struct fi_ibv_rdm_postponed_entry),
FI_IBV_RDM_MEM_ALIGNMENT, 0, 100);
- fi_ibv_rdm_tagged_extra_buffers_pool = util_buf_pool_create(
+ fi_ibv_rdm_extra_buffers_pool = util_buf_pool_create(
_ep->buff_len, FI_IBV_RDM_MEM_ALIGNMENT, 0, 100);
- _ep->max_inline_rc =
- fi_ibv_rdm_tagged_find_max_inline_size(_ep->domain->pd,
- _ep->domain->verbs);
+ _ep->max_inline_rc =
+ fi_ibv_rdm_find_max_inline(_ep->domain->pd, _ep->domain->verbs);
+
_ep->scq_depth = FI_IBV_RDM_TAGGED_DFLT_SCQ_SIZE;
_ep->rcq_depth = FI_IBV_RDM_TAGGED_DFLT_RCQ_SIZE;
@@ -479,9 +657,10 @@ int fi_ibv_open_rdm_ep(struct fid_domain *domain, struct fi_info *info,
*ep = &_ep->ep_fid;
_ep->is_closing = 0;
- fi_ibv_rdm_tagged_req_hndls_init();
+ fi_ibv_rdm_req_hndls_init();
pthread_mutex_init(&_ep->cm_lock, NULL);
+ _fi_ibv_rdm_tagged_cm_progress_running = 1;
ret = pthread_create(&_ep->cm_progress_thread, NULL,
&fi_ibv_rdm_tagged_cm_progress_thread,
(void *)_ep);
@@ -491,9 +670,9 @@ int fi_ibv_open_rdm_ep(struct fid_domain *domain, struct fi_info *info,
ret = -FI_EOTHER;
goto err;
}
-out:
+
return ret;
err:
free(_ep);
- goto out;
+ return ret;
}
diff --git a/prov/verbs/src/ep_rdm/verbs_queuing.h b/prov/verbs/src/ep_rdm/verbs_queuing.h
index 1f684fb..4bf86ef 100644
--- a/prov/verbs/src/ep_rdm/verbs_queuing.h
+++ b/prov/verbs/src/ep_rdm/verbs_queuing.h
@@ -37,120 +37,176 @@
#include <fi_list.h>
#include "verbs_rdm.h"
-/* managing of queues
+/*
+ * managing of queues
*/
-extern struct dlist_entry fi_ibv_rdm_tagged_request_ready_queue;
-extern struct dlist_entry fi_ibv_rdm_tagged_recv_unexp_queue;
+extern struct dlist_entry fi_ibv_rdm_unexp_queue;
/* TODO: implement posted recv queue per connection */
-extern struct dlist_entry fi_ibv_rdm_tagged_recv_posted_queue;
-extern struct dlist_entry fi_ibv_rdm_tagged_send_postponed_queue;
+extern struct dlist_entry fi_ibv_rdm_posted_queue;
+extern struct dlist_entry fi_ibv_rdm_postponed_queue;
-extern struct util_buf_pool* fi_ibv_rdm_tagged_postponed_pool;
+extern struct util_buf_pool* fi_ibv_rdm_postponed_pool;
+static inline void
+fi_ibv_rdm_move_to_cq(struct fi_ibv_rdm_cq *cq,
+ struct fi_ibv_rdm_request *request)
+{
+ FI_IBV_RDM_DBG_REQUEST("move_to_cq: ", request, FI_LOG_DEBUG);
+ dlist_insert_tail(&request->queue_entry, &cq->request_cq);
+}
static inline void
-fi_ibv_rdm_tagged_move_to_ready_queue(struct fi_ibv_rdm_tagged_request *request)
+fi_ibv_rdm_remove_from_cq(struct fi_ibv_rdm_request *request)
{
- FI_IBV_RDM_TAGGED_DBG_REQUEST("move_to_ready_queue: ",
- request, FI_LOG_DEBUG);
+ FI_IBV_RDM_DBG_REQUEST("remove_from_cq: ", request, FI_LOG_DEBUG);
+ dlist_remove(&request->queue_entry);
+}
+
+static inline struct fi_ibv_rdm_request *
+fi_ibv_rdm_take_first_from_cq(struct fi_ibv_rdm_cq *cq)
+{
+ if (cq && !dlist_empty(&cq->request_cq)) {
+ struct fi_ibv_rdm_request *entry =
+ container_of(cq->request_cq.next,
+ struct fi_ibv_rdm_request, queue_entry);
+ fi_ibv_rdm_remove_from_cq(entry);
+ return entry;
+ }
+ return NULL;
+}
+
+static inline void
+fi_ibv_rdm_move_to_errcq(struct fi_ibv_rdm_cq *cq,
+ struct fi_ibv_rdm_request *request, ssize_t err)
+{
+ request->state.err = llabs(err);
+ FI_IBV_RDM_DBG_REQUEST("move_to_errcq: ", request, FI_LOG_DEBUG);
+ assert(request->context);
dlist_insert_tail(&request->queue_entry,
- &fi_ibv_rdm_tagged_request_ready_queue);
+ &cq->request_errcq);
}
static inline void
-fi_ibv_rdm_tagged_remove_from_ready_queue(
- struct fi_ibv_rdm_tagged_request *request)
+fi_ibv_rdm_remove_from_errcq(struct fi_ibv_rdm_request *request)
{
- FI_IBV_RDM_TAGGED_DBG_REQUEST("remove_from_ready_queue: ",
- request, FI_LOG_DEBUG);
+ FI_IBV_RDM_DBG_REQUEST("remove_from_errcq: ", request, FI_LOG_DEBUG);
dlist_remove(&request->queue_entry);
}
+static inline struct fi_ibv_rdm_request *
+fi_ibv_rdm_take_first_from_errcq(struct fi_ibv_rdm_cq *cq)
+{
+ if (cq && !dlist_empty(&cq->request_errcq)) {
+ struct fi_ibv_rdm_request *entry =
+ container_of(cq->request_errcq.next,
+ struct fi_ibv_rdm_request, queue_entry);
+ fi_ibv_rdm_remove_from_errcq(entry);
+ return entry;
+ }
+ return NULL;
+}
+
static inline void
-fi_ibv_rdm_tagged_move_to_unexpected_queue(
- struct fi_ibv_rdm_tagged_request *request)
+fi_ibv_rdm_move_to_unexpected_queue(struct fi_ibv_rdm_request *request)
{
- FI_IBV_RDM_TAGGED_DBG_REQUEST("move_to_unexpected_queue: ",
- request, FI_LOG_DEBUG);
- dlist_insert_tail(&request->queue_entry,
- &fi_ibv_rdm_tagged_recv_unexp_queue);
+ FI_IBV_RDM_DBG_REQUEST("move_to_unexpected_queue: ", request,
+ FI_LOG_DEBUG);
+ dlist_insert_tail(&request->queue_entry, &fi_ibv_rdm_unexp_queue);
}
static inline void
-fi_ibv_rdm_tagged_remove_from_unexp_queue(
- struct fi_ibv_rdm_tagged_request *request)
+fi_ibv_rdm_remove_from_unexp_queue(struct fi_ibv_rdm_request *request)
{
- FI_IBV_RDM_TAGGED_DBG_REQUEST("remove_from_unexpected_queue: ", request,
- FI_LOG_DEBUG);
+ FI_IBV_RDM_DBG_REQUEST("remove_from_unexpected_queue: ", request,
+ FI_LOG_DEBUG);
dlist_remove(&request->queue_entry);
}
+static inline struct fi_ibv_rdm_request *
+fi_ibv_rdm_take_first_from_unexp_queue()
+{
+ if (!dlist_empty(&fi_ibv_rdm_unexp_queue)) {
+ struct fi_ibv_rdm_request *entry =
+ container_of(fi_ibv_rdm_unexp_queue.next,
+ struct fi_ibv_rdm_request, queue_entry);
+ fi_ibv_rdm_remove_from_unexp_queue(entry);
+ return entry;
+ }
+ return NULL;
+}
+
static inline void
-fi_ibv_rdm_tagged_move_to_posted_queue(
- struct fi_ibv_rdm_tagged_request *request,
- struct fi_ibv_rdm_ep *ep)
+fi_ibv_rdm_move_to_posted_queue(struct fi_ibv_rdm_request *request,
+ struct fi_ibv_rdm_ep *ep)
{
- FI_IBV_RDM_TAGGED_DBG_REQUEST("move_to_posted_queue: ", request,
- FI_LOG_DEBUG);
- dlist_insert_tail(&request->queue_entry,
- &fi_ibv_rdm_tagged_recv_posted_queue);
+ FI_IBV_RDM_DBG_REQUEST("move_to_posted_queue: ", request, FI_LOG_DEBUG);
+ dlist_insert_tail(&request->queue_entry, &fi_ibv_rdm_posted_queue);
ep->posted_recvs++;
}
static inline void
-fi_ibv_rdm_tagged_remove_from_posted_queue(
- struct fi_ibv_rdm_tagged_request *request,
- struct fi_ibv_rdm_ep *ep)
+fi_ibv_rdm_remove_from_posted_queue(struct fi_ibv_rdm_request *request,
+ struct fi_ibv_rdm_ep *ep)
{
- FI_IBV_RDM_TAGGED_DBG_REQUEST("remove_from_posted_queue: ", request,
- FI_LOG_DEBUG);
+ FI_IBV_RDM_DBG_REQUEST("remove_from_posted_queue: ", request,
+ FI_LOG_DEBUG);
dlist_remove(&request->queue_entry);
ep->posted_recvs--;
}
+static inline struct fi_ibv_rdm_request *
+fi_ibv_rdm_take_first_from_posted_queue()
+{
+ if (!dlist_empty(&fi_ibv_rdm_posted_queue)) {
+ struct fi_ibv_rdm_request *entry =
+ container_of(fi_ibv_rdm_posted_queue.next,
+ struct fi_ibv_rdm_request, queue_entry);
+ fi_ibv_rdm_remove_from_unexp_queue(entry);
+ return entry;
+ }
+ return NULL;
+}
+
static inline void
-fi_ibv_rdm_tagged_move_to_postponed_queue(
- struct fi_ibv_rdm_tagged_request *request)
+fi_ibv_rdm_move_to_postponed_queue(struct fi_ibv_rdm_request *request)
{
- FI_IBV_RDM_TAGGED_DBG_REQUEST("move_to_postponed_queue: ", request,
- FI_LOG_DEBUG);
- assert(request && request->conn);
+ FI_IBV_RDM_DBG_REQUEST("move_to_postponed_queue: ", request,
+ FI_LOG_DEBUG);
+ assert(request && request->minfo.conn);
- struct fi_ibv_rdm_tagged_conn *conn = request->conn;
+ struct fi_ibv_rdm_conn *conn = request->minfo.conn;
if (dlist_empty(&conn->postponed_requests_head)) {
- struct fi_ibv_rdm_tagged_postponed_entry *entry =
- util_buf_alloc(fi_ibv_rdm_tagged_postponed_pool);
+ struct fi_ibv_rdm_postponed_entry *entry =
+ util_buf_alloc(fi_ibv_rdm_postponed_pool);
entry->conn = conn;
conn->postponed_entry = entry;
dlist_insert_tail(&entry->queue_entry,
- &fi_ibv_rdm_tagged_send_postponed_queue);
+ &fi_ibv_rdm_postponed_queue);
}
dlist_insert_tail(&request->queue_entry,
&conn->postponed_requests_head);
}
static inline void
-fi_ibv_rdm_tagged_remove_from_postponed_queue(
- struct fi_ibv_rdm_tagged_request *request)
+fi_ibv_rdm_remove_from_postponed_queue(struct fi_ibv_rdm_request *request)
{
- FI_IBV_RDM_TAGGED_DBG_REQUEST("remove_from_postponed_queue: ", request,
- FI_LOG_DEBUG);
+ FI_IBV_RDM_DBG_REQUEST("remove_from_postponed_queue: ", request,
+ FI_LOG_DEBUG);
- struct fi_ibv_rdm_tagged_conn *conn = request->conn;
+ struct fi_ibv_rdm_conn *conn = request->minfo.conn;
assert(conn);
assert(!dlist_empty(&conn->postponed_requests_head));
/*
* remove from conn->postponed_requests_head at first
* then if conn->postponed_requests_head is empty
- * clean fi_ibv_rdm_tagged_send_postponed_queue
+ * clean fi_ibv_rdm_postponed_queue
*/
-
dlist_remove(&request->queue_entry);
request->queue_entry.next = request->queue_entry.prev = NULL;
@@ -161,10 +217,59 @@ fi_ibv_rdm_tagged_remove_from_postponed_queue(
conn->postponed_entry->queue_entry.prev = NULL;
conn->postponed_entry->conn = NULL;
- util_buf_release(fi_ibv_rdm_tagged_postponed_pool,
+ util_buf_release(fi_ibv_rdm_postponed_pool,
conn->postponed_entry);
conn->postponed_entry = NULL;
}
}
+static inline struct fi_ibv_rdm_request *
+fi_ibv_rdm_take_first_from_postponed_queue()
+{
+ if (!dlist_empty(&fi_ibv_rdm_postponed_queue)) {
+ struct fi_ibv_rdm_postponed_entry *entry =
+ container_of(fi_ibv_rdm_postponed_queue.next,
+ struct fi_ibv_rdm_postponed_entry, queue_entry);
+
+ if (!dlist_empty(&entry->conn->postponed_requests_head)) {
+ struct dlist_entry *req_entry =
+ entry->conn->postponed_requests_head.next;
+
+ struct fi_ibv_rdm_request *request =
+ container_of(req_entry, struct fi_ibv_rdm_request,
+ queue_entry);
+ fi_ibv_rdm_remove_from_postponed_queue(request);
+ return request;
+ }
+ }
+
+ return NULL;
+}
+
+static inline struct fi_ibv_rdm_request *
+fi_ibv_rdm_take_first_match_from_postponed_queue(dlist_func_t *match, const void *arg)
+{
+ struct dlist_entry *i, *j;
+ dlist_foreach((&fi_ibv_rdm_postponed_queue), i) {
+ struct fi_ibv_rdm_postponed_entry *entry =
+ container_of(i, struct fi_ibv_rdm_postponed_entry,
+ queue_entry);
+
+ j = dlist_find_first_match((&entry->conn->postponed_requests_head),
+ match, arg);
+ if (j) {
+ struct fi_ibv_rdm_request *request =
+ container_of(j, struct fi_ibv_rdm_request,
+ queue_entry);
+
+ fi_ibv_rdm_remove_from_postponed_queue(request);
+ return request;
+ }
+ }
+
+ return NULL;
+}
+
+void fi_ibv_rdm_clean_queues();
+
#endif // _VERBS_QUEING_H
diff --git a/prov/verbs/src/ep_rdm/verbs_rdm.h b/prov/verbs/src/ep_rdm/verbs_rdm.h
index b1e2703..52e9005 100644
--- a/prov/verbs/src/ep_rdm/verbs_rdm.h
+++ b/prov/verbs/src/ep_rdm/verbs_rdm.h
@@ -48,6 +48,7 @@
#define FI_IBV_RDM_RNDV_RTS_PKT 1
#define FI_IBV_RDM_RNDV_ACK_PKT 2
#define FI_IBV_RDM_RMA_PKT 3
+#define FI_IBV_RDM_MSG_PKT 4
#define FI_IBV_RDM_SET_PKTTYPE(dest, type) (dest |= type)
#define FI_IBV_RDM_GET_PKTTYPE(value) (value & FI_IBV_RDM_ST_PKTTYPE_MASK)
@@ -84,47 +85,46 @@ do { \
(_connection)->sends_outgoing); \
} while (0)
-#define FI_IBV_RDM_TAGGED_DEC_SEND_COUNTERS(_connection, _ep) \
-do { \
- (_connection)->sends_outgoing--; \
- (_ep)->posted_sends--; \
+#define FI_IBV_RDM_DEC_SIG_POST_COUNTERS(_connection, _ep) \
+do { \
+ (_connection)->sends_outgoing--; \
+ (_ep)->posted_sends--; \
\
- VERBS_DBG(FI_LOG_CQ, "SEND_COUNTER--, conn %p, sends_outgoing %d\n", \
- _connection, (_connection)->sends_outgoing); \
- assert((_ep)->posted_sends >= 0); \
- assert((_connection)->sends_outgoing >= 0); \
+ VERBS_DBG(FI_LOG_CQ, "SEND_COUNTER--, conn %p, sends_outgoing %d\n", \
+ _connection, (_connection)->sends_outgoing); \
+ assert((_ep)->posted_sends >= 0); \
+ assert((_connection)->sends_outgoing >= 0); \
} while (0)
-#define FI_IBV_RDM_TAGGED_SENDS_OUTGOING_ARE_LIMITED(_connection, _ep) \
- ((_connection)->sends_outgoing > 0.5*(_ep)->sq_wr_depth)
+#define OUTGOING_POST_LIMIT(_connection, _ep) \
+ ((_connection)->sends_outgoing >= (_ep)->sq_wr_depth - 1)
-#define PEND_SEND_IS_LIMITED(_ep) \
+#define PEND_POST_LIMIT(_ep) \
((_ep)->posted_sends > 0.5 * (_ep)->scq_depth)
-#define SEND_RESOURCES_IS_BUSY(_connection, _ep) \
- (FI_IBV_RDM_TAGGED_SENDS_OUTGOING_ARE_LIMITED(_connection, _ep) || \
- PEND_SEND_IS_LIMITED(_ep))
+#define TSEND_RESOURCES_IS_BUSY(_connection, _ep) \
+ (OUTGOING_POST_LIMIT(_connection, _ep) || PEND_POST_LIMIT(_ep))
#define RMA_RESOURCES_IS_BUSY(_connection, _ep) \
- (FI_IBV_RDM_TAGGED_SENDS_OUTGOING_ARE_LIMITED(_connection, _ep) || \
- PEND_SEND_IS_LIMITED(_ep))
+ (OUTGOING_POST_LIMIT(_connection, _ep) || PEND_POST_LIMIT(_ep))
struct fi_ibv_rdm_header {
- uint64_t imm_data; // TODO: not implemented
+/* uint64_t imm_data; TODO: not implemented */
uint64_t tag;
uint32_t service_tag;
- uint32_t seq_num;
+ uint32_t padding;
};
-struct fi_ibv_rdm_tagged_rndv_header {
+struct fi_ibv_rdm_rndv_header {
struct fi_ibv_rdm_header base;
uint64_t src_addr;
- void *id;
- int len;
+ uint64_t id; /* pointer to request on sender side */
+ uint64_t total_len;
uint32_t mem_key;
+ uint32_t is_tagged;
};
-struct fi_ibv_rdm_tagged_request {
+struct fi_ibv_rdm_request {
/* Accessors and match info */
@@ -132,14 +132,12 @@ struct fi_ibv_rdm_tagged_request {
struct dlist_entry queue_entry;
struct {
- enum fi_ibv_rdm_tagged_request_eager_state eager;
- enum fi_ibv_rdm_tagged_request_rndv_state rndv;
+ enum fi_ibv_rdm_request_eager_state eager;
+ enum fi_ibv_rdm_request_rndv_state rndv;
+ ssize_t err; /* filled in case of moving to errcq */
} state;
- struct fi_ibv_rdm_tagged_conn *conn;
-
- uint64_t tag;
- uint64_t tagmask;
+ struct fi_ibv_rdm_minfo minfo;
/* User data: buffers, lens, imm, context */
@@ -151,9 +149,9 @@ struct fi_ibv_rdm_tagged_request {
union {
void *exp_rbuf;
- void *unexp_rbuf;
- void *sbuf;
- void *rmabuf;
+ struct fi_ibv_rdm_buf *unexp_rbuf;
+ struct fi_ibv_rdm_buf *sbuf;
+ struct fi_ibv_rdm_buf *rmabuf;
struct iovec *rmaiovec_arr;
};
@@ -161,16 +159,19 @@ struct fi_ibv_rdm_tagged_request {
* iovec_arr is a mem pool entry if iov_count > 0
* and must be freed properly
*/
- size_t iov_count;
- size_t len;
+ uint64_t iov_count;
+ uint64_t len;
+ uint64_t rest_len;
+ uint64_t comp_flags;
struct fi_context *context;
+ uint32_t post_counter;
uint32_t imm;
union {
/* RNDV info */
struct {
/* pointer to request on sender side */
- void *id;
+ uint64_t id;
/* registered buffer on sender side */
void* remote_addr;
/* registered mr of local src_addr */
@@ -190,37 +191,76 @@ struct fi_ibv_rdm_tagged_request {
};
static inline void
-fi_ibv_rdm_tagged_zero_request(struct fi_ibv_rdm_tagged_request *request)
+fi_ibv_rdm_zero_request(struct fi_ibv_rdm_request *request)
{
memset(request, 0, sizeof(*request));
}
-void fi_ibv_rdm_tagged_print_request(char *buf,
- struct fi_ibv_rdm_tagged_request *request);
+void fi_ibv_rdm_print_request(char *buf, struct fi_ibv_rdm_request *request);
+
+#define BUF_STATUS_FREE ((uint16_t) 0)
+#define BUF_STATUS_BUSY ((uint16_t) 1)
+#define BUF_STATUS_RECVED ((uint16_t) 2)
+
+struct fi_ibv_rdm_buf_service_data {
+ volatile uint16_t status;
+ uint16_t seq_num;
+ int32_t pkt_len;
+};
+
+#define FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE \
+ (offsetof(struct fi_ibv_rdm_buf, header))
struct fi_ibv_rdm_buf {
+ struct fi_ibv_rdm_buf_service_data service_data;
struct fi_ibv_rdm_header header;
- char payload[sizeof(void *)];
+ uint8_t payload;
};
struct fi_ibv_rdm_cm {
- struct rdma_cm_id *listener;
struct rdma_event_channel *ec;
- struct sockaddr_in my_addr;
- struct rdma_addrinfo *rai;
+ struct rdma_cm_id *listener;
+ int is_bound;
+
+ /* conn_hash has a sockaddr_in -> conn associative */
+ struct fi_ibv_rdm_conn *conn_hash;
+ /* Used only for FI_AV_TABLE */
+ struct fi_ibv_rdm_conn **conn_table;
+};
+
+struct fi_ibv_rdm_cntr {
+ struct fid_cntr fid;
+ struct fi_ibv_domain *domain;
+ atomic_t ep_ref;
+ uint64_t value;
+ struct fi_cntr_attr attr;
+ uint64_t err_count;
};
struct fi_ibv_rdm_ep {
struct fid_ep ep_fid;
struct fi_ibv_domain *domain;
- struct fi_ibv_cq *fi_scq;
- struct fi_ibv_cq *fi_rcq;
+ struct fi_ibv_rdm_cq *fi_scq;
+ struct fi_ibv_rdm_cq *fi_rcq;
+
+ struct fi_ibv_rdm_cntr *send_cntr;
+ struct fi_ibv_rdm_cntr *recv_cntr;
+ struct fi_ibv_rdm_cntr *read_cntr;
+ struct fi_ibv_rdm_cntr *write_cntr;
- struct fi_ibv_rdm_cm cm;
size_t addrlen;
+ struct rdma_addrinfo *rai;
+ struct sockaddr_in my_addr;
struct fi_ibv_av *av;
+ int tx_selective_completion;
+ int rx_selective_completion;
+ /*
+ * ibv_post_send opcode for eager messaging.
+ * It must generate work completion in receive CQ
+ */
+ enum ibv_wr_opcode eopcode;
int buff_len;
int n_buffs;
int rq_wr_depth; // RQ depth
@@ -230,12 +270,17 @@ struct fi_ibv_rdm_ep {
int num_active_conns;
int max_inline_rc;
int rndv_threshold;
+ int rndv_seg_size;
struct ibv_cq *scq;
struct ibv_cq *rcq;
int scq_depth;
int rcq_depth;
+ int cqread_bunch_size;
+
+ /* TODO: move all CM things to domain */
pthread_t cm_progress_thread;
pthread_mutex_t cm_lock;
+ int cm_progress_timeout;
int is_closing;
int recv_preposted_threshold;
};
@@ -256,7 +301,7 @@ enum fi_rdm_cm_role {
FI_VERBS_CM_SELF,
};
-struct fi_ibv_rdm_tagged_conn {
+struct fi_ibv_rdm_conn {
/*
* In normal case only qp[0] and id[0] are used.
@@ -270,20 +315,21 @@ struct fi_ibv_rdm_tagged_conn {
int state;
char *sbuf_mem_reg;
- char *sbuf_head;
- char *sbuf_ack_head;
+ struct fi_ibv_rdm_buf *sbuf_head;
+ uint16_t sbuf_ack_status;
char *rbuf_mem_reg;
- char *rbuf_head;
+ struct fi_ibv_rdm_buf *rbuf_head;
char *rmabuf_mem_reg;
- char *rmabuf_head;
+ struct fi_ibv_rdm_buf *rmabuf_head;
struct dlist_entry postponed_requests_head;
- struct fi_ibv_rdm_tagged_postponed_entry *postponed_entry;
+ struct fi_ibv_rdm_postponed_entry *postponed_entry;
struct ibv_mr *s_mr;
struct ibv_mr *r_mr;
+ struct ibv_mr *ack_mr;
struct ibv_mr *rma_mr;
uint32_t remote_sbuf_rkey;
@@ -291,11 +337,14 @@ struct fi_ibv_rdm_tagged_conn {
char *remote_sbuf_mem_reg;
char *remote_rbuf_mem_reg;
- char *remote_sbuf_head;
+ struct fi_ibv_rdm_buf *remote_sbuf_head;
int sends_outgoing;
int recv_preposted;
- int recv_completions; /* counter for eager buffer releasing */
+ /* counter for eager buffer releasing */
+ uint16_t recv_completions;
+ /* counter to control OOO behaviour, works in pair with recv_completions */
+ uint16_t recv_processed;
UT_hash_handle hh;
#if ENABLE_DEBUG
size_t unexp_counter;
@@ -303,80 +352,56 @@ struct fi_ibv_rdm_tagged_conn {
#endif
};
-struct fi_ibv_rdm_tagged_postponed_entry {
+struct fi_ibv_rdm_postponed_entry {
struct dlist_entry queue_entry;
- struct fi_ibv_rdm_tagged_conn *conn;
-};
-
-enum fi_ibv_rdm_tagged_buffer_status {
- BUF_STATUS_FREE = 0,
- BUF_STATUS_BUSY
-};
-
-struct fi_ibv_rdm_tagged_buffer_service_data {
- volatile enum fi_ibv_rdm_tagged_buffer_status status;
- int seq_number;
+ struct fi_ibv_rdm_conn *conn;
};
-#define FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE \
- (sizeof(struct fi_ibv_rdm_tagged_buffer_service_data) \
- < FI_IBV_RDM_MEM_ALIGNMENT ? FI_IBV_RDM_MEM_ALIGNMENT \
- : (sizeof(struct fi_ibv_rdm_tagged_buffer_service_data) + \
- (FI_IBV_RDM_MEM_ALIGNMENT - \
- sizeof(struct fi_ibv_rdm_tagged_buffer_service_data) % \
- FI_IBV_RDM_MEM_ALIGNMENT)))
-
-static inline struct fi_ibv_rdm_tagged_buffer_service_data *
-fi_ibv_rdm_tagged_get_buff_service_data(char *buff)
-{
- return (struct fi_ibv_rdm_tagged_buffer_service_data *)
- (buff - FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE);
-}
-
static inline void
-fi_ibv_rdm_tagged_set_buffer_status(char *buff,
- enum fi_ibv_rdm_tagged_buffer_status status)
+fi_ibv_rdm_set_buffer_status(struct fi_ibv_rdm_buf *buff, uint16_t status)
{
- fi_ibv_rdm_tagged_get_buff_service_data(buff)->status = status;
+ buff->service_data.status = status;
+ if (status == BUF_STATUS_FREE) {
+ buff->service_data.pkt_len = 0;
+ }
}
-static inline enum fi_ibv_rdm_tagged_buffer_status
-fi_ibv_rdm_tagged_get_buffer_status(char *buff)
+static inline int
+fi_ibv_rdm_buffer_check_seq_num(struct fi_ibv_rdm_buf *buff, uint16_t seq_num)
{
- return fi_ibv_rdm_tagged_get_buff_service_data(buff)->status;
+ VERBS_DBG(FI_LOG_EP_DATA, "seq num: %d <-> %d\n",
+ buff->service_data.seq_num, seq_num);
+ return (seq_num == buff->service_data.seq_num);
}
static inline uintptr_t
-fi_ibv_rdm_tagged_get_remote_addr(struct fi_ibv_rdm_tagged_conn *conn,
- char *local_sbuff)
+fi_ibv_rdm_get_remote_addr(struct fi_ibv_rdm_conn *conn,
+ struct fi_ibv_rdm_buf *local_sbuff)
{
return (uintptr_t) (conn->remote_rbuf_mem_reg +
- (local_sbuff - conn->sbuf_mem_reg));
+ ((char *)local_sbuff - conn->sbuf_mem_reg));
}
static inline void
-fi_ibv_rdm_push_buff_pointer(char *area_start, size_t area_size, char **buff,
- size_t offset)
+fi_ibv_rdm_push_buff_pointer(char *area_start, size_t area_size,
+ struct fi_ibv_rdm_buf **rdm_buff, size_t offset)
{
- char *buff_tmp = (*buff) + offset;
+ char *buff = (char*)(*rdm_buff);
+ char *buff_tmp = buff + offset;
- VERBS_DBG(FI_LOG_EP_DATA, "old_pointer: %p, sn = %d\n", *buff,
- fi_ibv_rdm_tagged_get_buff_service_data(*buff)->seq_number);
+ VERBS_DBG(FI_LOG_EP_DATA, "old_pointer: %p\n", *buff);
- (*buff) = buff_tmp < (area_start + area_size) ?
- buff_tmp : area_start + FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE;
+ buff = buff_tmp < (area_start + area_size) ? buff_tmp : area_start;
+
+ VERBS_DBG(FI_LOG_EP_DATA, "new_pointer: %p\n", *buff);
- VERBS_DBG(FI_LOG_EP_DATA, "new_pointer: %p, sn = %d\n", *buff,
- fi_ibv_rdm_tagged_get_buff_service_data(*buff)->seq_number);
- assert(fi_ibv_rdm_tagged_get_buff_service_data(*buff)->seq_number ==
- (((*buff) - (area_start + FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE)) /
- offset));
+ *rdm_buff = (struct fi_ibv_rdm_buf *)buff;
}
static inline void
-fi_ibv_rdm_tagged_push_sbuff_head(struct fi_ibv_rdm_tagged_conn *conn,
- struct fi_ibv_rdm_ep *ep)
+fi_ibv_rdm_push_sbuff_head(struct fi_ibv_rdm_conn *conn,
+ struct fi_ibv_rdm_ep *ep)
{
fi_ibv_rdm_push_buff_pointer(conn->sbuf_mem_reg,
ep->buff_len * ep->n_buffs,
@@ -384,7 +409,7 @@ fi_ibv_rdm_tagged_push_sbuff_head(struct fi_ibv_rdm_tagged_conn *conn,
}
static inline void
-fi_ibv_rdm_push_rmabuff_head(struct fi_ibv_rdm_tagged_conn *conn,
+fi_ibv_rdm_push_rmabuff_head(struct fi_ibv_rdm_conn *conn,
struct fi_ibv_rdm_ep *ep)
{
fi_ibv_rdm_push_buff_pointer(conn->rmabuf_mem_reg,
@@ -393,81 +418,100 @@ fi_ibv_rdm_push_rmabuff_head(struct fi_ibv_rdm_tagged_conn *conn,
}
static inline struct fi_ibv_rdm_buf *
-fi_ibv_rdm_tagged_get_rbuf(struct fi_ibv_rdm_tagged_conn *conn,
- struct fi_ibv_rdm_ep *ep,
- int seq_num)
+fi_ibv_rdm_get_rmabuf(struct fi_ibv_rdm_conn *conn,
+ struct fi_ibv_rdm_ep *ep, uint16_t seq_num)
{
- char *rbuf = (conn->rbuf_mem_reg +
- FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE +
- (seq_num * ep->buff_len));
- VERBS_DBG(FI_LOG_EP_DATA, "recv buf %d\n", seq_num);
- assert(fi_ibv_rdm_tagged_get_buff_service_data(rbuf)->seq_number ==
- seq_num);
- return (struct fi_ibv_rdm_buf *) rbuf;
+ char *rmabuf = conn->rmabuf_mem_reg + (seq_num * ep->buff_len);
+ VERBS_DBG(FI_LOG_EP_DATA, "rma buf %d\n", seq_num);
+ return (struct fi_ibv_rdm_buf *) rmabuf;
}
-static inline void *
-fi_ibv_rdm_tagged_get_sbuf(struct fi_ibv_rdm_tagged_conn *conn,
- struct fi_ibv_rdm_ep *ep,
- int seq_num)
+static inline struct fi_ibv_rdm_buf *
+fi_ibv_rdm_get_rbuf(struct fi_ibv_rdm_conn *conn,
+ struct fi_ibv_rdm_ep *ep, uint16_t seq_num)
+{
+ struct fi_ibv_rdm_buf *rbuf = (struct fi_ibv_rdm_buf *)
+ (conn->rbuf_mem_reg + (seq_num * ep->buff_len));
+
+ VERBS_DBG(FI_LOG_EP_DATA, "recv buf %d <-> %d\n",
+ seq_num, rbuf->service_data.seq_num);
+
+ return rbuf;
+}
+
+static inline struct fi_ibv_rdm_buf *
+fi_ibv_rdm_get_sbuf(struct fi_ibv_rdm_conn *conn,
+ struct fi_ibv_rdm_ep *ep, uint16_t seq_num)
{
- char * sbuf = (conn->sbuf_mem_reg +
- FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE +
- (seq_num * ep->buff_len));
+ char *sbuf = conn->sbuf_mem_reg + (seq_num * ep->buff_len);
VERBS_DBG(FI_LOG_EP_DATA, "send buf %d\n", seq_num);
- return (struct fi_ibv_rdm_buf *) sbuf;
+ return (struct fi_ibv_rdm_buf *)sbuf;
}
static inline void
-fi_ibv_rdm_tagged_buffer_lists_init(struct fi_ibv_rdm_tagged_conn *conn,
- struct fi_ibv_rdm_ep *ep)
+fi_ibv_rdm_buffer_lists_init(struct fi_ibv_rdm_conn *conn,
+ struct fi_ibv_rdm_ep *ep)
{
int i;
- conn->sbuf_head = conn->sbuf_mem_reg +
- FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE;
- conn->rbuf_head = conn->rbuf_mem_reg +
- FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE;
- conn->sbuf_ack_head = conn->rbuf_head; // used only service data
+ conn->sbuf_head = (struct fi_ibv_rdm_buf *)conn->sbuf_mem_reg;
+ conn->rbuf_head = (struct fi_ibv_rdm_buf *)conn->rbuf_mem_reg;
+ conn->sbuf_ack_status = BUF_STATUS_FREE;
- conn->rmabuf_head = conn->rmabuf_mem_reg +
- FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE;
+ conn->rmabuf_head = (struct fi_ibv_rdm_buf *)conn->rmabuf_mem_reg;
for (i = 0; i < ep->n_buffs; ++i) {
- fi_ibv_rdm_tagged_set_buffer_status(conn->sbuf_head +
- i * ep->buff_len, BUF_STATUS_FREE);
- fi_ibv_rdm_tagged_set_buffer_status(conn->sbuf_ack_head +
- i * ep->buff_len, BUF_STATUS_FREE);
-
- fi_ibv_rdm_tagged_get_buff_service_data(conn->sbuf_head +
- i * ep->buff_len)->seq_number = i;
- fi_ibv_rdm_tagged_get_buff_service_data(conn->sbuf_ack_head +
- i * ep->buff_len)->seq_number = i;
-
- fi_ibv_rdm_tagged_get_buff_service_data(conn->rmabuf_head +
- i * ep->buff_len)->seq_number = i;
+ fi_ibv_rdm_set_buffer_status(fi_ibv_rdm_get_sbuf(conn, ep, i),
+ BUF_STATUS_FREE);
+ fi_ibv_rdm_get_sbuf(conn, ep, i)->service_data.seq_num = i;
+
+ fi_ibv_rdm_set_buffer_status(fi_ibv_rdm_get_rbuf(conn, ep, i),
+ BUF_STATUS_FREE);
+ /* should be initialized by sender */
+ fi_ibv_rdm_get_rbuf(conn, ep, i)->service_data.seq_num =
+ (uint16_t)(-1);
+
+ fi_ibv_rdm_set_buffer_status(fi_ibv_rdm_get_rmabuf(conn, ep, i),
+ BUF_STATUS_FREE);
+ fi_ibv_rdm_get_rmabuf(conn, ep, i)->service_data.seq_num = i;
+ }
+}
+
+static inline void fi_ibv_rdm_cntr_inc(struct fi_ibv_rdm_cntr *cntr)
+{
+ if (cntr) {
+ cntr->fid.ops->add(&cntr->fid, 1);
+ }
+}
+
+static inline void fi_ibv_rdm_cntr_inc_err(struct fi_ibv_rdm_cntr *cntr)
+{
+ if (cntr) {
+ cntr->err_count++;
}
}
int fi_ibv_rdm_tagged_poll(struct fi_ibv_rdm_ep *ep);
-int fi_ibv_rdm_tagged_cm_progress(struct fi_ibv_rdm_ep *ep);
-int fi_ibv_rdm_start_disconnection(struct fi_ibv_rdm_tagged_conn *conn);
-int fi_ibv_rdm_tagged_conn_cleanup(struct fi_ibv_rdm_tagged_conn *conn);
-int fi_ibv_rdm_start_connection(struct fi_ibv_rdm_ep *ep,
- struct fi_ibv_rdm_tagged_conn *conn);
-int fi_ibv_rdm_tagged_repost_receives(struct fi_ibv_rdm_tagged_conn *conn,
- struct fi_ibv_rdm_ep *ep,
- int num_to_post);
+ssize_t fi_ibv_rdm_cm_progress(struct fi_ibv_rdm_ep *ep);
+ssize_t fi_ibv_rdm_start_disconnection(struct fi_ibv_rdm_conn *conn);
+ssize_t fi_ibv_rdm_conn_cleanup(struct fi_ibv_rdm_conn *conn);
+ssize_t fi_ibv_rdm_start_connection(struct fi_ibv_rdm_ep *ep,
+ struct fi_ibv_rdm_conn *conn);
+ssize_t fi_ibv_rdm_repost_receives(struct fi_ibv_rdm_conn *conn,
+ struct fi_ibv_rdm_ep *ep,
+ int num_to_post);
int fi_ibv_rdm_tagged_open_ep(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context);
-int fi_ibv_rdm_tagged_prepare_send_request(
- struct fi_ibv_rdm_tagged_request *request, struct fi_ibv_rdm_ep *ep);
-int fi_ibv_rdm_prepare_rma_request(struct fi_ibv_rdm_tagged_request *request,
+int fi_ibv_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
+ struct fid_cq **cq, void *context);
+
+int fi_ibv_rdm_tagged_prepare_send_request(struct fi_ibv_rdm_request *request,
+ struct fi_ibv_rdm_ep *ep);
+int fi_ibv_rdm_prepare_rma_request(struct fi_ibv_rdm_request *request,
struct fi_ibv_rdm_ep *ep);
-static inline void *fi_ibv_rdm_tagged_get_sbuf_head(
- struct fi_ibv_rdm_tagged_conn
- *conn, struct fi_ibv_rdm_ep *ep)
+static inline struct fi_ibv_rdm_buf *
+fi_ibv_rdm_get_sbuf_head(struct fi_ibv_rdm_conn *conn, struct fi_ibv_rdm_ep *ep)
{
assert(conn);
@@ -479,47 +523,36 @@ static inline void *fi_ibv_rdm_tagged_get_sbuf_head(
sprintf(p, "N:%1d ", ep->n_buffs);
p += 4;
for (i = 0; i < ep->n_buffs; ++i, p += 4) {
- sprintf(p, "%1d:%1d ",
- fi_ibv_rdm_tagged_get_buff_service_data(
- conn->sbuf_mem_reg +
- FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE +
- i * ep->buff_len)->seq_number,
- fi_ibv_rdm_tagged_get_buffer_status(
- conn->sbuf_mem_reg +
- FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE +
- i * ep->buff_len));
+ struct fi_ibv_rdm_buf *buf =
+ fi_ibv_rdm_get_sbuf(conn, ep, i);
+ sprintf(p, "%1d:%1d ", buf->service_data.seq_num,
+ buf->service_data.status);
}
VERBS_DBG(FI_LOG_EP_DATA,
"conn %p sbufs status before: %s\n", conn, s);
}
#endif // ENABLE_DEBUG
- int i = 0;
- void *sbuf = NULL;
+ struct fi_ibv_rdm_buf *sbuf = NULL;
- if (fi_ibv_rdm_tagged_get_buffer_status(conn->sbuf_head) ==
- BUF_STATUS_FREE) {
+ if (conn->sbuf_head->service_data.status == BUF_STATUS_FREE) {
/* We have made whole circle. Reset buffer states */
- if (conn->sbuf_head ==
- fi_ibv_rdm_tagged_get_sbuf(conn, ep, 0)) {
- for (i = 1; i < ep->n_buffs; ++i) {
- fi_ibv_rdm_tagged_set_buffer_status(
- conn->sbuf_mem_reg +
- FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE +
- i * ep->buff_len, BUF_STATUS_FREE);
- }
- }
-
- fi_ibv_rdm_tagged_set_buffer_status(conn->sbuf_head,
- BUF_STATUS_BUSY);
+ if (conn->sbuf_head == fi_ibv_rdm_get_sbuf(conn, ep, 0)) {
+ do {
+ fi_ibv_rdm_set_buffer_status(conn->sbuf_head,
+ BUF_STATUS_FREE);
+ fi_ibv_rdm_push_sbuff_head(conn, ep);
+ } while (conn->sbuf_head != fi_ibv_rdm_get_sbuf(conn, ep, 0));
+ }
+
+ /* notification for receiver */
+ fi_ibv_rdm_set_buffer_status(conn->sbuf_head, BUF_STATUS_RECVED);
sbuf = conn->sbuf_head;
- fi_ibv_rdm_tagged_push_sbuff_head(conn, ep);
+ fi_ibv_rdm_push_sbuff_head(conn, ep);
}
#if ENABLE_DEBUG
- assert(sbuf
- ? (fi_ibv_rdm_tagged_get_buffer_status(sbuf) == BUF_STATUS_BUSY)
- : 1);
+ assert(sbuf ? (sbuf->service_data.status == BUF_STATUS_RECVED) : 1);
{
int i;
char s[1024];
@@ -527,53 +560,37 @@ static inline void *fi_ibv_rdm_tagged_get_sbuf_head(
sprintf(p, "N:%1d ", ep->n_buffs);
p += 4;
for (i = 0; i < ep->n_buffs; ++i, p += 4) {
- sprintf(p, "%1d:%1d ",
- fi_ibv_rdm_tagged_get_buff_service_data(
- conn->sbuf_mem_reg +
- FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE +
- i * ep->buff_len)->seq_number,
- fi_ibv_rdm_tagged_get_buffer_status(
- conn->sbuf_mem_reg +
- FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE +
- i * ep->buff_len));
+ struct fi_ibv_rdm_buf *buf =
+ fi_ibv_rdm_get_sbuf(conn, ep, i);
+ sprintf(p, "%1d:%1d ", buf->service_data.seq_num,
+ buf->service_data.status);
}
VERBS_DBG(FI_LOG_EP_DATA,
"conn %p sbufs status after: %s\n", conn, s);
}
- static int seq_number = 0;
if (sbuf) {
- struct fi_ibv_rdm_buf *stbuf = (struct fi_ibv_rdm_buf *)sbuf;
- seq_number++;
- stbuf->header.seq_num = seq_number;
- VERBS_DBG(FI_LOG_EP_DATA, "sending pkt # %d\n", stbuf->header.seq_num);
+ VERBS_DBG(FI_LOG_EP_DATA, "sending pkt # %d\n",
+ sbuf->service_data.seq_num);
}
#endif // ENABLE_DEBUG
VERBS_DBG(FI_LOG_EP_DATA,
- "conn %p sbuf allocated: %d:%p, head: %p, begin: %p\n",
- conn,
- (sbuf) ?
- fi_ibv_rdm_tagged_get_buff_service_data(sbuf)->
- seq_number : -1, sbuf, conn->sbuf_head,
- conn->sbuf_mem_reg);
+ "conn %p sbuf allocated: %p, head: %p, begin: %p\n",
+ conn, sbuf, conn->sbuf_head, conn->sbuf_mem_reg);
return sbuf;
}
static inline void *
-fi_ibv_rdm_rma_get_buf_head(struct fi_ibv_rdm_tagged_conn *conn,
+fi_ibv_rdm_rma_get_buf_head(struct fi_ibv_rdm_conn *conn,
struct fi_ibv_rdm_ep *ep)
{
assert(conn);
void *buf = NULL;
- if (fi_ibv_rdm_tagged_get_buffer_status(conn->rmabuf_head) ==
- BUF_STATUS_FREE) {
-
- fi_ibv_rdm_tagged_set_buffer_status(conn->rmabuf_head,
- BUF_STATUS_BUSY);
-
+ if (conn->rmabuf_head->service_data.status == BUF_STATUS_FREE) {
+ fi_ibv_rdm_set_buffer_status(conn->rmabuf_head, BUF_STATUS_BUSY);
buf = conn->rmabuf_head;
fi_ibv_rdm_push_rmabuff_head(conn, ep);
}
@@ -581,7 +598,7 @@ fi_ibv_rdm_rma_get_buf_head(struct fi_ibv_rdm_tagged_conn *conn,
}
static inline int
-fi_ibv_rdm_check_connection(struct fi_ibv_rdm_tagged_conn *conn,
+fi_ibv_rdm_check_connection(struct fi_ibv_rdm_conn *conn,
struct fi_ibv_rdm_ep *ep)
{
const int status = (conn->state == FI_VERBS_CONN_ESTABLISHED);
@@ -591,25 +608,24 @@ fi_ibv_rdm_check_connection(struct fi_ibv_rdm_tagged_conn *conn,
fi_ibv_rdm_start_connection(ep, conn);
}
pthread_mutex_unlock(&ep->cm_lock);
- usleep(1000);
}
return status;
}
-static inline void *
-fi_ibv_rdm_tagged_prepare_send_resources(struct fi_ibv_rdm_tagged_conn *conn,
- struct fi_ibv_rdm_ep *ep)
+static inline struct fi_ibv_rdm_buf *
+fi_ibv_rdm_prepare_send_resources(struct fi_ibv_rdm_conn *conn,
+ struct fi_ibv_rdm_ep *ep)
{
if (fi_ibv_rdm_check_connection(conn, ep)) {
- return (!SEND_RESOURCES_IS_BUSY(conn, ep)) ?
- fi_ibv_rdm_tagged_get_sbuf_head(conn, ep) : NULL;
+ return (!TSEND_RESOURCES_IS_BUSY(conn, ep)) ?
+ fi_ibv_rdm_get_sbuf_head(conn, ep) : NULL;
}
return NULL;
}
static inline void *
-fi_ibv_rdm_rma_prepare_resources(struct fi_ibv_rdm_tagged_conn *conn,
+fi_ibv_rdm_rma_prepare_resources(struct fi_ibv_rdm_conn *conn,
struct fi_ibv_rdm_ep *ep)
{
if (fi_ibv_rdm_check_connection(conn, ep)) {
diff --git a/prov/verbs/src/ep_rdm/verbs_rdm_cm.c b/prov/verbs/src/ep_rdm/verbs_rdm_cm.c
index b997861..5213fbc 100644
--- a/prov/verbs/src/ep_rdm/verbs_rdm_cm.c
+++ b/prov/verbs/src/ep_rdm/verbs_rdm_cm.c
@@ -40,8 +40,6 @@
extern struct fi_provider fi_ibv_prov;
-extern struct fi_ibv_rdm_tagged_conn *fi_ibv_rdm_tagged_conn_hash;
-
static struct ibv_mr *
fi_ibv_rdm_alloc_and_reg(struct fi_ibv_rdm_ep *ep, void **buf, size_t size)
@@ -56,82 +54,94 @@ fi_ibv_rdm_alloc_and_reg(struct fi_ibv_rdm_ep *ep, void **buf, size_t size)
return NULL;
}
-static int
+static ssize_t
fi_ibv_rdm_dereg_and_free(struct ibv_mr **mr, char **buff)
{
- int ret = ibv_dereg_mr(*mr);
-
- if (ret == 0) {
- *mr = NULL;
- free(*buff);
- *buff = NULL;
+ ssize_t ret = FI_SUCCESS;
+ if (ibv_dereg_mr(*mr)) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "ibv_dereg_mr failed\n", errno);
+ ret = -errno;
}
+ *mr = NULL;
+ free(*buff);
+ *buff = NULL;
+
return ret;
}
-static inline int
-fi_ibv_rdm_tagged_batch_repost_receives(struct fi_ibv_rdm_tagged_conn *conn,
- struct fi_ibv_rdm_ep *ep,
- int num_to_post)
+static inline ssize_t
+fi_ibv_rdm_batch_repost_receives(struct fi_ibv_rdm_conn *conn,
+ struct fi_ibv_rdm_ep *ep, int num_to_post)
{
const size_t idx = (conn->cm_role == FI_VERBS_CM_SELF) ? 1 : 0;
struct ibv_recv_wr *bad_wr = NULL;
struct ibv_recv_wr wr[num_to_post];
- int ret = 0;
+ struct ibv_sge sge[num_to_post];
int last = num_to_post - 1;
int i;
+ /* IBV_WR_SEND opcode specific */
+ assert((num_to_post % ep->n_buffs) == 0);
+
+ assert(ep->eopcode == IBV_WR_SEND ||
+ ep->eopcode == IBV_WR_RDMA_WRITE_WITH_IMM);
+
+ if (ep->eopcode == IBV_WR_SEND) {
+ for (i = 0; i < num_to_post; i++) {
+ sge[i].addr = (uint64_t)(void *)
+ fi_ibv_rdm_get_rbuf(conn, ep, i % ep->n_buffs);
+ sge[i].length = FI_IBV_RDM_DFLT_BUFFER_SIZE;
+ sge[i].lkey = conn->r_mr->lkey;
+ }
+ }
+
for (i = 0; i < num_to_post; i++) {
wr[i].wr_id = (uintptr_t) conn;
wr[i].next = &wr[i + 1];
- wr[i].sg_list = NULL;
- wr[i].num_sge = 0;
+ wr[i].sg_list = &sge[i];
+ wr[i].num_sge = 1;
}
wr[last].next = NULL;
if (ibv_post_recv(conn->qp[idx], wr, &bad_wr) == 0) {
- ret = num_to_post;
- } else {
- int found = 0;
- for (i = 0; !found && (i < num_to_post); i++) {
- found = (&wr[i] == bad_wr);
- }
-
- if (!found) {
- VERBS_INFO(FI_LOG_EP_DATA, "Failed to post recv\n");
- }
-
- ret = i;
+ conn->recv_preposted += num_to_post;
+ return num_to_post;
}
- conn->recv_preposted += ret;
- return ret;
+
+ VERBS_INFO(FI_LOG_EP_DATA, "Failed to post recv\n");
+ return -FI_ENOMEM;
}
-int fi_ibv_rdm_tagged_repost_receives(struct fi_ibv_rdm_tagged_conn *conn,
- struct fi_ibv_rdm_ep *ep, int num_to_post)
+ssize_t fi_ibv_rdm_repost_receives(struct fi_ibv_rdm_conn *conn,
+ struct fi_ibv_rdm_ep *ep, int num_to_post)
{
assert(num_to_post > 0);
- const int batch_size = 100;
+ const ssize_t batch_size = ep->n_buffs * 10;
- int rest = num_to_post;
+ ssize_t rest = num_to_post - (num_to_post % ep->n_buffs);
+ ssize_t count = 0;
while (rest) {
- const int batch = MIN(rest, batch_size);
- const int ret =
- fi_ibv_rdm_tagged_batch_repost_receives(conn, ep, batch);
+ const ssize_t batch = MIN(rest, batch_size);
+ const ssize_t ret =
+ fi_ibv_rdm_batch_repost_receives(conn, ep, batch);
- rest -= ret;
- if (ret != batch) {
- break;
+ if (ret < 0) {
+ return ret;
}
+
+ count += ret;
+ rest -= ret;
+
+ assert(ret == batch);
}
- return num_to_post - rest;
+ return count;
}
-static inline int
-fi_ibv_rdm_tagged_prepare_conn_memory(struct fi_ibv_rdm_ep *ep,
- struct fi_ibv_rdm_tagged_conn *conn)
+static ssize_t
+fi_ibv_rdm_prepare_conn_memory(struct fi_ibv_rdm_ep *ep,
+ struct fi_ibv_rdm_conn *conn)
{
assert(conn->s_mr == NULL);
assert(conn->r_mr == NULL);
@@ -139,18 +149,51 @@ fi_ibv_rdm_tagged_prepare_conn_memory(struct fi_ibv_rdm_ep *ep,
const size_t size = ep->buff_len * ep->n_buffs;
conn->s_mr = fi_ibv_rdm_alloc_and_reg(ep,
(void **) &conn->sbuf_mem_reg, size);
- assert(conn->s_mr);
+ if (!conn->s_mr) {
+ assert(conn->s_mr);
+ goto s_err;
+ }
conn->r_mr = fi_ibv_rdm_alloc_and_reg(ep,
(void **) &conn->rbuf_mem_reg, size);
- assert(conn->r_mr);
+ if (!conn->r_mr) {
+ assert(conn->r_mr);
+ goto r_err;
+ }
+
+ conn->ack_mr = ibv_reg_mr(ep->domain->pd, &conn->sbuf_ack_status,
+ sizeof(conn->sbuf_ack_status),
+ IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
+
+ if (!conn->ack_mr) {
+ assert(conn->ack_mr);
+ goto ack_err;
+ }
conn->rma_mr = fi_ibv_rdm_alloc_and_reg(ep,
(void **) &conn->rmabuf_mem_reg, size);
- assert(conn->rma_mr);
+ if (!conn->rma_mr) {
+ assert(conn->rma_mr);
+ goto rma_err;
+ }
- fi_ibv_rdm_tagged_buffer_lists_init(conn, ep);
- return 0;
+ fi_ibv_rdm_buffer_lists_init(conn, ep);
+
+ return FI_SUCCESS;
+
+/* Error handling */
+rma_err:
+ free(conn->rmabuf_mem_reg);
+ack_err: /*
+ * Ack buffer is a part of connection structure, freeing is not needed
+ */
+r_err:
+ free(conn->rbuf_mem_reg);
+s_err:
+ free(conn->sbuf_mem_reg);
+
+ /* The is a lack of host or HCA memory */
+ return -FI_ENOMEM;
}
static inline void
@@ -172,7 +215,7 @@ fi_ibv_rdm_tagged_init_qp_attributes(struct ibv_qp_init_attr *qp_attr,
static inline void
fi_ibv_rdm_pack_cm_params(struct rdma_conn_param *cm_params,
- struct fi_ibv_rdm_tagged_conn *conn,
+ struct fi_ibv_rdm_conn *conn,
struct fi_ibv_rdm_ep *ep)
{
memset(cm_params, 0, sizeof(struct rdma_conn_param));
@@ -191,7 +234,7 @@ fi_ibv_rdm_pack_cm_params(struct rdma_conn_param *cm_params,
cm_params->private_data = malloc(cm_params->private_data_len);
char *p = (char *) cm_params->private_data;
- memcpy(p, &ep->cm.my_addr, FI_IBV_RDM_DFLT_ADDRLEN);
+ memcpy(p, &ep->my_addr, FI_IBV_RDM_DFLT_ADDRLEN);
p += FI_IBV_RDM_DFLT_ADDRLEN;
if ((conn->cm_role != FI_VERBS_CM_SELF) && (conn->r_mr && conn->s_mr)) {
@@ -210,14 +253,14 @@ fi_ibv_rdm_pack_cm_params(struct rdma_conn_param *cm_params,
static inline void
fi_ibv_rdm_unpack_cm_params(struct rdma_conn_param *cm_param,
- struct fi_ibv_rdm_tagged_conn *conn,
+ struct fi_ibv_rdm_conn *conn,
struct fi_ibv_rdm_ep *ep)
{
char *p = (char *)cm_param->private_data;
if (conn->cm_role == FI_VERBS_CM_SELF) {
if (conn->r_mr && conn->s_mr) {
- memcpy(&conn->addr, &ep->cm.my_addr,
+ memcpy(&conn->addr, &ep->my_addr,
FI_IBV_RDM_DFLT_ADDRLEN);
conn->remote_rbuf_rkey = conn->r_mr->rkey;
conn->remote_rbuf_mem_reg = conn->r_mr->addr;
@@ -225,8 +268,8 @@ fi_ibv_rdm_unpack_cm_params(struct rdma_conn_param *cm_param,
conn->remote_sbuf_rkey = conn->s_mr->rkey;
conn->remote_sbuf_mem_reg = conn->s_mr->addr;
- conn->remote_sbuf_head = conn->remote_sbuf_mem_reg +
- FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE;
+ conn->remote_sbuf_head = (struct fi_ibv_rdm_buf *)
+ conn->remote_sbuf_mem_reg;
}
} else {
if (conn->state == FI_VERBS_CONN_ALLOCATED) {
@@ -244,17 +287,18 @@ fi_ibv_rdm_unpack_cm_params(struct rdma_conn_param *cm_param,
conn->remote_sbuf_mem_reg = *(char **)(p);
p += sizeof(conn->remote_sbuf_mem_reg);
- conn->remote_sbuf_head = conn->remote_sbuf_mem_reg +
- FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE;
+ conn->remote_sbuf_head = (struct fi_ibv_rdm_buf *)
+ conn->remote_sbuf_mem_reg;
}
}
-static inline int
-fi_ibv_rdm_tagged_process_addr_resolved(struct rdma_cm_id *id,
- struct fi_ibv_rdm_ep *ep)
+static ssize_t
+fi_ibv_rdm_process_addr_resolved(struct rdma_cm_id *id,
+ struct fi_ibv_rdm_ep *ep)
{
+ ssize_t ret = FI_SUCCESS;
struct ibv_qp_init_attr qp_attr;
- struct fi_ibv_rdm_tagged_conn *conn = id->context;
+ struct fi_ibv_rdm_conn *conn = id->context;
VERBS_INFO(FI_LOG_AV, "ADDR_RESOLVED conn %p, addr %s:%u\n",
conn, inet_ntoa(conn->addr.sin_addr),
@@ -265,7 +309,9 @@ fi_ibv_rdm_tagged_process_addr_resolved(struct rdma_cm_id *id,
do {
fi_ibv_rdm_tagged_init_qp_attributes(&qp_attr, ep);
if (rdma_create_qp(id, ep->domain->pd, &qp_attr)) {
- VERBS_INFO(FI_LOG_AV, "rdma_create_qp failed\n");
+ VERBS_INFO_ERRNO(FI_LOG_AV,
+ "rdma_create_qp failed\n", errno);
+ return -errno;
}
if (conn->cm_role == FI_VERBS_CM_PASSIVE) {
@@ -278,50 +324,68 @@ fi_ibv_rdm_tagged_process_addr_resolved(struct rdma_cm_id *id,
break;
}
- fi_ibv_rdm_tagged_prepare_conn_memory(ep, conn);
- if (ep->rq_wr_depth != fi_ibv_rdm_tagged_repost_receives(conn,
- ep, ep->rq_wr_depth)) {
+ ret = fi_ibv_rdm_prepare_conn_memory(ep, conn);
+ if (ret != FI_SUCCESS) {
+ goto err;
+ }
+
+ ret = fi_ibv_rdm_repost_receives(conn, ep, ep->rq_wr_depth);
+ if (ret < 0) {
VERBS_INFO(FI_LOG_AV, "repost receives failed\n");
- return -FI_ENOMEM;
+ goto err;
+ } else {
+ ret = FI_SUCCESS;
}
} while (0);
if (rdma_resolve_route(id, FI_IBV_RDM_CM_RESOLVEADDR_TIMEOUT)) {
- fprintf(stderr, " rdma_resolve_route failed\n");
+ VERBS_INFO(FI_LOG_AV, "rdma_resolve_route failed\n");
+ ret = -FI_EHOSTUNREACH;
+ goto err;
}
- return 0;
+
+ return ret;
+err:
+ rdma_destroy_qp(id);
+ return ret;
}
-static inline int
-fi_ibv_rdm_tagged_process_connect_request(struct rdma_cm_event *event,
- struct fi_ibv_rdm_ep *ep)
+static ssize_t
+fi_ibv_rdm_process_connect_request(struct rdma_cm_event *event,
+ struct fi_ibv_rdm_ep *ep)
{
struct ibv_qp_init_attr qp_attr;
struct rdma_conn_param cm_params;
- struct fi_ibv_rdm_tagged_conn *conn = NULL;
+ struct fi_ibv_rdm_conn *conn = NULL;
struct rdma_cm_id *id = event->id;
- int ret = 0;
+ ssize_t ret = FI_SUCCESS;
char *p = (char *) event->param.conn.private_data;
if (ep->is_closing) {
int rej_message = 0xdeadbeef;
- if (rdma_reject(id, &rej_message, sizeof(int))) {
- fprintf(stderr, "rdma_reject failed\n");
- rdma_destroy_id(id);
+ if (rdma_reject(id, &rej_message, sizeof(rej_message))) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_reject\n", errno);
+ ret = -errno;
+ if (rdma_destroy_id(id)) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_destroy_id\n",
+ errno);
+ ret = (ret == FI_SUCCESS) ? -errno : ret;
+ }
}
+ assert(ret == FI_SUCCESS);
return ret;
}
- HASH_FIND(hh, fi_ibv_rdm_tagged_conn_hash, p, FI_IBV_RDM_DFLT_ADDRLEN,
+ HASH_FIND(hh, ep->domain->rdm_cm->conn_hash, p, FI_IBV_RDM_DFLT_ADDRLEN,
conn);
if (!conn) {
- conn = memalign(FI_IBV_RDM_MEM_ALIGNMENT, sizeof *conn);
+ conn = memalign(FI_IBV_RDM_MEM_ALIGNMENT, sizeof(*conn));
if (!conn)
return -FI_ENOMEM;
- memset(conn, 0, sizeof(struct fi_ibv_rdm_tagged_conn));
+ memset(conn, 0, sizeof(*conn));
conn->state = FI_VERBS_CONN_ALLOCATED;
dlist_init(&conn->postponed_requests_head);
@@ -333,8 +397,8 @@ fi_ibv_rdm_tagged_process_connect_request(struct rdma_cm_event *event,
conn, conn->cm_role, inet_ntoa(conn->addr.sin_addr),
ntohs(conn->addr.sin_port));
- HASH_ADD(hh, fi_ibv_rdm_tagged_conn_hash, addr,
- FI_IBV_RDM_DFLT_ADDRLEN, conn);
+ HASH_ADD(hh, ep->domain->rdm_cm->conn_hash, addr,
+ FI_IBV_RDM_DFLT_ADDRLEN, conn);
} else {
if (conn->cm_role != FI_VERBS_CM_ACTIVE) {
/*
@@ -353,12 +417,19 @@ fi_ibv_rdm_tagged_process_connect_request(struct rdma_cm_event *event,
if (conn->cm_role == FI_VERBS_CM_ACTIVE) {
int rej_message = 0xdeadbeef;
- if (rdma_reject(id, &rej_message, sizeof(int))) {
- fprintf(stderr, "rdma_reject failed\n");
- rdma_destroy_id(id);
+ if (rdma_reject(id, &rej_message, sizeof(rej_message))) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_reject\n", errno);
+ ret = -errno;
+ if (rdma_destroy_id(id)) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_destroy_id\n",
+ errno);
+ ret = (ret == FI_SUCCESS) ? -errno : ret;
+ }
}
if (conn->state == FI_VERBS_CONN_ALLOCATED) {
- fi_ibv_rdm_start_connection(ep, conn);
+ ret = fi_ibv_rdm_start_connection(ep, conn);
+ if (ret != FI_SUCCESS)
+ goto err;
}
} else {
assert(conn->state == FI_VERBS_CONN_ALLOCATED ||
@@ -372,37 +443,52 @@ fi_ibv_rdm_tagged_process_connect_request(struct rdma_cm_event *event,
assert (conn->id[idx] == NULL);
conn->id[idx] = id;
- fi_ibv_rdm_tagged_prepare_conn_memory(ep, conn);
+ ret = fi_ibv_rdm_prepare_conn_memory(ep, conn);
+ if (ret != FI_SUCCESS)
+ goto err;
+
fi_ibv_rdm_tagged_init_qp_attributes(&qp_attr, ep);
- rdma_create_qp(id, ep->domain->pd, &qp_attr);
+ if (rdma_create_qp(id, ep->domain->pd, &qp_attr)) {
+ ret = -errno;
+ goto err;
+ }
conn->qp[idx] = id->qp;
- if (ep->rq_wr_depth !=
- fi_ibv_rdm_tagged_repost_receives(conn, ep,
- ep->rq_wr_depth))
- {
+
+ ret = fi_ibv_rdm_repost_receives(conn, ep, ep->rq_wr_depth);
+ if (ret < 0) {
VERBS_INFO(FI_LOG_AV, "repost receives failed\n");
- return -FI_ENOMEM;
+ goto err;
+ } else {
+ ret = FI_SUCCESS;
}
id->context = conn;
fi_ibv_rdm_pack_cm_params(&cm_params, conn, ep);
- ret = rdma_accept(id, &cm_params);
- assert(ret == 0);
+ if (rdma_accept(id, &cm_params)) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_accept\n", errno);
+ ret = -errno;
+ goto err;
+ }
if (cm_params.private_data) {
free((void *) cm_params.private_data);
}
- }
+ }
+
+ return ret;
+err:
+ /* ret err code is already set here, just cleanup resources */
+ fi_ibv_rdm_conn_cleanup(conn);
return ret;
}
-static inline int
-fi_ibv_rdm_tagged_process_route_resolved(struct rdma_cm_event *event,
- struct fi_ibv_rdm_ep *ep)
+static ssize_t
+fi_ibv_rdm_process_route_resolved(struct rdma_cm_event *event,
+ struct fi_ibv_rdm_ep *ep)
{
- struct fi_ibv_rdm_tagged_conn *conn = event->id->context;
- int ret = 0;
+ struct fi_ibv_rdm_conn *conn = event->id->context;
+ ssize_t ret = FI_SUCCESS;
struct rdma_conn_param cm_params;
fi_ibv_rdm_pack_cm_params(&cm_params, conn, ep);
@@ -411,28 +497,32 @@ fi_ibv_rdm_tagged_process_route_resolved(struct rdma_cm_event *event,
"ROUTE RESOLVED, conn %p, addr %s:%u\n", conn,
inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port));
- ret = rdma_connect(event->id, &cm_params);
- assert(ret == 0);
+ if (rdma_connect(event->id, &cm_params)) {
+ VERBS_INFO_ERRNO(FI_LOG_AV,
+ "rdma_connect failed\n", errno);
+ ret = -errno;
- free((void *)cm_params.private_data);
+ free((void *)cm_params.private_data);
+ assert(0);
+ }
return ret;
}
-static inline int
-fi_ibv_rdm_tagged_process_event_established(struct rdma_cm_event *event,
- struct fi_ibv_rdm_ep *ep)
+static ssize_t
+fi_ibv_rdm_process_event_established(struct rdma_cm_event *event,
+ struct fi_ibv_rdm_ep *ep)
{
- struct fi_ibv_rdm_tagged_conn *conn =
- (struct fi_ibv_rdm_tagged_conn *)event->id->context;
-#if defined(ENABLE_DEBUG) && ENABLE_DEBUG > 0
+ struct fi_ibv_rdm_conn *conn =
+ (struct fi_ibv_rdm_conn *)event->id->context;
+
if (conn->state != FI_VERBS_CONN_STARTED &&
conn->cm_role != FI_VERBS_CM_SELF)
{
- fprintf(stderr, "state = %d, conn %p", conn->state, conn);
+ VERBS_INFO(FI_LOG_AV, "state = %d, conn %p", conn->state, conn);
assert(0 && "Wrong state");
+ return -FI_ECONNABORTED;
}
-#endif
if (conn->cm_role == FI_VERBS_CM_ACTIVE ||
conn->cm_role == FI_VERBS_CM_SELF)
@@ -449,50 +539,85 @@ fi_ibv_rdm_tagged_process_event_established(struct rdma_cm_event *event,
ep->num_active_conns++;
conn->state = FI_VERBS_CONN_ESTABLISHED;
}
- return 0;
+ return FI_SUCCESS;
}
-int fi_ibv_rdm_tagged_conn_cleanup(struct fi_ibv_rdm_tagged_conn *conn)
+ssize_t fi_ibv_rdm_conn_cleanup(struct fi_ibv_rdm_conn *conn)
{
- int ret = 0;
+ ssize_t ret = FI_SUCCESS;
+ ssize_t err = FI_SUCCESS;
VERBS_DBG(FI_LOG_AV, "conn %p, exp = %lld unexp = %lld\n", conn,
conn->exp_counter, conn->unexp_counter);
+ errno = 0;
if (conn->id[0]) {
rdma_destroy_qp(conn->id[0]);
- if ((ret = rdma_destroy_id(conn->id[0]))) {
- VERBS_INFO(FI_LOG_AV,
- "rdma_destroy_id failed, ret = %d\n", ret);
+ if (errno) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_destroy_qp\n", errno);
+ ret = -errno;
+ }
+
+ if (rdma_destroy_id(conn->id[0])) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_destroy_id\n", errno);
+ if (ret == FI_SUCCESS)
+ ret = -errno;
}
}
if (conn->id[1]) {
assert(conn->cm_role == FI_VERBS_CM_SELF);
rdma_destroy_qp(conn->id[1]);
- if ((ret = rdma_destroy_id(conn->id[1]))) {
- VERBS_INFO(FI_LOG_AV,
- "rdma_destroy_id failed, ret = %d\n", ret);
+ if (errno) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_destroy_qp\n", errno);
+ if (ret == FI_SUCCESS)
+ ret = -errno;
+ }
+ if (rdma_destroy_id(conn->id[1])) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_destroy_id\n", errno);
+ if (ret == FI_SUCCESS)
+ ret = -errno;
}
}
- if (conn->s_mr)
- fi_ibv_rdm_dereg_and_free(&conn->s_mr, &conn->sbuf_mem_reg);
- if (conn->r_mr)
- fi_ibv_rdm_dereg_and_free(&conn->r_mr, &conn->rbuf_mem_reg);
- if (conn->rma_mr)
- fi_ibv_rdm_dereg_and_free(&conn->rma_mr, &conn->rmabuf_mem_reg);
+ if (conn->s_mr) {
+ err = fi_ibv_rdm_dereg_and_free(&conn->s_mr, &conn->sbuf_mem_reg);
+ if ((err != FI_SUCCESS) && (ret == FI_SUCCESS)) {
+ ret = err;
+ }
+ }
+ if (conn->r_mr) {
+ err = fi_ibv_rdm_dereg_and_free(&conn->r_mr, &conn->rbuf_mem_reg);
+ if ((err != FI_SUCCESS) && (ret == FI_SUCCESS)) {
+ ret = err;
+ }
+ }
+ if (conn->ack_mr) {
+ if (ibv_dereg_mr(conn->ack_mr)) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "ibv_dereg_mr failed\n",
+ errno);
+ if (ret == FI_SUCCESS)
+ ret = -errno;
+ }
+ }
+
+ if (conn->rma_mr) {
+ err = fi_ibv_rdm_dereg_and_free(&conn->rma_mr,
+ &conn->rmabuf_mem_reg);
+ if ((err != FI_SUCCESS) && (ret == FI_SUCCESS)) {
+ ret = err;
+ }
+ }
- memset(conn, 0, sizeof(*conn));
free(conn);
return ret;
}
-static inline int
-fi_ibv_rdm_tagged_process_event_disconnected(struct fi_ibv_rdm_ep *ep,
- struct rdma_cm_event *event)
+static ssize_t
+fi_ibv_rdm_process_event_disconnected(struct fi_ibv_rdm_ep *ep,
+ struct rdma_cm_event *event)
{
- struct fi_ibv_rdm_tagged_conn *conn = event->id->context;
+ struct fi_ibv_rdm_conn *conn = event->id->context;
ep->num_active_conns--;
@@ -507,24 +632,41 @@ fi_ibv_rdm_tagged_process_event_disconnected(struct fi_ibv_rdm_ep *ep,
conn, inet_ntoa(conn->addr.sin_addr),
ntohs(conn->addr.sin_port));
if (conn->state == FI_VERBS_CONN_CLOSED) {
- fi_ibv_rdm_tagged_conn_cleanup(conn);
+ return fi_ibv_rdm_conn_cleanup(conn);
}
- return 0;
+ return FI_SUCCESS;
}
-static inline int
-fi_ibv_rdm_tagged_process_event_rejected(struct fi_ibv_rdm_ep *ep,
- struct rdma_cm_event *event)
+static ssize_t
+fi_ibv_rdm_process_event_rejected(struct fi_ibv_rdm_ep *ep,
+ struct rdma_cm_event *event)
{
- struct fi_ibv_rdm_tagged_conn *conn = event->id->context;
- int ret = 0;
-
- if (NULL != event->param.conn.private_data &&
- *((int *)event->param.conn.private_data) == 0xdeadbeef ) {
- assert(conn->cm_role == FI_VERBS_CM_PASSIVE);
+ struct fi_ibv_rdm_conn *conn = event->id->context;
+ ssize_t ret = FI_SUCCESS;
+ const int *pdata = event->param.conn.private_data;
+
+ if ((pdata && *pdata == 0xdeadbeef) ||
+ /*
+ * TODO: this is a workaround of the case when private_data is not
+ * arriving from rdma_reject call on iWarp devices
+ */
+ (conn->cm_role == FI_VERBS_CM_PASSIVE &&
+ event->status == -ECONNREFUSED))
+ {
+ errno = 0;
rdma_destroy_qp(event->id);
- ret = rdma_destroy_id(event->id);
+ if (errno) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_destroy_qp failed\n",
+ errno);
+ ret = -errno;
+ }
+ if (rdma_destroy_id(event->id)) {
+ VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_destroy_id failed\n",
+ errno);
+ if (ret == FI_SUCCESS)
+ ret = -errno;
+ }
VERBS_INFO(FI_LOG_AV,
"Rejected from conn %p, addr %s:%u, cm_role %d, status %d\n",
conn, inet_ntoa(conn->addr.sin_addr),
@@ -533,89 +675,136 @@ fi_ibv_rdm_tagged_process_event_rejected(struct fi_ibv_rdm_ep *ep,
event->status);
} else {
VERBS_INFO(FI_LOG_AV,
- "Unexpected REJECT from conn %p, addr %s:%u, cm_role %d, msg len %d, msg %x, status %d\n",
+ "Unexpected REJECT from conn %p, addr %s:%u, cm_role %d, "
+ "msg len %d, msg %x, status %d, err %d\n",
conn, inet_ntoa(conn->addr.sin_addr),
ntohs(conn->addr.sin_port),
conn->cm_role,
event->param.conn.private_data_len,
event->param.conn.private_data ?
*(int *)event->param.conn.private_data : 0,
- event->status);
+ event->status, errno);
conn->state = FI_VERBS_CONN_REJECTED;
}
return ret;
}
-static int fi_ibv_rdm_tagged_process_event(struct rdma_cm_event *event,
- struct fi_ibv_rdm_ep *ep)
+static ssize_t
+fi_ibv_rdm_process_event(struct rdma_cm_event *event, struct fi_ibv_rdm_ep *ep)
{
- int ret;
+ ssize_t ret = FI_SUCCESS;
switch (event->event) {
case RDMA_CM_EVENT_ADDR_RESOLVED:
- ret = fi_ibv_rdm_tagged_process_addr_resolved(event->id, ep);
+ ret = fi_ibv_rdm_process_addr_resolved(event->id, ep);
break;
case RDMA_CM_EVENT_ROUTE_RESOLVED:
- ret = fi_ibv_rdm_tagged_process_route_resolved(event, ep);
+ ret = fi_ibv_rdm_process_route_resolved(event, ep);
break;
case RDMA_CM_EVENT_ESTABLISHED:
- ret = fi_ibv_rdm_tagged_process_event_established(event, ep);
+ ret = fi_ibv_rdm_process_event_established(event, ep);
break;
case RDMA_CM_EVENT_DISCONNECTED:
- ret = fi_ibv_rdm_tagged_process_event_disconnected(ep, event);
+ ret = fi_ibv_rdm_process_event_disconnected(ep, event);
break;
case RDMA_CM_EVENT_CONNECT_REQUEST:
- ret = fi_ibv_rdm_tagged_process_connect_request(event, ep);
+ ret = fi_ibv_rdm_process_connect_request(event, ep);
break;
case RDMA_CM_EVENT_REJECTED:
- ret = fi_ibv_rdm_tagged_process_event_rejected(ep, event);
+ ret = fi_ibv_rdm_process_event_rejected(ep, event);
break;
case RDMA_CM_EVENT_TIMEWAIT_EXIT:
ret = FI_SUCCESS;
break;
+ /* All cases below fall to default case to print error message*/
case RDMA_CM_EVENT_ADDR_ERROR:
+ ret = -FI_EADDRNOTAVAIL;
case RDMA_CM_EVENT_ROUTE_ERROR:
+ ret = (ret == FI_SUCCESS) ? -FI_EHOSTUNREACH : ret;
case RDMA_CM_EVENT_CONNECT_ERROR:
+ ret = (ret == FI_SUCCESS) ? -FI_ECONNREFUSED : ret;
case RDMA_CM_EVENT_UNREACHABLE:
-
+ ret = (ret == FI_SUCCESS) ? -FI_EADDRNOTAVAIL : ret;
default:
- ret = -FI_EOTHER;
VERBS_INFO(FI_LOG_AV, "got unexpected rdmacm event, %s\n",
- rdma_event_str(event->event));
- abort();
+ rdma_event_str(event->event));
+ ret = (ret == FI_SUCCESS) ? -FI_ECONNABORTED : ret;
break;
}
- return ret;
+ return ret;
}
-int fi_ibv_rdm_tagged_cm_progress(struct fi_ibv_rdm_ep *ep)
+ssize_t fi_ibv_rdm_cm_progress(struct fi_ibv_rdm_ep *ep)
{
struct rdma_cm_event *event = NULL;
- int ret = 0;
-
- rdma_get_cm_event(ep->cm.ec, &event);
+ void *data = NULL;
+ ssize_t ret = FI_SUCCESS;
+
+ if (rdma_get_cm_event(ep->domain->rdm_cm->ec, &event)) {
+ if(errno == EAGAIN) {
+ errno = 0;
+ usleep(ep->cm_progress_timeout);
+ return FI_SUCCESS;
+ } else {
+ VERBS_INFO_ERRNO(FI_LOG_AV,
+ "rdma_get_cm_event failed\n", errno);
+ ret = -errno;
+ }
+ }
- while (event) {
+ while (ret == FI_SUCCESS && event) {
pthread_mutex_lock(&ep->cm_lock);
- void *data = NULL;
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
if (event->param.conn.private_data_len) {
data = malloc(event->param.conn.private_data_len);
+ if (!data) {
+ pthread_mutex_unlock(&ep->cm_lock);
+ ret = -FI_ENOMEM;
+ break;
+ }
memcpy(data, event->param.conn.private_data,
event->param.conn.private_data_len);
event_copy.param.conn.private_data = data;
event_copy.param.conn.private_data_len =
event->param.conn.private_data_len;
}
- rdma_ack_cm_event(event);
- ret = fi_ibv_rdm_tagged_process_event(&event_copy, ep);
+ if (rdma_ack_cm_event(event)) {
+ VERBS_INFO_ERRNO(FI_LOG_AV,
+ "rdma_get_cm_event failed\n", errno);
+ ret = -errno;
+ }
+
+ if (ret == FI_SUCCESS){
+ ret = fi_ibv_rdm_process_event(&event_copy, ep);
+ }
+
free(data);
+ data = NULL;
+
event = NULL;
+
pthread_mutex_unlock(&ep->cm_lock);
- rdma_get_cm_event(ep->cm.ec, &event);
+
+ if (ret != FI_SUCCESS) {
+ break;
+ }
+
+ if(rdma_get_cm_event(ep->domain->rdm_cm->ec, &event)) {
+ if(errno == EAGAIN) {
+ errno = 0;
+ usleep(ep->cm_progress_timeout);
+ break;
+ } else {
+ VERBS_INFO_ERRNO(FI_LOG_AV,
+ "rdma_get_cm_event failed\n",
+ errno);
+ ret = -errno;
+ }
+ }
}
+
return ret;
}
diff --git a/prov/verbs/src/ep_rdm/verbs_rdm_cntr.c b/prov/verbs/src/ep_rdm/verbs_rdm_cntr.c
new file mode 100644
index 0000000..51d6220
--- /dev/null
+++ b/prov/verbs/src/ep_rdm/verbs_rdm_cntr.c
@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2013-2016 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "fi_enosys.h"
+
+#include "verbs_rdm.h"
+
+static uint64_t fi_ibv_rdm_cntr_read(struct fid_cntr *cntr_fid)
+{
+ struct fi_ibv_rdm_cntr *cntr =
+ container_of(cntr_fid, struct fi_ibv_rdm_cntr, fid);
+ return cntr->value;
+}
+
+static uint64_t fi_ibv_rdm_cntr_readerr(struct fid_cntr *cntr_fid)
+{
+ struct fi_ibv_rdm_cntr *cntr =
+ container_of(cntr_fid, struct fi_ibv_rdm_cntr, fid);
+ return cntr->err_count;
+}
+
+static int fi_ibv_rdm_cntr_add(struct fid_cntr *cntr_fid, uint64_t value)
+{
+ struct fi_ibv_rdm_cntr *cntr =
+ container_of(cntr_fid, struct fi_ibv_rdm_cntr, fid);
+ cntr->value += value;
+ return 0;
+}
+
+static int fi_ibv_rdm_cntr_set(struct fid_cntr *cntr_fid, uint64_t value)
+{
+ struct fi_ibv_rdm_cntr *cntr =
+ container_of(cntr_fid, struct fi_ibv_rdm_cntr, fid);
+ cntr->value = value;
+ return 0;
+}
+
+static struct fi_ops_cntr fi_ibv_rdm_cntr_ops = {
+ .size = sizeof(struct fi_ops_cntr),
+ .read = fi_ibv_rdm_cntr_read,
+ .readerr = fi_ibv_rdm_cntr_readerr,
+ .add = fi_ibv_rdm_cntr_add,
+ .set = fi_ibv_rdm_cntr_set,
+ .wait = fi_no_cntr_wait,
+};
+
+static int fi_ibv_rdm_cntr_close(struct fid *fid)
+{
+ struct fi_ibv_rdm_cntr *cntr =
+ container_of(fid, struct fi_ibv_rdm_cntr, fid);
+
+ if (atomic_get(&cntr->ep_ref) > 0) {
+ return -FI_EBUSY;
+ }
+
+ free(cntr);
+ return 0;
+}
+
+static struct fi_ops fi_ibv_rdm_cntr_fi_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = fi_ibv_rdm_cntr_close,
+ .bind = fi_no_bind,
+ .control = fi_no_control,
+ .ops_open = fi_no_ops_open,
+};
+
+int fi_rbv_rdm_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
+ struct fid_cntr **cntr_fid, void *context)
+{
+ struct fi_ibv_rdm_cntr *cntr;
+
+ struct fi_ibv_domain *dom =
+ container_of(domain, struct fi_ibv_domain, domain_fid);
+
+ if (attr) {
+ switch (attr->events) {
+ case FI_CNTR_EVENTS_COMP:
+ break;
+ default:
+ return -FI_ENOSYS;
+ }
+
+ switch (attr->wait_obj) {
+ case FI_WAIT_NONE:
+ case FI_WAIT_UNSPEC:
+ break;
+ case FI_WAIT_MUTEX_COND:
+ case FI_WAIT_SET:
+ case FI_WAIT_FD:
+ default:
+ return -FI_ENOSYS;
+ }
+
+ if (attr->flags) {
+ return -FI_EINVAL;
+ }
+ }
+
+ cntr = calloc(1, sizeof(*cntr));
+ if (!cntr)
+ return -FI_ENOMEM;
+
+ if (attr) {
+ assert(sizeof(cntr->attr) == sizeof(*attr));
+ memcpy(&cntr->attr, attr, sizeof(*attr));
+ }
+
+ cntr->fid.fid.fclass = FI_CLASS_CNTR;
+ cntr->fid.fid.context = context;
+ cntr->fid.fid.ops = &fi_ibv_rdm_cntr_fi_ops;
+ cntr->fid.ops = &fi_ibv_rdm_cntr_ops;
+ cntr->domain = dom;
+ atomic_initialize(&cntr->ep_ref, 0);
+
+ *cntr_fid = &cntr->fid;
+
+ return FI_SUCCESS;
+}
diff --git a/prov/verbs/src/ep_rdm/verbs_rdm_msg.c b/prov/verbs/src/ep_rdm/verbs_rdm_msg.c
new file mode 100644
index 0000000..5e75534
--- /dev/null
+++ b/prov/verbs/src/ep_rdm/verbs_rdm_msg.c
@@ -0,0 +1,312 @@
+/*
+ * Copyright (c) 2013-2016 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "fi_enosys.h"
+
+#include "verbs_rdm.h"
+
+extern struct util_buf_pool *fi_ibv_rdm_request_pool;
+extern struct util_buf_pool *fi_ibv_rdm_extra_buffers_pool;
+
+static ssize_t fi_ibv_rdm_recvmsg(struct fid_ep *ep, const struct fi_msg *msg,
+ uint64_t flags)
+{
+ ssize_t ret = FI_SUCCESS;
+ struct fi_ibv_rdm_ep *ep_rdm =
+ container_of(ep, struct fi_ibv_rdm_ep, ep_fid);
+
+ if (msg->iov_count > 1) {
+ assert(0);
+ return -FI_EMSGSIZE;
+ }
+
+ struct fi_ibv_rdm_conn *conn = ep_rdm->av->addr_to_conn(ep_rdm, msg->addr);
+
+ struct fi_ibv_rdm_tagged_recv_start_data recv_data = {
+ .peek_data = {
+ .minfo = {
+ .conn = conn,
+ .tag = 0,
+ .tagmask = 0,
+ .is_tagged = 0
+ },
+ .context = msg->context,
+ .flags = (ep_rdm->rx_selective_completion ?
+ flags : (flags | FI_COMPLETION))
+ },
+ .dest_addr =
+ (msg->iov_count) ? msg->msg_iov[0].iov_base : NULL,
+ .data_len = (msg->iov_count) ? msg->msg_iov[0].iov_len : 0,
+ .ep = ep_rdm
+ };
+ struct fi_ibv_rdm_request *request =
+ util_buf_alloc(fi_ibv_rdm_request_pool);
+
+ fi_ibv_rdm_zero_request(request);
+ FI_IBV_RDM_DBG_REQUEST("get_from_pool: ", request, FI_LOG_DEBUG);
+
+ ret = fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_RECV_START,
+ &recv_data);
+
+ VERBS_DBG(FI_LOG_EP_DATA,
+ "conn %p, len %llu, rbuf %p, fi_ctx %p, posted_recv %d\n",
+ conn, recv_data.data_len, recv_data.dest_addr,
+ msg->context, ep_rdm->posted_recvs);
+
+ if (!ret && !request->state.err) {
+ ret = rdm_trecv_second_event(request, ep_rdm);
+ }
+
+ return ret;
+}
+
+static ssize_t
+fi_ibv_rdm_recvv(struct fid_ep *ep, const struct iovec *iov,
+ void **desc, size_t count, fi_addr_t src_addr,
+ void *context)
+{
+ struct fi_ibv_rdm_ep *ep_rdm =
+ container_of(ep, struct fi_ibv_rdm_ep, ep_fid);
+
+ const struct fi_msg msg = {
+ .msg_iov = iov,
+ .desc = desc,
+ .iov_count = count,
+ .addr = src_addr,
+ .context = context,
+ .data = 0
+ };
+
+ return fi_ibv_rdm_recvmsg(ep, &msg,
+ (ep_rdm->rx_selective_completion ? 0ULL : FI_COMPLETION));
+}
+
+static ssize_t
+fi_ibv_rdm_recv(struct fid_ep *ep, void *buf, size_t len, void *desc,
+ fi_addr_t src_addr, void *context)
+{
+ const struct iovec iov = {
+ .iov_base = buf,
+ .iov_len = len
+ };
+ return fi_ibv_rdm_recvv(ep, &iov, &desc, 1, src_addr, context);
+}
+
+static ssize_t fi_ibv_rdm_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
+ uint64_t flags)
+{
+ struct fi_ibv_rdm_ep *ep_rdm =
+ container_of(ep, struct fi_ibv_rdm_ep, ep_fid);
+
+ struct fi_ibv_rdm_send_start_data sdata = {
+ .ep_rdm = ep_rdm,
+ .conn = ep_rdm->av->addr_to_conn(ep_rdm, msg->addr),
+ .data_len = 0,
+ .context = msg->context,
+ .flags = FI_TAGGED | FI_SEND | (ep_rdm->tx_selective_completion ?
+ (flags & FI_COMPLETION) : FI_COMPLETION),
+ .tag = 0,
+ .is_tagged = 0,
+ .buf.src_addr = NULL,
+ .iov_count = 0,
+ .imm = (uint32_t) 0,
+ .stype = IBV_RDM_SEND_TYPE_UND
+ };
+
+ size_t i;
+ for (i = 0; i < msg->iov_count; i++) {
+ sdata.data_len += msg->msg_iov[i].iov_len;
+ }
+
+ if ((msg->iov_count > (sdata.ep_rdm->rndv_threshold / sizeof(struct iovec))) ||
+ (msg->iov_count > 1 && (sdata.data_len > sdata.ep_rdm->rndv_threshold)))
+ {
+ return -FI_EMSGSIZE;
+ }
+
+ switch (msg->iov_count)
+ {
+ case 1:
+ sdata.buf.src_addr = msg->msg_iov[0].iov_base;
+ case 0:
+ sdata.stype = IBV_RDM_SEND_TYPE_GEN;
+ break;
+ default:
+ /* TODO:
+ * extra allocation & memcpy can be optimized if it's possible
+ * to send immediately
+ */
+ sdata.buf.iovec_arr =
+ util_buf_alloc(fi_ibv_rdm_extra_buffers_pool);
+ for (i = 0; i < msg->iov_count; i++) {
+ sdata.buf.iovec_arr[i].iov_base = msg->msg_iov[i].iov_base;
+ sdata.buf.iovec_arr[i].iov_len = msg->msg_iov[i].iov_len;
+ }
+ sdata.iov_count = msg->iov_count;
+ sdata.stype = IBV_RDM_SEND_TYPE_VEC;
+ break;
+ }
+
+ return fi_ibv_rdm_send_common(&sdata);
+}
+
+static ssize_t fi_ibv_rdm_sendv(struct fid_ep *ep, const struct iovec *iov,
+ void **desc, size_t count, fi_addr_t dest_addr,
+ void *context)
+{
+ struct fi_ibv_rdm_ep *ep_rdm =
+ container_of(ep, struct fi_ibv_rdm_ep, ep_fid);
+
+ const struct fi_msg msg = {
+ .msg_iov = iov,
+ .desc = desc,
+ .iov_count = count,
+ .addr = dest_addr,
+ .context = context,
+ .data = 0
+ };
+
+ return fi_ibv_rdm_sendmsg(ep, &msg,
+ (ep_rdm->tx_selective_completion ? 0ULL : FI_COMPLETION));
+}
+
+static ssize_t fi_ibv_rdm_send(struct fid_ep *ep, const void *buf, size_t len,
+ void *desc, fi_addr_t dest_addr, void *context)
+{
+ const struct iovec iov = {
+ .iov_base = (void *)buf,
+ .iov_len = len
+ };
+ return fi_ibv_rdm_sendv(ep, &iov, &desc, 1, dest_addr, context);
+}
+
+static ssize_t fi_ibv_rdm_inject(struct fid_ep *ep_fid, const void *buf,
+ size_t len, fi_addr_t dest_addr)
+{
+ struct fi_ibv_rdm_ep *ep =
+ container_of(ep_fid, struct fi_ibv_rdm_ep, ep_fid);
+ struct fi_ibv_rdm_conn *conn = ep->av->addr_to_conn(ep, dest_addr);
+
+ const size_t size = len + sizeof(struct fi_ibv_rdm_header);
+
+ if (len > ep->rndv_threshold) {
+ return -FI_EMSGSIZE;
+ }
+
+ const int in_order = (conn->postponed_entry) ? 0 : 1;
+
+ if (in_order) {
+ struct fi_ibv_rdm_buf *sbuf =
+ fi_ibv_rdm_prepare_send_resources(conn, ep);
+ if (sbuf) {
+ struct ibv_sge sge = {0};
+ struct ibv_send_wr wr = {0};
+ struct ibv_send_wr *bad_wr = NULL;
+
+ sge.addr = (uintptr_t)(void*)sbuf;
+ sge.length = size + FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE;
+ sge.lkey = conn->s_mr->lkey;
+
+ wr.wr_id = FI_IBV_RDM_PACK_SERVICE_WR(conn);
+ wr.sg_list = &sge;
+ wr.num_sge = 1;
+ wr.wr.rdma.remote_addr = (uintptr_t)
+ fi_ibv_rdm_get_remote_addr(conn, sbuf);
+ wr.wr.rdma.rkey = conn->remote_rbuf_rkey;
+ wr.send_flags = (sge.length < ep->max_inline_rc)
+ ? IBV_SEND_INLINE : 0;
+ wr.imm_data = 0;
+ wr.opcode = ep->eopcode;
+
+ sbuf->service_data.pkt_len = size;
+ sbuf->header.tag = 0;
+ sbuf->header.service_tag = 0;
+
+ FI_IBV_RDM_SET_PKTTYPE(sbuf->header.service_tag,
+ FI_IBV_RDM_MSG_PKT);
+ if ((len > 0) && (buf)) {
+ memcpy(&sbuf->payload, buf, len);
+ }
+
+ FI_IBV_RDM_INC_SIG_POST_COUNTERS(conn, ep,
+ wr.send_flags);
+ if (ibv_post_send(conn->qp[0], &wr, &bad_wr)) {
+ assert(0);
+ return -errno;
+ } else {
+ VERBS_DBG(FI_LOG_EP_DATA,
+ "posted %d bytes, conn %p, len %d\n",
+ sge.length, conn, len);
+ return FI_SUCCESS;
+ }
+ }
+ }
+
+ fi_ibv_rdm_tagged_poll(ep);
+
+ return -FI_EAGAIN;
+}
+
+static ssize_t fi_ibv_rdm_senddata(struct fid_ep *ep, const void *buf,
+ size_t len, void *desc, uint64_t data,
+ fi_addr_t dest_addr, void *context)
+{
+ assert(0);
+ return -FI_ENOSYS;
+}
+
+static ssize_t fi_ibv_rdm_injectdata(struct fid_ep *ep, const void *buf,
+ size_t len, uint64_t data,
+ fi_addr_t dest_addr)
+{
+ assert(0);
+ return -FI_ENOSYS;
+}
+
+static struct fi_ops_msg fi_ibv_rdm_ep_msg_ops = {
+ .size = sizeof(struct fi_ops_msg),
+ .recv = fi_ibv_rdm_recv,
+ .recvv = fi_ibv_rdm_recvv,
+ .recvmsg = fi_ibv_rdm_recvmsg,
+ .send = fi_ibv_rdm_send,
+ .sendv = fi_ibv_rdm_sendv,
+ .sendmsg = fi_ibv_rdm_sendmsg,
+ .inject = fi_ibv_rdm_inject,
+ .senddata = fi_ibv_rdm_senddata,
+ .injectdata = fi_ibv_rdm_injectdata
+};
+
+struct fi_ops_msg *fi_ibv_rdm_ep_ops_msg()
+{
+ return &fi_ibv_rdm_ep_msg_ops;
+}
diff --git a/prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c b/prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c
index fa1109c..263d540 100644
--- a/prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c
+++ b/prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm.c
@@ -41,199 +41,182 @@
#include "verbs_queuing.h"
#include "verbs_tagged_ep_rdm_states.h"
-DEFINE_LIST(fi_ibv_rdm_tagged_request_ready_queue);
-DEFINE_LIST(fi_ibv_rdm_tagged_recv_posted_queue);
-DEFINE_LIST(fi_ibv_rdm_tagged_recv_unexp_queue);
-DEFINE_LIST(fi_ibv_rdm_tagged_send_postponed_queue);
+struct fi_ibv_rdm_cq fi_ibv_rdm_comp_queue;
-struct util_buf_pool *fi_ibv_rdm_tagged_request_pool;
-struct util_buf_pool *fi_ibv_rdm_tagged_postponed_pool;
+extern struct util_buf_pool *fi_ibv_rdm_request_pool;
+extern struct util_buf_pool *fi_ibv_rdm_extra_buffers_pool;
-/*
- * extra buffer size equal eager buffer size, it is used for any intermediate
- * needs like unexpected recv, pack/unpack noncontig messages, etc
- */
-struct util_buf_pool *fi_ibv_rdm_tagged_extra_buffers_pool;
+DEFINE_LIST(fi_ibv_rdm_posted_queue);
+DEFINE_LIST(fi_ibv_rdm_unexp_queue);
+DEFINE_LIST(fi_ibv_rdm_postponed_queue);
static inline int fi_ibv_rdm_tagged_poll_send(struct fi_ibv_rdm_ep *ep);
static inline int fi_ibv_rdm_tagged_poll_recv(struct fi_ibv_rdm_ep *ep);
-static inline int
-fi_ibv_rdm_tagged_init_request_sbuf(struct fi_ibv_rdm_tagged_request *request,
- struct fi_ibv_rdm_ep *ep);
-
-int fi_ibv_rdm_tagged_prepare_send_request(
- struct fi_ibv_rdm_tagged_request *request, struct fi_ibv_rdm_ep *ep)
+int
+fi_ibv_rdm_tagged_prepare_send_request(struct fi_ibv_rdm_request *request,
+ struct fi_ibv_rdm_ep *ep)
{
#if ENABLE_DEBUG
- int res =
- FI_IBV_RDM_TAGGED_SENDS_OUTGOING_ARE_LIMITED(request->conn, ep);
+ int res = OUTGOING_POST_LIMIT(request->minfo.conn, ep);
if (res) {
- FI_IBV_RDM_TAGGED_DBG_REQUEST
- ("failed because SENDS_OUTGOING_ARE_LIMITED", request,
- FI_LOG_DEBUG);
+ FI_IBV_RDM_DBG_REQUEST
+ ("failed because OUTGOING_POST_LIMIT", request,
+ FI_LOG_DEBUG);
return !res;
}
- res = PEND_SEND_IS_LIMITED(ep);
+ res = PEND_POST_LIMIT(ep);
if (res) {
- FI_IBV_RDM_TAGGED_DBG_REQUEST
- ("failed because PEND_SEND_IS_LIMITED", request,
- FI_LOG_DEBUG);
+ FI_IBV_RDM_DBG_REQUEST
+ ("failed because PEND_POST_LIMIT", request,
+ FI_LOG_DEBUG);
return !res;
}
#endif // ENABLE_DEBUG
- request->sbuf =
- fi_ibv_rdm_tagged_prepare_send_resources(request->conn, ep);
+ request->sbuf = fi_ibv_rdm_prepare_send_resources(request->minfo.conn, ep);
return !!request->sbuf;
}
-int fi_ibv_rdm_prepare_rma_request(struct fi_ibv_rdm_tagged_request *request,
- struct fi_ibv_rdm_ep *ep)
+int
+fi_ibv_rdm_prepare_rma_request(struct fi_ibv_rdm_request *request,
+ struct fi_ibv_rdm_ep *ep)
{
request->rmabuf =
- fi_ibv_rdm_rma_prepare_resources(request->conn, ep);
+ fi_ibv_rdm_rma_prepare_resources(request->minfo.conn, ep);
return !!request->rmabuf;
}
-
static int fi_ibv_rdm_tagged_getname(fid_t fid, void *addr, size_t * addrlen)
{
struct fi_ibv_rdm_ep *ep =
- container_of(fid, struct fi_ibv_rdm_ep, ep_fid);
+ container_of(fid, struct fi_ibv_rdm_ep, ep_fid);
if (FI_IBV_RDM_DFLT_ADDRLEN > *addrlen) {
*addrlen = FI_IBV_RDM_DFLT_ADDRLEN;
return -FI_ETOOSMALL;
}
memset(addr, 0, *addrlen);
- memcpy(addr, &ep->cm.my_addr, FI_IBV_RDM_DFLT_ADDRLEN);
+ memcpy(addr, &ep->my_addr, FI_IBV_RDM_DFLT_ADDRLEN);
ep->addrlen = *addrlen;
return 0;
}
-static ssize_t fi_ibv_rdm_tagged_recvfrom(struct fid_ep *ep_fid, void *buf,
- size_t len, void *desc,
- fi_addr_t src_addr, uint64_t tag,
- uint64_t ignore, void *context)
+static ssize_t
+fi_ibv_rdm_tagged_recvmsg(struct fid_ep *ep_fid, const struct fi_msg_tagged *msg,
+ uint64_t flags)
{
- int ret = 0;
+ ssize_t ret = FI_SUCCESS;
+ struct fi_ibv_rdm_ep *ep_rdm =
+ container_of(ep_fid, struct fi_ibv_rdm_ep, ep_fid);
- struct fi_ibv_rdm_tagged_request *request =
- util_buf_alloc(fi_ibv_rdm_tagged_request_pool);
- fi_ibv_rdm_tagged_zero_request(request);
- FI_IBV_RDM_TAGGED_DBG_REQUEST("get_from_pool: ", request, FI_LOG_DEBUG);
+ if (msg->iov_count > 1) {
+ assert(0);
+ return -FI_EMSGSIZE;
+ }
- struct fi_ibv_rdm_tagged_conn *conn = (src_addr == FI_ADDR_UNSPEC)
- ? NULL : (struct fi_ibv_rdm_tagged_conn *)src_addr;
- struct fi_ibv_rdm_ep *ep = container_of(ep_fid,
- struct fi_ibv_rdm_ep, ep_fid);
+ struct fi_ibv_rdm_conn *conn = ep_rdm->av->addr_to_conn(ep_rdm, msg->addr);
+
+ struct fi_ibv_rdm_tagged_recv_start_data recv_data = {
+ .peek_data = {
+ .minfo = {
+ .conn = conn,
+ .tag = msg->tag,
+ .tagmask = ~(msg->ignore),
+ .is_tagged = 1
+ },
+ .context = msg->context,
+ .flags = (ep_rdm->rx_selective_completion ?
+ flags : (flags | FI_COMPLETION))
+ },
+ .dest_addr =
+ (msg->iov_count) ? msg->msg_iov[0].iov_base : NULL,
+ .data_len = (msg->iov_count) ? msg->msg_iov[0].iov_len : 0,
+ .ep = ep_rdm
+ };
- {
- struct fi_ibv_rdm_tagged_recv_start_data req_data = {
- .tag = tag,
- .tagmask = ~ignore,
- .context = context,
- .dest_addr = buf,
- .conn = conn,
- .ep = ep,
- .data_len = len
- };
+ struct fi_ibv_rdm_request *request =
+ util_buf_alloc(fi_ibv_rdm_request_pool);
- ret = fi_ibv_rdm_tagged_req_hndl(request,
- FI_IBV_EVENT_RECV_START, &req_data);
+ fi_ibv_rdm_zero_request(request);
+ FI_IBV_RDM_DBG_REQUEST("get_from_pool: ", request, FI_LOG_DEBUG);
- VERBS_DBG(FI_LOG_EP_DATA,
- "fi_recvfrom: conn %p, tag 0x%llx, len %d, rbuf %p, fi_ctx %p, "
- "pend_recv %d\n",
- conn, (long long unsigned int)tag, (int)len, buf, context,
- ep->posted_recvs);
-
- if (ret || request->state.eager ==
- FI_IBV_STATE_EAGER_RECV_WAIT4PKT) {
- goto out;
+ if (flags & FI_PEEK) {
+ recv_data.peek_data.flags |= FI_COMPLETION;
+ ret = fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_RECV_PEEK,
+ &recv_data);
+ if (ret == -FI_ENOMSG) {
+ fi_ibv_rdm_tagged_poll(ep_rdm);
}
- }
+ } else if (flags & FI_CLAIM) {
+ recv_data.peek_data.flags |= FI_COMPLETION;
+ ret = fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_RECV_START,
+ &recv_data);
+ if (!ret) {
+ ret = rdm_trecv_second_event(request, ep_rdm);
+ }
+ } else {
+ ret = fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_RECV_START,
+ &recv_data);
- struct fi_ibv_recv_got_pkt_process_data data = {
- .ep = ep
- };
+ VERBS_DBG(FI_LOG_EP_DATA,
+ "fi_recvfrom: conn %p, tag 0x%llx, len %llu, rbuf %p, fi_ctx %p, posted_recv %d\n",
+ conn, msg->tag, recv_data.data_len, recv_data.dest_addr,
+ msg->context, ep_rdm->posted_recvs);
- if (request->state.rndv == FI_IBV_STATE_RNDV_RECV_WAIT4RES) {
- if (fi_ibv_rdm_tagged_prepare_send_request(request, ep)) {
- ret =
- fi_ibv_rdm_tagged_req_hndl(request,
- FI_IBV_EVENT_SEND_READY,
- &data);
+ if (!ret && !request->state.err) {
+ ret = rdm_trecv_second_event(request, ep_rdm);
}
- } else {
- ret =
- fi_ibv_rdm_tagged_req_hndl(request, FI_IBV_EVENT_RECV_START,
- &data);
}
-out:
return ret;
}
-static ssize_t fi_ibv_rdm_tagged_recvv(struct fid_ep *ep, const struct iovec *iov, void **desc,
- size_t count, fi_addr_t src_addr, uint64_t tag, uint64_t ignore,
- void *context)
+static ssize_t
+fi_ibv_rdm_tagged_recvv(struct fid_ep *ep_fid, const struct iovec *iov, void **desc,
+ size_t count, fi_addr_t src_addr, uint64_t tag,
+ uint64_t ignore, void *context)
{
- void *buf = NULL;
- size_t total_len = 0;
- size_t i;
-
- for (i = 0; i < count; i++) {
- total_len += iov[i].iov_len;
- }
-
struct fi_ibv_rdm_ep *ep_rdm =
- container_of(ep, struct fi_ibv_rdm_ep, ep_fid);
- if ((count > 1) && (total_len > ep_rdm->rndv_threshold)) {
- return -FI_EMSGSIZE;
- }
+ container_of(ep_fid, struct fi_ibv_rdm_ep, ep_fid);
- switch (count)
- {
- case 0: break;
- case 1:
- buf = iov[0].iov_base;
- break;
- default:
- return -FI_EINVAL;
- break;
- }
+ const struct fi_msg_tagged msg = {
+ .msg_iov = iov,
+ .desc = desc,
+ .iov_count = count,
+ .addr = src_addr,
+ .tag = tag,
+ .ignore = ignore,
+ .context = context,
+ .data = 0
+ };
+
+ return fi_ibv_rdm_tagged_recvmsg(ep_fid, &msg,
+ (ep_rdm->rx_selective_completion ? 0ULL : FI_COMPLETION));
- return fi_ibv_rdm_tagged_recvfrom(ep, buf, total_len,
- desc ? desc[0] : NULL,
- src_addr, tag, ignore,
- context);
}
-static ssize_t fi_ibv_rdm_tagged_recvmsg(struct fid_ep *ep,
- const struct fi_msg_tagged *msg,
- uint64_t flags)
+static ssize_t fi_ibv_rdm_tagged_recvfrom(struct fid_ep *ep_fid, void *buf,
+ size_t len, void *desc,
+ fi_addr_t src_addr, uint64_t tag,
+ uint64_t ignore, void *context)
{
- return fi_ibv_rdm_tagged_recvv(ep, msg->msg_iov, msg->desc,
- msg->iov_count, msg->addr, msg->tag, msg->ignore, msg->context);
+ const struct iovec iov = {
+ .iov_base = buf,
+ .iov_len = len
+ };
+
+ return fi_ibv_rdm_tagged_recvv(ep_fid, &iov, &desc, 1, src_addr, tag,
+ ignore, context);
}
-static ssize_t fi_ibv_rdm_tagged_sendto(struct fid_ep *fid, const void *buf,
- size_t len, void *desc,
- fi_addr_t dest_addr,
- uint64_t tag, void *context);
-
-static inline ssize_t fi_ibv_rdm_tagged_inject(struct fid_ep *fid,
- const void *buf,
- size_t len,
- fi_addr_t dest_addr,
- uint64_t tag)
+static inline ssize_t
+fi_ibv_rdm_tagged_inject(struct fid_ep *fid, const void *buf, size_t len,
+ fi_addr_t dest_addr, uint64_t tag)
{
- struct fi_ibv_rdm_tagged_conn *conn =
- (struct fi_ibv_rdm_tagged_conn *) dest_addr;
struct fi_ibv_rdm_ep *ep =
- container_of(fid, struct fi_ibv_rdm_ep, ep_fid);
+ container_of(fid, struct fi_ibv_rdm_ep, ep_fid);
+ struct fi_ibv_rdm_conn *conn = ep->av->addr_to_conn(ep, dest_addr);
const size_t size = len + sizeof(struct fi_ibv_rdm_header);
@@ -244,39 +227,36 @@ static inline ssize_t fi_ibv_rdm_tagged_inject(struct fid_ep *fid,
const int in_order = (conn->postponed_entry) ? 0 : 1;
if (in_order) {
- void *raw_sbuf =
- fi_ibv_rdm_tagged_prepare_send_resources(conn, ep);
- if (raw_sbuf) {
- struct ibv_sge sge = { 0 };
- struct ibv_send_wr wr = { 0 };
+ struct fi_ibv_rdm_buf *sbuf =
+ fi_ibv_rdm_prepare_send_resources(conn, ep);
+ if (sbuf) {
+ struct ibv_sge sge = {0};
+ struct ibv_send_wr wr = {0};
struct ibv_send_wr *bad_wr = NULL;
+
+ sge.addr = (uintptr_t)(void*)sbuf;
+ sge.length = size + FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE;
+ sge.lkey = conn->s_mr->lkey;
+
wr.wr_id = FI_IBV_RDM_PACK_SERVICE_WR(conn);
wr.sg_list = &sge;
wr.num_sge = 1;
- wr.wr.rdma.remote_addr =
- fi_ibv_rdm_tagged_get_remote_addr(conn, raw_sbuf);
+ wr.wr.rdma.remote_addr = (uintptr_t)
+ fi_ibv_rdm_get_remote_addr(conn, sbuf);
wr.wr.rdma.rkey = conn->remote_rbuf_rkey;
- wr.send_flags = (size < ep->max_inline_rc)
+ wr.send_flags = (sge.length < ep->max_inline_rc)
? IBV_SEND_INLINE : 0;
- wr.imm_data =
- fi_ibv_rdm_tagged_get_buff_service_data(raw_sbuf)->
- seq_number;
- wr.opcode = IBV_WR_RDMA_WRITE_WITH_IMM;
-
- sge.addr = (uintptr_t) (raw_sbuf);
- sge.length = size;
- sge.lkey = conn->s_mr->lkey;
-
- struct fi_ibv_rdm_buf *sbuf =
- (struct fi_ibv_rdm_buf *)raw_sbuf;
- void *payload = (void *)&(sbuf->payload[0]);
+ wr.imm_data = 0;
+ wr.opcode = ep->eopcode;
+ sbuf->service_data.pkt_len = size;
sbuf->header.tag = tag;
sbuf->header.service_tag = 0;
+
FI_IBV_RDM_SET_PKTTYPE(sbuf->header.service_tag,
FI_IBV_RDM_EAGER_PKT);
if ((len > 0) && (buf)) {
- memcpy(payload, buf, len);
+ memcpy(&sbuf->payload, buf, len);
}
FI_IBV_RDM_INC_SIG_POST_COUNTERS(conn, ep,
@@ -298,74 +278,56 @@ static inline ssize_t fi_ibv_rdm_tagged_inject(struct fid_ep *fid,
return -FI_EAGAIN;
}
-static ssize_t
-fi_ibv_rdm_tagged_send_common(struct fi_ibv_rdm_tagged_send_start_data* sdata)
-{
- struct fi_ibv_rdm_tagged_request *request =
- util_buf_alloc(fi_ibv_rdm_tagged_request_pool);
- FI_IBV_RDM_TAGGED_DBG_REQUEST("get_from_pool: ", request, FI_LOG_DEBUG);
-
- /* Initial state */
- request->state.eager = FI_IBV_STATE_EAGER_BEGIN;
- request->state.rndv = FI_IBV_STATE_RNDV_NOT_USED;
-
- const int in_order = (sdata->conn->postponed_entry) ? 0 : 1;
- int ret = fi_ibv_rdm_tagged_req_hndl(request, FI_IBV_EVENT_SEND_START,
- sdata);
-
- if (!ret && in_order &&
- fi_ibv_rdm_tagged_prepare_send_request(request, sdata->ep_rdm)) {
-
- struct fi_ibv_rdm_tagged_send_ready_data req_data =
- {.ep = sdata->ep_rdm };
- ret = fi_ibv_rdm_tagged_req_hndl(request,
- FI_IBV_EVENT_SEND_READY, &req_data);
- }
-
- return ret;
-}
-
-
static ssize_t fi_ibv_rdm_tagged_senddatato(struct fid_ep *fid, const void *buf,
size_t len, void *desc,
- uint64_t data,
- fi_addr_t dest_addr, uint64_t tag,
- void *context)
+ uint64_t data, fi_addr_t dest_addr,
+ uint64_t tag, void *context)
{
- struct fi_ibv_rdm_tagged_send_start_data sdata = {
+ struct fi_ibv_rdm_ep *ep_rdm =
+ container_of(fid, struct fi_ibv_rdm_ep, ep_fid);
+
+ struct fi_ibv_rdm_send_start_data sdata = {
.ep_rdm = container_of(fid, struct fi_ibv_rdm_ep, ep_fid),
- .conn = (struct fi_ibv_rdm_tagged_conn *) dest_addr,
+ .conn = ep_rdm->av->addr_to_conn(ep_rdm, dest_addr),
.data_len = len,
.context = context,
+ .flags = FI_TAGGED | FI_SEND |
+ (ep_rdm->tx_selective_completion ? 0ULL : FI_COMPLETION),
.tag = tag,
+ .is_tagged = 1,
.buf.src_addr = (void*)buf,
.iov_count = 0,
.imm = (uint32_t) data,
.stype = IBV_RDM_SEND_TYPE_GEN
};
- return fi_ibv_rdm_tagged_send_common(&sdata);
+ return fi_ibv_rdm_send_common(&sdata);
}
static ssize_t fi_ibv_rdm_tagged_sendto(struct fid_ep *fid, const void *buf,
size_t len, void *desc,
- fi_addr_t dest_addr,
- uint64_t tag, void *context)
+ fi_addr_t dest_addr, uint64_t tag,
+ void *context)
{
return fi_ibv_rdm_tagged_senddatato(fid, buf, len, desc, 0, dest_addr,
tag, context);
}
-static ssize_t fi_ibv_rdm_tagged_sendv(struct fid_ep *ep,
- const struct iovec *iov, void **desc, size_t count, fi_addr_t dest_addr,
- uint64_t tag, void *context)
+static ssize_t fi_ibv_rdm_tagged_sendmsg(struct fid_ep *ep,
+ const struct fi_msg_tagged *msg, uint64_t flags)
{
- struct fi_ibv_rdm_tagged_send_start_data sdata = {
+ struct fi_ibv_rdm_ep *ep_rdm =
+ container_of(ep, struct fi_ibv_rdm_ep, ep_fid);
+
+ struct fi_ibv_rdm_send_start_data sdata = {
.ep_rdm = container_of(ep, struct fi_ibv_rdm_ep, ep_fid),
- .conn = (struct fi_ibv_rdm_tagged_conn *) dest_addr,
+ .conn = ep_rdm->av->addr_to_conn(ep_rdm, msg->addr),
.data_len = 0,
- .context = context,
- .tag = tag,
+ .context = msg->context,
+ .flags = FI_TAGGED | FI_SEND | (ep_rdm->tx_selective_completion ?
+ (flags & FI_COMPLETION) : FI_COMPLETION),
+ .tag = msg->tag,
+ .is_tagged = 1,
.buf.src_addr = NULL,
.iov_count = 0,
.imm = (uint32_t) 0,
@@ -373,19 +335,20 @@ static ssize_t fi_ibv_rdm_tagged_sendv(struct fid_ep *ep,
};
size_t i;
- for (i = 0; i < count; i++) {
- sdata.data_len += iov[i].iov_len;
+ for (i = 0; i < msg->iov_count; i++) {
+ sdata.data_len += msg->msg_iov[i].iov_len;
}
- if ((count > (sdata.ep_rdm->rndv_threshold / sizeof(struct iovec))) ||
- (count > 1 && (sdata.data_len > sdata.ep_rdm->rndv_threshold))) {
+ if ((msg->iov_count > (sdata.ep_rdm->rndv_threshold / sizeof(struct iovec))) ||
+ (msg->iov_count > 1 && (sdata.data_len > sdata.ep_rdm->rndv_threshold)))
+ {
return -FI_EMSGSIZE;
}
- switch (count)
+ switch (msg->iov_count)
{
case 1:
- sdata.buf.src_addr = iov[0].iov_base;
+ sdata.buf.src_addr = msg->msg_iov[0].iov_base;
case 0:
sdata.stype = IBV_RDM_SEND_TYPE_GEN;
break;
@@ -394,25 +357,41 @@ static ssize_t fi_ibv_rdm_tagged_sendv(struct fid_ep *ep,
* extra allocation & memcpy can be optimized if it's possible
* to send immediately
*/
- sdata.buf.iovec_arr =
- util_buf_alloc(fi_ibv_rdm_tagged_extra_buffers_pool);
- for (i = 0; i < count; i++) {
- sdata.buf.iovec_arr[i].iov_base = iov[i].iov_base;
- sdata.buf.iovec_arr[i].iov_len = iov[i].iov_len;
+ sdata.buf.iovec_arr =
+ util_buf_alloc(fi_ibv_rdm_extra_buffers_pool);
+ for (i = 0; i < msg->iov_count; i++) {
+ sdata.buf.iovec_arr[i].iov_base = msg->msg_iov[i].iov_base;
+ sdata.buf.iovec_arr[i].iov_len = msg->msg_iov[i].iov_len;
}
- sdata.iov_count = count;
+ sdata.iov_count = msg->iov_count;
sdata.stype = IBV_RDM_SEND_TYPE_VEC;
break;
}
- return fi_ibv_rdm_tagged_send_common(&sdata);
+ return fi_ibv_rdm_send_common(&sdata);
}
-static ssize_t fi_ibv_rdm_tagged_sendmsg(struct fid_ep *ep,
- const struct fi_msg_tagged *msg, uint64_t flags)
+static ssize_t fi_ibv_rdm_tagged_sendv(struct fid_ep *ep,
+ const struct iovec *iov, void **desc,
+ size_t count, fi_addr_t dest_addr,
+ uint64_t tag, void *context)
{
- return fi_ibv_rdm_tagged_sendv(ep, msg->msg_iov,
- msg->desc, msg->iov_count, msg->addr, msg->tag, msg->context);
+ struct fi_ibv_rdm_ep *ep_rdm =
+ container_of(ep, struct fi_ibv_rdm_ep, ep_fid);
+
+ const struct fi_msg_tagged msg = {
+ .msg_iov = iov,
+ .desc = desc,
+ .iov_count = count,
+ .addr = dest_addr,
+ .tag = tag,
+ .ignore = 0,
+ .context = context,
+ .data = 0
+ };
+
+ return fi_ibv_rdm_tagged_sendmsg(ep, &msg,
+ (ep_rdm->tx_selective_completion ? 0ULL : FI_COMPLETION));
}
struct fi_ops_tagged fi_ibv_rdm_tagged_ops = {
@@ -434,22 +413,13 @@ struct fi_ops_cm fi_ibv_rdm_tagged_ep_cm_ops = {
};
static inline void
-fi_ibv_rdm_tagged_release_remote_sbuff(struct fi_ibv_rdm_tagged_conn *conn,
- struct fi_ibv_rdm_ep *ep)
+fi_ibv_rdm_tagged_release_remote_sbuff(struct fi_ibv_rdm_conn *conn,
+ struct fi_ibv_rdm_ep *ep)
{
- char *buff = conn->sbuf_ack_head;
- // not needed till nobody use service data of recv buffers
- fi_ibv_rdm_tagged_set_buffer_status(buff, BUF_STATUS_FREE);
-#if ENABLE_DEBUG
- char *rbuf = (char *)fi_ibv_rdm_tagged_get_rbuf(conn, ep, 0);
- memset(rbuf, 0,
- ep->buff_len - FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE);
-#endif // ENABLE_DEBUG
struct ibv_sge sge;
- sge.addr =
- (uintptr_t) &fi_ibv_rdm_tagged_get_buff_service_data(buff)->status;
- sge.length = sizeof(enum fi_ibv_rdm_tagged_buffer_status);
- sge.lkey = conn->r_mr->lkey;
+ sge.addr = (uint64_t) & conn->sbuf_ack_status;
+ sge.length = sizeof(conn->sbuf_ack_status);
+ sge.lkey = conn->ack_mr->lkey;
struct ibv_send_wr *bad_wr = NULL;
struct ibv_send_wr wr = { 0 };
@@ -457,13 +427,12 @@ fi_ibv_rdm_tagged_release_remote_sbuff(struct fi_ibv_rdm_tagged_conn *conn,
wr.wr_id = FI_IBV_RDM_PACK_SERVICE_WR(conn);
wr.sg_list = &sge;
wr.num_sge = 1;
- wr.wr.rdma.remote_addr =
- (uintptr_t) &fi_ibv_rdm_tagged_get_buff_service_data(
- conn->remote_sbuf_head)->status;
+ wr.wr.rdma.remote_addr = (uint64_t)
+ &conn->remote_sbuf_head->service_data.status;
wr.wr.rdma.rkey = conn->remote_sbuf_rkey;
- wr.send_flags = IBV_SEND_INLINE;
- wr.opcode = IBV_WR_RDMA_WRITE; // w/o imm - do not put it into recv
- // completion queue
+ wr.send_flags = (sge.length < ep->max_inline_rc) ? IBV_SEND_INLINE : 0;
+ /* w/o imm - do not put it into recv completion queue */
+ wr.opcode = IBV_WR_RDMA_WRITE;
FI_IBV_RDM_INC_SIG_POST_COUNTERS(conn, ep, wr.send_flags);
VERBS_DBG(FI_LOG_EP_DATA,
@@ -480,67 +449,51 @@ fi_ibv_rdm_tagged_release_remote_sbuff(struct fi_ibv_rdm_tagged_conn *conn,
}
static inline void
-fi_ibv_rdm_tagged_process_recv(struct fi_ibv_rdm_ep *ep,
- struct fi_ibv_rdm_tagged_conn *conn,
- int arrived_len,
- uint32_t imm_data,
- struct fi_ibv_rdm_buf *rbuf)
+fi_ibv_rdm_process_recv(struct fi_ibv_rdm_ep *ep, struct fi_ibv_rdm_conn *conn,
+ int arrived_len, struct fi_ibv_rdm_buf *rbuf)
{
- struct fi_ibv_rdm_tagged_request *request = NULL;
-
-#if ENABLE_DEBUG
- VERBS_DBG(FI_LOG_EP_DATA, "processing pkt # %d, imm %d \n", rbuf->header.seq_num, imm_data);
-#endif /* ENABLE_DEBUG */
+ struct fi_ibv_rdm_request *request = NULL;
int pkt_type = FI_IBV_RDM_GET_PKTTYPE(rbuf->header.service_tag);
if (pkt_type == FI_IBV_RDM_RNDV_ACK_PKT) {
- memcpy(&request, rbuf->payload,
- sizeof(struct fi_ibv_rdm_tagged_request *));
+ memcpy(&request, &rbuf->payload, sizeof(request));
assert(request);
VERBS_DBG(FI_LOG_EP_DATA,
- "GOT RNDV ACK from conn %p, id %d\n", conn, request);
+ "GOT RNDV ACK from conn %p, id %p\n", conn, request);
} else if (pkt_type != FI_IBV_RDM_RMA_PKT) {
- struct fi_verbs_rdm_tagged_request_minfo minfo = {
+ struct fi_ibv_rdm_minfo minfo = {
.conn = conn,
.tag = rbuf->header.tag,
- .tagmask = 0
+ .tagmask = 0,
+ .is_tagged = (pkt_type == FI_IBV_RDM_MSG_PKT) ? 0 : 1
};
+ if (pkt_type == FI_IBV_RDM_RNDV_RTS_PKT) {
+ struct fi_ibv_rdm_rndv_header* h = (void *)&rbuf->header;
+ minfo.is_tagged = h->is_tagged;
+ }
+
struct dlist_entry *found_entry =
- dlist_find_first_match(&fi_ibv_rdm_tagged_recv_posted_queue,
- fi_ibv_rdm_tagged_req_match_by_info,
- &minfo);
+ dlist_find_first_match(&fi_ibv_rdm_posted_queue,
+ fi_ibv_rdm_req_match_by_info,
+ &minfo);
if (found_entry) {
- struct fi_ibv_rdm_tagged_request *found_request =
- container_of(found_entry,
- struct fi_ibv_rdm_tagged_request,
- queue_entry);
-
- const int data_len = arrived_len -
- sizeof(struct fi_ibv_rdm_header);
-
- if (found_request->len < data_len) {
- VERBS_INFO(FI_LOG_EP_DATA,
- "%s: %d RECV TRUNCATE, data_len=%d, posted_len=%d, "
- "conn %p, tag 0x%llx, tagmask %llx\n",
- __FUNCTION__, __LINE__, data_len,
- found_request->len, found_request->conn,
- found_request->tag,
- found_request->tagmask);
- assert(0);
- }
+ struct fi_ibv_rdm_request *found_request =
+ container_of(found_entry,
+ struct fi_ibv_rdm_request,
+ queue_entry);
+
+ fi_ibv_rdm_remove_from_posted_queue(found_request, ep);
- fi_ibv_rdm_tagged_remove_from_posted_queue
- (found_request, ep);
request = found_request;
} else {
- request = util_buf_alloc(fi_ibv_rdm_tagged_request_pool);
- fi_ibv_rdm_tagged_zero_request(request);
+ request = util_buf_alloc(fi_ibv_rdm_request_pool);
+ fi_ibv_rdm_zero_request(request);
- FI_IBV_RDM_TAGGED_DBG_REQUEST("get_from_pool: ",
- request, FI_LOG_DEBUG);
+ FI_IBV_RDM_DBG_REQUEST("get_from_pool: ", request,
+ FI_LOG_DEBUG);
}
}
@@ -553,29 +506,26 @@ fi_ibv_rdm_tagged_process_recv(struct fi_ibv_rdm_ep *ep,
.rbuf = rbuf,
.arrived_len = arrived_len,
.pkt_type = pkt_type,
- .imm_data = imm_data
+ .imm_data = 0 // TODO:
};
- fi_ibv_rdm_tagged_req_hndl(request,
- FI_IBV_EVENT_RECV_GOT_PKT_PROCESS, &p);
-
- conn->recv_completions++;
- if (conn->recv_completions & ep->n_buffs) {
- conn->recv_completions = 0;
- fi_ibv_rdm_tagged_release_remote_sbuff(conn, ep);
- }
+ fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_RECV_GOT_PKT_PROCESS,
+ &p);
}
}
static inline
void check_and_repost_receives(struct fi_ibv_rdm_ep *ep,
- struct fi_ibv_rdm_tagged_conn *conn)
+ struct fi_ibv_rdm_conn *conn)
{
+ conn->recv_preposted--;
+ VERBS_DBG(FI_LOG_EP_DATA, "conn %p remain prepost recvs %d\n", conn, conn->recv_preposted);
if (conn->recv_preposted < ep->recv_preposted_threshold) {
int to_post = ep->rq_wr_depth - conn->recv_preposted;
- if (fi_ibv_rdm_tagged_repost_receives(conn, ep, to_post) !=
- to_post) {
- VERBS_INFO(FI_LOG_EP_DATA, "repost recv failed\n");
+ ssize_t res = fi_ibv_rdm_repost_receives(conn, ep, to_post);
+ if (res < 0) {
+ VERBS_INFO(FI_LOG_EP_DATA, "repost recv failed %d\n", res);
+ /* TODO: err code propagation */
abort();
}
VERBS_DBG(FI_LOG_EP_DATA,
@@ -584,143 +534,200 @@ void check_and_repost_receives(struct fi_ibv_rdm_ep *ep,
}
}
-static inline void
-fi_ibv_rdm_tagged_got_recv_completion(struct fi_ibv_rdm_ep *ep,
- struct fi_ibv_rdm_tagged_conn *conn,
- int arrived_len, uint32_t imm_data)
+static inline int
+fi_ibv_rdm_process_recv_wc(struct fi_ibv_rdm_ep *ep, struct ibv_wc *wc)
{
- assert(arrived_len > 0);
- conn->recv_preposted--;
- check_and_repost_receives(ep, conn);
+ struct fi_ibv_rdm_conn *conn = (void *) wc->wr_id;
+
struct fi_ibv_rdm_buf *rbuf =
- fi_ibv_rdm_tagged_get_rbuf(conn, ep, imm_data);
- fi_ibv_rdm_tagged_process_recv(ep, conn, arrived_len, imm_data, rbuf);
+ fi_ibv_rdm_get_rbuf(conn, ep, conn->recv_processed);
+
+ FI_IBV_PREFETCH_ADDR(rbuf);
+
+ FI_IBV_DBG_OPCODE(wc->opcode, "RECV");
+
+ if (!FI_IBV_RDM_CHECK_RECV_WC(wc)) {
+
+ VERBS_INFO(FI_LOG_EP_DATA, "conn %p state %d, wc status %d\n",
+ conn, conn->state, wc->status);
+ if (wc->status == IBV_WC_WR_FLUSH_ERR &&
+ conn->state == FI_VERBS_CONN_ESTABLISHED)
+ {
+ /*
+ * It means that remote side initiated disconnection
+ * and QP is flushed earlier then disconnect event was
+ * handled or arrived. Just initiate disconnect to
+ * opposite direction.
+ */
+ fi_ibv_rdm_start_disconnection(conn);
+ } else {
+ assert("Error recv wc\n" &&
+ (!ep->is_closing ||
+ conn->state != FI_VERBS_CONN_ESTABLISHED));
+ }
+
+ return 1;
+ }
+
+ conn->recv_completions++;
+ if (conn->recv_completions & ep->n_buffs) {
+ conn->recv_completions = 0;
+ }
+
+ VERBS_DBG(FI_LOG_EP_DATA, "conn %p recv_completions %d\n",
+ conn, conn->recv_completions);
+
+ check_and_repost_receives(ep, conn);
+
+ if ((rbuf->service_data.status == BUF_STATUS_RECVED) &&
+ /* NOTE: the ibverbs bug?
+ * In case of out of order arriving we may check seq_num only if
+ * send was posted with IBV_WR_RDMA_WRITE_WITH_IMM opcode because
+ * the sender controls this.
+ * Otherwise, the sender with IBV_WR_SEND opcode consumes pre-posted
+ * buffers in the same order as they were pre-posted by recv.
+ * So, we should handle it as is. Potentially, this way may cause
+ * broken ordering of completions in fi_cq.
+ */
+ (wc->opcode == IBV_WC_RECV_RDMA_WITH_IMM ?
+ fi_ibv_rdm_buffer_check_seq_num(rbuf, conn->recv_processed) : 1))
+ {
+ do {
+ assert(rbuf->service_data.pkt_len > 0);
+
+ fi_ibv_rdm_process_recv(ep, conn,
+ rbuf->service_data.pkt_len, rbuf);
+
+ VERBS_DBG(FI_LOG_EP_DATA, "processed: conn %p, pkt # %d\n",
+ conn, rbuf->service_data.seq_num);
+
+ fi_ibv_rdm_set_buffer_status(rbuf, BUF_STATUS_FREE);
+ rbuf->service_data.seq_num = (uint16_t)(-1);
+
+ conn->recv_processed++;
+ if (conn->recv_processed & ep->n_buffs) {
+ conn->recv_processed = 0;
+ fi_ibv_rdm_tagged_release_remote_sbuff(conn, ep);
+ }
+ VERBS_DBG(FI_LOG_EP_DATA, "conn %p recv_processed %d\n",
+ conn, conn->recv_processed);
+
+ rbuf = fi_ibv_rdm_get_rbuf(conn, ep,
+ conn->recv_processed);
+
+ /* Do not process w/o completion! */
+ } while (conn->recv_processed != conn->recv_completions &&
+ rbuf->service_data.status == BUF_STATUS_RECVED);
+ } else {
+ VERBS_DBG(FI_LOG_EP_DATA, "not processed: conn %p, status: %d\n",
+ conn, rbuf->service_data.status);
+ }
+
+ return 0;
}
static inline int fi_ibv_rdm_tagged_poll_recv(struct fi_ibv_rdm_ep *ep)
{
- const int wc_count = ep->n_buffs; // TODO: to set from upper level
+ const int wc_count = ep->fi_rcq->read_bunch_size;
struct ibv_wc wc[wc_count];
- int i;
- int ret;
+ int ret = 0;
+ int err = 0;
+ int i = 0;
do {
ret = ibv_poll_cq(ep->rcq, wc_count, wc);
- for (i = 0; i < ret; ++i) {
- if (wc[i].status == IBV_WC_SUCCESS
- && wc[i].opcode == IBV_WC_RECV_RDMA_WITH_IMM) {
- FI_IBV_DBG_OPCODE(wc[i].opcode, "RECV");
-
- struct fi_ibv_rdm_tagged_conn *conn =
- (struct fi_ibv_rdm_tagged_conn *)(uintptr_t)
- wc[i].wr_id;
-
- FI_IBV_PREFETCH_ADDR(
- fi_ibv_rdm_tagged_get_rbuf(conn,
- ep,
- wc[i].imm_data));
-
-
- fi_ibv_rdm_tagged_got_recv_completion(ep, conn,
- wc[i].
- byte_len,
- wc[i].
- imm_data);
- } else {
- goto wc_error;
- }
+ for (i = 0; i < ret && !err; ++i) {
+ err = fi_ibv_rdm_process_recv_wc(ep, &wc[i]);
}
- } while (ret == wc_count);
+ } while (!err && ret == wc_count);
- if (ret >= 0) // Success
- {
- return 0;
+ if (!err && ret >= 0) {
+ return FI_SUCCESS;
}
-wc_error:
+ /* error handling */
+
VERBS_INFO(FI_LOG_EP_DATA, "ibv_poll_cq returned %d\n", ret);
- if (wc[i].status != IBV_WC_SUCCESS) {
- struct fi_ibv_rdm_tagged_conn *conn =
- (struct fi_ibv_rdm_tagged_conn *)(uintptr_t)wc[i].wr_id;
- if (wc[i].status == IBV_WC_WR_FLUSH_ERR &&
- conn && conn->state != FI_VERBS_CONN_ESTABLISHED)
- {
- return FI_SUCCESS;
+ for(i = 0; i < wc_count; i++) {
+
+ if (wc[i].status != IBV_WC_SUCCESS) {
+ struct fi_ibv_rdm_conn *conn = (void *)wc[i].wr_id;
+
+ if (wc[i].status == IBV_WC_WR_FLUSH_ERR && conn &&
+ conn->state != FI_VERBS_CONN_ESTABLISHED)
+ {
+ return FI_SUCCESS;
+ }
+
+ VERBS_INFO(FI_LOG_EP_DATA, "got ibv_wc[%d].status = %d:%s\n",
+ i, wc[i].status, ibv_wc_status_str(wc[i].status));
+ return -FI_EOTHER;
}
- VERBS_INFO(FI_LOG_EP_DATA, "got ibv_wc.status = %d:%s\n",
- wc[i].status, ibv_wc_status_str(wc[i].status));
- assert(0);
- return -FI_EOTHER;
+ if (wc[i].opcode != IBV_WC_RECV_RDMA_WITH_IMM &&
+ wc[i].opcode != IBV_WC_RECV)
+ {
+ VERBS_INFO(FI_LOG_EP_DATA, "got ibv_wc[%d].opcode = %d\n",
+ i, wc[i].opcode);
+ }
}
- if (wc[i].opcode != IBV_WC_RECV_RDMA_WITH_IMM) {
- VERBS_INFO(FI_LOG_EP_DATA, "got ibv_wc[i].opcode = %d\n",
- wc[i].opcode);
+ return -FI_EOTHER;
+}
+
+static inline int
+fi_ibv_rdm_process_send_wc(struct fi_ibv_rdm_ep *ep, struct ibv_wc *wc)
+{
+ if (wc->status != IBV_WC_SUCCESS) {
+ return 1;
}
- assert(0);
+ if (FI_IBV_RDM_CHECK_SERVICE_WR_FLAG(wc->wr_id)) {
+ VERBS_DBG(FI_LOG_EP_DATA, "CQ COMPL: SEND -> 0x1\n");
+ struct fi_ibv_rdm_conn *conn =
+ (struct fi_ibv_rdm_conn *)
+ FI_IBV_RDM_UNPACK_SERVICE_WR(wc->wr_id);
+ FI_IBV_RDM_DEC_SIG_POST_COUNTERS(conn, ep);
- return -FI_EOTHER;
+ return 0;
+ } else {
+ FI_IBV_DBG_OPCODE(wc->opcode, "SEND");
+ struct fi_ibv_rdm_request *request =
+ (void *)FI_IBV_RDM_UNPACK_WR(wc->wr_id);
+
+ struct fi_ibv_rdm_tagged_send_completed_data data =
+ { .ep = ep };
+
+ return fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_POST_LC, &data);
+ }
}
static inline int fi_ibv_rdm_tagged_poll_send(struct fi_ibv_rdm_ep *ep)
{
- const int wc_count = ep->n_buffs;
+ const int wc_count = ep->fi_scq->read_bunch_size;
struct ibv_wc wc[wc_count];
- int i = 0;
int ret = 0;
+ int err = 0;
+ int i = 0;
if (ep->posted_sends > 0) {
do {
ret = ibv_poll_cq(ep->scq, wc_count, wc);
- for (i = 0; i < ret; ++i) {
- if (wc[i].status == IBV_WC_SUCCESS) {
- if (FI_IBV_RDM_CHECK_SERVICE_WR_FLAG
- (wc[i].wr_id)) {
- VERBS_DBG(FI_LOG_EP_DATA,
- "CQ COMPL: SEND -> 0x1\n");
- struct fi_ibv_rdm_tagged_conn
- *conn =
- (struct
- fi_ibv_rdm_tagged_conn *)
- FI_IBV_RDM_UNPACK_SERVICE_WR
- (wc[i].wr_id);
- FI_IBV_RDM_TAGGED_DEC_SEND_COUNTERS
- (conn, ep);
- } else {
- FI_IBV_DBG_OPCODE(wc[i].opcode,
- "SEND");
- struct fi_ibv_rdm_tagged_request
- *request =
- (struct
- fi_ibv_rdm_tagged_request
- *)
- FI_IBV_RDM_UNPACK_WR(wc[i].
- wr_id);
-
- struct fi_ibv_rdm_tagged_send_completed_data
- data = {
- .ep = ep
- };
-
- fi_ibv_rdm_tagged_req_hndl
- (request,
- FI_IBV_EVENT_SEND_GOT_LC,
- &data);
- }
- } else {
- goto wc_error;
- }
+ for (i = 0; i < ret && !err; ++i) {
+ err = fi_ibv_rdm_process_send_wc(ep, &wc[i]);
}
- } while (ret == wc_count);
+ } while (!err && ret == wc_count);
+ }
+
+ if (err || ret < 0) {
+ goto wc_error;
}
- struct fi_ibv_rdm_tagged_send_ready_data data = {.ep = ep };
+ struct fi_ibv_rdm_tagged_send_ready_data data = { .ep = ep };
struct dlist_entry *item;
- dlist_foreach((&fi_ibv_rdm_tagged_send_postponed_queue), item) {
- if (fi_ibv_rdm_tagged_send_postponed_process(item, &data)) {
+ dlist_foreach((&fi_ibv_rdm_postponed_queue), item) {
+ if (fi_ibv_rdm_postponed_process(item, &data)) {
/* we can't process all postponed items till foreach */
/* implementation is not safety for removing during */
/* iterating */
@@ -728,26 +735,24 @@ static inline int fi_ibv_rdm_tagged_poll_send(struct fi_ibv_rdm_ep *ep)
}
}
- if (ret >= 0) // Success
- {
- return 0;
- }
+ return FI_SUCCESS;
wc_error:
- if (ret < 0 || wc[i].status != IBV_WC_SUCCESS) {
- if (ret < 0) {
- VERBS_INFO(FI_LOG_EP_DATA, "ibv_poll_cq returned %d\n",
- ret);
- assert(0);
- }
+ if (ret < 0) {
+ VERBS_INFO(FI_LOG_EP_DATA, "ibv_poll_cq returned %d\n", ret);
+ assert(0);
+ }
- if (wc[i].status != IBV_WC_SUCCESS && ret != 0) {
- struct fi_ibv_rdm_tagged_conn *conn =
- (FI_IBV_RDM_CHECK_SERVICE_WR_FLAG(wc[i].wr_id))
- ? (struct fi_ibv_rdm_tagged_conn *)
- FI_IBV_RDM_UNPACK_SERVICE_WR(wc[i].wr_id)
- : ((struct fi_ibv_rdm_tagged_request *)wc[i].
- wr_id)->conn;
+ for (i = 0; i < wc_count; i++) {
+ if (wc[i].status != IBV_WC_SUCCESS) {
+ struct fi_ibv_rdm_conn *conn;
+ if (FI_IBV_RDM_CHECK_SERVICE_WR_FLAG(wc[i].wr_id)) {
+ conn = FI_IBV_RDM_UNPACK_SERVICE_WR(wc[i].wr_id);
+ } else {
+ struct fi_ibv_rdm_request *req =
+ (void *)wc[i].wr_id;
+ conn = req->minfo.conn;
+ }
VERBS_INFO(FI_LOG_EP_DATA,
"got ibv_wc.status = %d:%s, pend_send: %d, connection: %p\n",
@@ -757,24 +762,17 @@ wc_error:
assert(0);
}
}
+
return -FI_EOTHER;
}
int fi_ibv_rdm_tagged_poll(struct fi_ibv_rdm_ep *ep)
{
int ret = fi_ibv_rdm_tagged_poll_send(ep);
- if (ret) {
+ /* Only already posted sends should be processed during EP closing */
+ if (ret || ep->is_closing) {
return ret;
}
return fi_ibv_rdm_tagged_poll_recv(ep);
}
-
-static inline int
-fi_ibv_rdm_tagged_init_request_sbuf(struct fi_ibv_rdm_tagged_request *request,
- struct fi_ibv_rdm_ep *ep)
-{
- assert(request->sbuf == NULL);
- request->sbuf = fi_ibv_rdm_tagged_get_sbuf_head(request->conn, ep);
- return !!request->sbuf;
-}
diff --git a/prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c b/prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c
index 53d5832..3223db9 100644
--- a/prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c
+++ b/prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.c
@@ -39,57 +39,58 @@
#include "verbs_queuing.h"
#include "verbs_tagged_ep_rdm_states.h"
-extern struct dlist_entry fi_ibv_rdm_tagged_send_postponed_queue;
-extern struct util_buf_pool *fi_ibv_rdm_tagged_request_pool;
-extern struct util_buf_pool *fi_ibv_rdm_tagged_extra_buffers_pool;
+extern struct dlist_entry fi_ibv_rdm_postponed_queue;
+extern struct util_buf_pool *fi_ibv_rdm_request_pool;
+extern struct util_buf_pool *fi_ibv_rdm_extra_buffers_pool;
-typedef enum fi_rdm_tagged_req_hndl_ret (*fi_ep_rdm_request_handler_t)
- (struct fi_ibv_rdm_tagged_request * request, void *data);
+typedef ssize_t (*fi_ep_rdm_request_handler_t)
+ (struct fi_ibv_rdm_request *request, void *data);
static fi_ep_rdm_request_handler_t
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_COUNT]
- [FI_IBV_STATE_RNDV_COUNT]
- [FI_IBV_EVENT_COUNT];
+ fi_ibv_rdm_req_hndl_arr [FI_IBV_STATE_EAGER_COUNT]
+ [FI_IBV_STATE_RNDV_COUNT]
+ [FI_IBV_EVENT_COUNT];
#if ENABLE_DEBUG
-enum fi_ibv_rdm_tagged_hndl_req_log_state {
+enum fi_ibv_rdm_hndl_req_log_state {
hndl_req_log_state_in = 0,
hndl_req_log_state_out = 10000
};
-#define FI_IBV_RDM_TAGGED_HANDLER_LOG_IN() \
-enum fi_ibv_rdm_tagged_hndl_req_log_state state = hndl_req_log_state_in; \
-do { \
- FI_IBV_RDM_TAGGED_DBG_REQUEST("\t> IN\t< ", request, FI_LOG_DEBUG); \
+#define FI_IBV_RDM_HNDL_REQ_LOG_IN() \
+enum fi_ibv_rdm_hndl_req_log_state state = hndl_req_log_state_in; \
+do { \
+ FI_IBV_RDM_DBG_REQUEST("\t> IN\t< ", request, FI_LOG_DEBUG); \
} while(0)
-#define FI_IBV_RDM_TAGGED_HANDLER_LOG() do { \
- state++; \
- char prefix[128]; \
- snprintf(prefix, 128, "\t> %d\t< ", state); \
- FI_IBV_RDM_TAGGED_DBG_REQUEST(prefix, request, FI_LOG_DEBUG); \
+#define FI_IBV_RDM_HNDL_REQ_LOG() do { \
+ state++; \
+ char prefix[128]; \
+ snprintf(prefix, 128, "\t> %d\t< ", state); \
+ FI_IBV_RDM_DBG_REQUEST(prefix, request, FI_LOG_DEBUG); \
} while(0)
-#define FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT() do { \
- assert(state < hndl_req_log_state_out); \
- FI_IBV_RDM_TAGGED_DBG_REQUEST("\t> OUT\t< ", request, FI_LOG_DEBUG); \
+#define FI_IBV_RDM_HNDL_REQ_LOG_OUT() do { \
+ assert(state < hndl_req_log_state_out); \
+ FI_IBV_RDM_DBG_REQUEST("\t> OUT\t< ", request, FI_LOG_DEBUG); \
} while(0)
#else // ENABLE_DEBUG
-#define FI_IBV_RDM_TAGGED_HANDLER_LOG_IN()
-#define FI_IBV_RDM_TAGGED_HANDLER_LOG()
-#define FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT()
+#define FI_IBV_RDM_HNDL_REQ_LOG_IN()
+#define FI_IBV_RDM_HNDL_REQ_LOG()
+#define FI_IBV_RDM_HNDL_REQ_LOG_OUT()
#endif // ENABLE_DEBUG
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_init_send_request(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+static ssize_t
+fi_ibv_rdm_init_send_request(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
- struct fi_ibv_rdm_tagged_send_start_data *p = data;
- request->tag = p->tag;
+ struct fi_ibv_rdm_send_start_data *p = data;
+ request->minfo.conn = p->conn;
+ request->minfo.tag = p->tag;
+ request->minfo.is_tagged = p->is_tagged;
request->iov_count = p->iov_count;
/* Indeed, both branches are the same, just for readability */
@@ -99,45 +100,45 @@ fi_ibv_rdm_tagged_init_send_request(struct fi_ibv_rdm_tagged_request *request,
request->src_addr = p->buf.src_addr;
}
+ request->sbuf = NULL;
request->len = p->data_len;
+ request->comp_flags = p->flags;
request->imm = p->imm;
request->context = p->context;
- request->conn = p->conn;
request->state.eager = FI_IBV_STATE_EAGER_BEGIN;
request->state.rndv =
(p->data_len + sizeof(struct fi_ibv_rdm_header)
<= p->ep_rdm->rndv_threshold)
? FI_IBV_STATE_RNDV_NOT_USED : FI_IBV_STATE_RNDV_SEND_BEGIN;
- FI_IBV_RDM_TAGGED_HANDLER_LOG();
+ FI_IBV_RDM_HNDL_REQ_LOG();
- fi_ibv_rdm_tagged_move_to_postponed_queue(request);
+ fi_ibv_rdm_move_to_postponed_queue(request);
request->state.eager = FI_IBV_STATE_EAGER_SEND_POSTPONED;
if (request->state.rndv == FI_IBV_STATE_RNDV_SEND_BEGIN) {
request->state.rndv = FI_IBV_STATE_RNDV_SEND_WAIT4SEND;
}
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return FI_SUCCESS;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_eager_send_ready(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+static ssize_t
+fi_ibv_rdm_eager_send_ready(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
assert(request->state.eager == FI_IBV_STATE_EAGER_SEND_POSTPONED);
assert(request->state.rndv == FI_IBV_STATE_RNDV_NOT_USED);
- fi_ibv_rdm_tagged_remove_from_postponed_queue(request);
+ fi_ibv_rdm_remove_from_postponed_queue(request);
struct fi_ibv_rdm_tagged_send_ready_data *p = data;
- int ret = 0;
+ ssize_t ret = FI_SUCCESS;
struct ibv_sge sge;
- struct fi_ibv_rdm_tagged_conn *conn = request->conn;
- int size = request->len + sizeof(struct fi_ibv_rdm_header);
+ struct fi_ibv_rdm_conn *conn = request->minfo.conn;
+ const int size = request->len + sizeof(struct fi_ibv_rdm_header);
assert(request->sbuf);
@@ -147,13 +148,13 @@ fi_ibv_rdm_tagged_eager_send_ready(struct fi_ibv_rdm_tagged_request *request,
wr.sg_list = &sge;
wr.num_sge = 1;
- wr.wr.rdma.remote_addr =
- fi_ibv_rdm_tagged_get_remote_addr(conn, request->sbuf);
+ wr.wr.rdma.remote_addr = fi_ibv_rdm_get_remote_addr(conn, request->sbuf);
wr.wr.rdma.rkey = conn->remote_rbuf_rkey;
wr.send_flags = 0;
- sge.addr = (uintptr_t) request->sbuf;
- sge.length = size;
+ sge.addr = (uintptr_t)request->sbuf;
+ sge.length = size + FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE;
+ request->sbuf->service_data.pkt_len = size;
if (sge.length <= p->ep->max_inline_rc) {
wr.send_flags |= IBV_SEND_INLINE;
@@ -161,15 +162,18 @@ fi_ibv_rdm_tagged_eager_send_ready(struct fi_ibv_rdm_tagged_request *request,
sge.lkey = conn->s_mr->lkey;
- wr.imm_data =
- fi_ibv_rdm_tagged_get_buff_service_data(request->sbuf)->seq_number;
- wr.opcode = IBV_WR_RDMA_WRITE_WITH_IMM;
+ wr.imm_data = 0;
+ wr.opcode = p->ep->eopcode;
struct fi_ibv_rdm_buf *sbuf = (struct fi_ibv_rdm_buf *)request->sbuf;
- char *payload = &(sbuf->payload[0]);
+ uint8_t *payload = &sbuf->payload;
- sbuf->header.tag = request->tag;
sbuf->header.service_tag = 0;
- FI_IBV_RDM_SET_PKTTYPE(sbuf->header.service_tag, FI_IBV_RDM_EAGER_PKT);
+ if (request->minfo.is_tagged) {
+ sbuf->header.tag = request->minfo.tag;
+ FI_IBV_RDM_SET_PKTTYPE(sbuf->header.service_tag, FI_IBV_RDM_EAGER_PKT);
+ } else {
+ FI_IBV_RDM_SET_PKTTYPE(sbuf->header.service_tag, FI_IBV_RDM_MSG_PKT);
+ }
if (request->len > 0) {
if (request->iov_count == 0) {
@@ -184,79 +188,83 @@ fi_ibv_rdm_tagged_eager_send_ready(struct fi_ibv_rdm_tagged_request *request,
}
}
- FI_IBV_RDM_INC_SIG_POST_COUNTERS(request->conn, p->ep, wr.send_flags);
+ FI_IBV_RDM_INC_SIG_POST_COUNTERS(request->minfo.conn,
+ p->ep, wr.send_flags);
VERBS_DBG(FI_LOG_EP_DATA, "posted %d bytes, conn %p, tag 0x%llx\n",
- sge.length, request->conn, request->tag);
+ sge.length, request->minfo.conn, request->minfo.tag);
ret = ibv_post_send(conn->qp[0], &wr, &bad_wr);
if (ret) {
VERBS_INFO_ERRNO(FI_LOG_EP_DATA, "ibv_post_send", errno);
-
+ ret = -errno;
assert(0);
};
- fi_ibv_rdm_tagged_move_to_ready_queue(request);
- request->state.eager = FI_IBV_STATE_EAGER_SEND_WAIT4LC;
+ fi_ibv_rdm_cntr_inc(p->ep->send_cntr);
+
+ if (request->comp_flags & FI_COMPLETION) {
+ fi_ibv_rdm_move_to_cq(p->ep->fi_scq, request);
+ request->state.eager = FI_IBV_STATE_EAGER_SEND_WAIT4LC;
+ } else {
+ request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
+ }
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
return ret;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_eager_send_lc(struct fi_ibv_rdm_tagged_request *request,
+static ssize_t
+fi_ibv_rdm_eager_send_lc(struct fi_ibv_rdm_request *request,
void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
assert(request->state.eager == FI_IBV_STATE_EAGER_SEND_WAIT4LC ||
request->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE);
assert(request->state.rndv == FI_IBV_STATE_RNDV_NOT_USED);
- VERBS_DBG(FI_LOG_EP_DATA, "conn %p, tag 0x%llx, len %d\n", request->conn,
- request->tag, request->len);
+ VERBS_DBG(FI_LOG_EP_DATA, "conn %p, tag 0x%llx, len %d\n",
+ request->minfo.conn, request->minfo.tag, request->len);
struct fi_ibv_rdm_tagged_send_completed_data *p = data;
- FI_IBV_RDM_TAGGED_DEC_SEND_COUNTERS(request->conn, p->ep);
+ FI_IBV_RDM_DEC_SIG_POST_COUNTERS(request->minfo.conn, p->ep);
if (request->iov_count) {
- util_buf_release(fi_ibv_rdm_tagged_extra_buffers_pool,
+ util_buf_release(fi_ibv_rdm_extra_buffers_pool,
request->iovec_arr);
}
if (request->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE) {
- FI_IBV_RDM_TAGGED_DBG_REQUEST("to_pool: ", request,
- FI_LOG_DEBUG);
- util_buf_release(fi_ibv_rdm_tagged_request_pool, request);
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
} else {
request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
}
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return FI_SUCCESS;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_rndv_rts_send_ready(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+static ssize_t
+fi_ibv_rdm_rndv_rts_send_ready(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
assert(request->state.eager == FI_IBV_STATE_EAGER_SEND_POSTPONED);
assert(request->state.rndv == FI_IBV_STATE_RNDV_SEND_WAIT4SEND);
assert(request->sbuf);
- VERBS_DBG(FI_LOG_EP_DATA, "conn %p, tag 0x%llx, len %d\n", request->conn,
- request->tag, request->len);
+ VERBS_DBG(FI_LOG_EP_DATA, "conn %p, tag 0x%llx, len %d\n",
+ request->minfo.conn, request->minfo.tag, request->len);
- fi_ibv_rdm_tagged_remove_from_postponed_queue(request);
+ fi_ibv_rdm_remove_from_postponed_queue(request);
struct fi_ibv_rdm_tagged_send_ready_data *p = data;
struct ibv_sge sge;
- struct fi_ibv_rdm_tagged_conn *conn = request->conn;
- struct fi_ibv_rdm_tagged_rndv_header *header =
- (struct fi_ibv_rdm_tagged_rndv_header *)request->sbuf;
+ struct fi_ibv_rdm_conn *conn = request->minfo.conn;
+ struct fi_ibv_rdm_rndv_header *header = (void *)&request->sbuf->header;
struct ibv_mr *mr = NULL;
struct ibv_send_wr wr, *bad_wr = NULL;
@@ -266,112 +274,116 @@ fi_ibv_rdm_tagged_rndv_rts_send_ready(struct fi_ibv_rdm_tagged_request *request,
wr.sg_list = &sge;
wr.num_sge = 1;
- wr.wr.rdma.remote_addr =
- fi_ibv_rdm_tagged_get_remote_addr(conn, request->sbuf);
+ wr.wr.rdma.remote_addr = (uintptr_t)
+ fi_ibv_rdm_get_remote_addr(conn, request->sbuf);
wr.wr.rdma.rkey = conn->remote_rbuf_rkey;
wr.send_flags = 0;
- wr.opcode = IBV_WR_RDMA_WRITE_WITH_IMM;
- wr.imm_data =
- fi_ibv_rdm_tagged_get_buff_service_data(request->sbuf)->seq_number;
+ wr.opcode = p->ep->eopcode;
+ wr.imm_data = 0;
- sge.addr = (uintptr_t) request->sbuf;
- sge.length = sizeof(struct fi_ibv_rdm_tagged_rndv_header);
+ sge.addr = (uintptr_t)request->sbuf;
+ sge.length = FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE + sizeof(*header);
sge.lkey = conn->s_mr->lkey;
+ request->sbuf->service_data.pkt_len = sizeof(*header);
- header->base.tag = request->tag;
+ if (request->minfo.is_tagged) {
+ header->base.tag = request->minfo.tag;
+ header->is_tagged = 1;
+ } else {
+ header->is_tagged = 0;
+ }
header->base.service_tag = 0;
- header->len = request->len;
- header->src_addr = (uint64_t) (uintptr_t) request->src_addr;
+ header->total_len = request->len;
+ header->src_addr = (uintptr_t)request->src_addr;
- header->id = request;
- request->rndv.id = request;
+ header->id = (uintptr_t)request;
+ request->rndv.id = (uintptr_t)request;
mr = ibv_reg_mr(p->ep->domain->pd, (void *)request->src_addr,
request->len, IBV_ACCESS_REMOTE_READ);
- if (NULL == mr) {
+ if (!mr) {
VERBS_INFO_ERRNO(FI_LOG_EP_DATA, "ibv_reg_mr", errno);
assert(0);
- return FI_EOTHER;
+ return -errno;
}
header->mem_key = mr->rkey;
request->rndv.mr = mr;
- struct fi_ibv_rdm_buf *sbuf = request->sbuf;
-
- FI_IBV_RDM_SET_PKTTYPE(sbuf->header.service_tag,
+ FI_IBV_RDM_SET_PKTTYPE(header->base.service_tag,
FI_IBV_RDM_RNDV_RTS_PKT);
VERBS_DBG(FI_LOG_EP_DATA,
"fi_senddatato: RNDV conn %p, tag 0x%llx, len %d, src_addr %p,"
"rkey 0x%lx, fi_ctx %p, imm %d, post_send %d\n", conn,
- (long long unsigned int)request->tag, (int)request->len,
- request->src_addr, (long unsigned int)mr->rkey, request->context,
+ request->minfo.tag, (int)request->len,
+ request->src_addr, mr->rkey, request->context,
(int)wr.imm_data, p->ep->posted_sends);
- FI_IBV_RDM_INC_SIG_POST_COUNTERS(request->conn, p->ep, wr.send_flags);
+ FI_IBV_RDM_INC_SIG_POST_COUNTERS(request->minfo.conn, p->ep,
+ wr.send_flags);
VERBS_DBG(FI_LOG_EP_DATA, "posted %d bytes, conn %p, tag 0x%llx\n",
- sge.length, request->conn, request->tag);
+ sge.length, request->minfo.conn, request->minfo.tag);
int ret = ibv_post_send(conn->qp[0], &wr, &bad_wr);
if (ret) {
VERBS_INFO_ERRNO(FI_LOG_EP_DATA, "ibv_post_send", errno);
assert(0);
+ return -errno;
};
request->state.eager = FI_IBV_STATE_EAGER_SEND_WAIT4LC;
request->state.rndv = FI_IBV_STATE_RNDV_SEND_WAIT4ACK;
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return FI_SUCCESS;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_rndv_rts_lc(struct fi_ibv_rdm_tagged_request *request,
+static ssize_t
+fi_ibv_rdm_rndv_rts_lc(struct fi_ibv_rdm_request *request,
void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
- enum fi_rdm_tagged_req_hndl_ret ret = FI_EP_RDM_HNDL_SUCCESS;
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
assert(((request->state.eager == FI_IBV_STATE_EAGER_SEND_WAIT4LC) &&
(request->state.rndv == FI_IBV_STATE_RNDV_SEND_WAIT4ACK)) ||
((request->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE) &&
+ (request->state.rndv == FI_IBV_STATE_RNDV_SEND_END)) ||
+ ((request->state.eager == FI_IBV_STATE_EAGER_SEND_WAIT4LC) &&
(request->state.rndv == FI_IBV_STATE_RNDV_SEND_END)));
- assert(request->conn);
+ assert(request->minfo.conn);
- VERBS_DBG(FI_LOG_EP_DATA, "conn %p, tag 0x%llx, len %d\n", request->conn,
- request->tag, request->len);
+ VERBS_DBG(FI_LOG_EP_DATA, "conn %p, tag 0x%llx, len %d\n",
+ request->minfo.conn, request->minfo.tag, request->len);
struct fi_ibv_rdm_tagged_send_completed_data *p = data;
- FI_IBV_RDM_TAGGED_DEC_SEND_COUNTERS(request->conn, p->ep);
+ FI_IBV_RDM_DEC_SIG_POST_COUNTERS(request->minfo.conn, p->ep);
if (request->state.eager == FI_IBV_STATE_EAGER_SEND_WAIT4LC) {
request->state.eager = FI_IBV_STATE_EAGER_SEND_END;
} else { /* (request->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE) */
- FI_IBV_RDM_TAGGED_DBG_REQUEST("to_pool: ", request,
- FI_LOG_DEBUG);
- util_buf_release(fi_ibv_rdm_tagged_request_pool, request);
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
}
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
- return ret;
+ return FI_SUCCESS;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_rndv_end(struct fi_ibv_rdm_tagged_request *request,
+static ssize_t
+fi_ibv_rdm_rndv_end(struct fi_ibv_rdm_request *request,
void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ struct fi_ibv_recv_got_pkt_preprocess_data *p = data;
+
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
+
assert(request->state.eager == FI_IBV_STATE_EAGER_SEND_END ||
(request->state.eager == FI_IBV_STATE_EAGER_SEND_WAIT4LC));
assert(request->state.rndv == FI_IBV_STATE_RNDV_SEND_WAIT4ACK);
-#ifndef NDEBUG
- struct fi_ibv_recv_got_pkt_preprocess_data *p = data;
-#endif /* NDEBUG */
-
- assert((sizeof(struct fi_ibv_rdm_tagged_request *) +
+ assert((sizeof(struct fi_ibv_rdm_request *) +
sizeof(struct fi_ibv_rdm_header)) == p->arrived_len);
assert(request->rndv.mr);
assert(p->rbuf);
@@ -386,149 +398,277 @@ fi_ibv_rdm_tagged_rndv_end(struct fi_ibv_rdm_tagged_request *request,
}
request->state.rndv = FI_IBV_STATE_RNDV_SEND_END;
+ FI_IBV_RDM_HNDL_REQ_LOG();
+
+ fi_ibv_rdm_cntr_inc(p->ep->send_cntr);
- fi_ibv_rdm_tagged_move_to_ready_queue(request);
+ if (request->comp_flags & FI_COMPLETION) {
+ fi_ibv_rdm_move_to_cq(p->ep->fi_scq, request);
+ } else if (request->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE) {
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
+ }
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ return FI_SUCCESS;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_init_recv_request(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+
+static inline ssize_t
+fi_ibv_rdm_copy_unexp_request(struct fi_ibv_rdm_request *request,
+ struct fi_ibv_rdm_request *unexp)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ ssize_t ret = FI_SUCCESS;
+ if (request->len && (request->len < unexp->len)) {
+ VERBS_INFO(FI_LOG_EP_DATA,
+ "RECV TRUNCATE, unexp len %d, "
+ "req->len=%d, conn %p, tag 0x%llx, "
+ "tagmask %llx\n",
+ unexp->len, request->len,
+ request->minfo.conn, request->minfo.tag,
+ request->minfo.tagmask);
+
+ util_buf_release(fi_ibv_rdm_extra_buffers_pool,
+ unexp->unexp_rbuf);
+ ret = -FI_ETRUNC;
+ return ret;
+ }
+
+ request->minfo.conn = unexp->minfo.conn;
+ request->minfo.tag = unexp->minfo.tag;
+ request->minfo.is_tagged = unexp->minfo.is_tagged;
+ request->len = unexp->len;
+ request->rest_len = unexp->rest_len;
+ request->unexp_rbuf = unexp->unexp_rbuf;
+ request->state = unexp->state;
+
+ assert((request->state.eager == FI_IBV_STATE_EAGER_RECV_WAIT4RECV) ||
+ (request->state.eager == FI_IBV_STATE_EAGER_RECV_CLAIMED));
+
+ VERBS_DBG(FI_LOG_EP_DATA, "found req: len = %d, eager_state = %s, rndv_state = %s \n",
+ unexp->len,
+ fi_ibv_rdm_req_eager_state_to_str(unexp->state.eager),
+ fi_ibv_rdm_req_rndv_state_to_str(unexp->state.rndv));
+ if (request->state.rndv != FI_IBV_STATE_RNDV_NOT_USED) {
+ assert(request->state.rndv == FI_IBV_STATE_RNDV_RECV_WAIT4RES);
+
+ request->rndv.rkey = unexp->rndv.rkey;
+ request->rndv.id = unexp->rndv.id;
+ request->rndv.remote_addr = unexp->rndv.remote_addr;
+ }
+
+ return ret;
+}
+
+static ssize_t
+fi_ibv_rdm_init_recv_request(struct fi_ibv_rdm_request *request, void *data)
+{
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
+
+ ssize_t ret = FI_SUCCESS;
struct fi_ibv_rdm_tagged_recv_start_data *p = data;
- request->tag = p->tag;
- request->tagmask = p->tagmask;
- request->context = p->context;
+ request->minfo = p->peek_data.minfo;
+ request->context = p->peek_data.context;
request->context->internal[0] = (void *)request;
request->dest_buf = p->dest_addr;
- request->conn = p->conn;
request->len = p->data_len;
+ request->comp_flags =
+ (p->peek_data.minfo.is_tagged ? FI_TAGGED : FI_MSG ) | FI_RECV |
+ (p->peek_data.flags & FI_COMPLETION);
request->state.eager = FI_IBV_STATE_EAGER_RECV_WAIT4PKT;
request->state.rndv = FI_IBV_STATE_RNDV_NOT_USED;
+ request->state.err = FI_SUCCESS;
- VERBS_DBG(FI_LOG_EP_DATA, "conn %p, tag 0x%llx, len %d\n", request->conn,
- request->tag, request->len);
-
- struct fi_verbs_rdm_tagged_request_minfo minfo = {
- .conn = p->conn,
- .tag = p->tag,
- .tagmask = p->tagmask
- };
+ VERBS_DBG(FI_LOG_EP_DATA, "conn %p, tag 0x%llx, len %d\n",
+ request->minfo.conn, request->minfo.tag, request->len);
struct dlist_entry *found_entry =
- dlist_find_first_match(&fi_ibv_rdm_tagged_recv_unexp_queue,
- fi_ibv_rdm_tagged_req_match_by_info2,
- &minfo);
+ dlist_find_first_match(&fi_ibv_rdm_unexp_queue,
+ fi_ibv_rdm_req_match_by_info3,
+ &p->peek_data);
if (found_entry) {
- struct fi_ibv_rdm_tagged_request *found_request =
- container_of(found_entry, struct fi_ibv_rdm_tagged_request,
+ struct fi_ibv_rdm_request *found_request =
+ container_of(found_entry, struct fi_ibv_rdm_request,
queue_entry);
- fi_ibv_rdm_tagged_remove_from_unexp_queue(found_request);
+ fi_ibv_rdm_remove_from_unexp_queue(found_request);
- if (request->len < found_request->len) {
- VERBS_INFO(FI_LOG_EP_DATA,
- "RECV TRUNCATE, unexp len %d, "
- "req->len=%d, conn %p, tag 0x%llx, "
- "tagmask %llx\n",
- found_request->len, request->len, request->conn,
- request->tag, request->tagmask);
- abort();
- }
+ ret = fi_ibv_rdm_copy_unexp_request(request, found_request);
- request->tag = found_request->tag;
- request->len = found_request->len;
- request->conn = found_request->conn;
- request->unexp_rbuf = found_request->unexp_rbuf;
- request->state.eager = found_request->state.eager;
- request->state.rndv = found_request->state.rndv;
-
- assert(request->state.eager ==
- FI_IBV_STATE_EAGER_RECV_WAIT4RECV);
- if (request->state.rndv != FI_IBV_STATE_RNDV_NOT_USED) {
- assert(request->state.rndv ==
- FI_IBV_STATE_RNDV_RECV_WAIT4RES);
-
- request->rndv.rkey =
- found_request->rndv.rkey;
- request->rndv.id = found_request->rndv.id;
- request->rndv.remote_addr =
- found_request->rndv.remote_addr;
- }
- VERBS_DBG(FI_LOG_EP_DATA, "found req: len = %d, eager_state = %s, "
- "rndv_state = %s \n",
- found_request->len,
- fi_ibv_rdm_tagged_req_eager_state_to_str
- (found_request->state.eager),
- fi_ibv_rdm_tagged_req_rndv_state_to_str
- (found_request->state.rndv));
+ assert((ret != FI_SUCCESS) ||
+ ((p->peek_data.flags & FI_CLAIM) &&
+ (request->state.eager ==
+ FI_IBV_STATE_EAGER_RECV_CLAIMED) &&
+ (request->context == found_request->context)) ||
+ (!(p->peek_data.flags & FI_CLAIM) &&
+ (request->state.eager ==
+ FI_IBV_STATE_EAGER_RECV_WAIT4RECV)));
- FI_IBV_RDM_TAGGED_HANDLER_LOG();
+ FI_IBV_RDM_HNDL_REQ_LOG();
- FI_IBV_RDM_TAGGED_DBG_REQUEST("to_pool: ", found_request,
- FI_LOG_DEBUG);
- util_buf_release(fi_ibv_rdm_tagged_request_pool, found_request);
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", found_request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, found_request);
- if (request->state.rndv == FI_IBV_STATE_RNDV_RECV_WAIT4RES) {
+ if (ret == FI_SUCCESS &&
+ request->state.rndv == FI_IBV_STATE_RNDV_RECV_WAIT4RES)
+ {
request->state.eager = FI_IBV_STATE_EAGER_RECV_END;
- fi_ibv_rdm_tagged_move_to_postponed_queue(request);
+ FI_IBV_RDM_HNDL_REQ_LOG();
+ fi_ibv_rdm_move_to_postponed_queue(request);
}
#if ENABLE_DEBUG
- request->conn->unexp_counter++;
+ request->minfo.conn->unexp_counter++;
#endif // ENABLE_DEBUG
} else {
#if ENABLE_DEBUG
- if (request->conn) {
- request->conn->exp_counter++;
+ if (request->minfo.conn) {
+ request->minfo.conn->exp_counter++;
}
#endif // ENABLE_DEBUG
- fi_ibv_rdm_tagged_move_to_posted_queue(request, p->ep);
+ if (ret == FI_SUCCESS) {
+ fi_ibv_rdm_move_to_posted_queue(request, p->ep);
+ }
+ }
+
+ if (ret != FI_SUCCESS) {
+ request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
+
+ fi_ibv_rdm_cntr_inc_err(p->ep->recv_cntr);
+
+ if (request->comp_flags & FI_COMPLETION) {
+ fi_ibv_rdm_move_to_errcq(p->ep->fi_rcq, request, ret);
+ }
+ ret = FI_SUCCESS;
}
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return ret;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_init_unexp_recv_request(
- struct fi_ibv_rdm_tagged_request *request, void *data)
+static ssize_t
+fi_ibv_rdm_tagged_peek_request(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
+ assert(request->state.eager == FI_IBV_STATE_EAGER_BEGIN);
+ assert(request->state.rndv == FI_IBV_STATE_RNDV_NOT_USED);
+
+ ssize_t ret = FI_SUCCESS;
+
+ struct fi_ibv_rdm_tagged_recv_start_data *p = data;
+ struct fi_ibv_rdm_tagged_peek_data *peek_data = &p->peek_data;
+ struct dlist_entry *found_entry =
+ dlist_find_first_match(&fi_ibv_rdm_unexp_queue,
+ fi_ibv_rdm_req_match_by_info2,
+ &peek_data->minfo);
+
+ request->context = peek_data->context;
+ request->comp_flags = peek_data->flags;
+
+ if (found_entry) {
+ struct fi_ibv_rdm_request *found_request =
+ container_of(found_entry, struct fi_ibv_rdm_request,
+ queue_entry);
+ assert(found_request);
+
+ ret = fi_ibv_rdm_copy_unexp_request(request, found_request);
+
+ if (ret) {
+ goto err;
+ }
+
+ if (peek_data->flags & FI_CLAIM) {
+ ret = fi_ibv_rdm_req_hndl(found_request,
+ FI_IBV_EVENT_RECV_CLAIM,
+ peek_data);
+
+ if (ret) {
+ goto err;
+ }
+ }
+
+ if (peek_data->flags & FI_DISCARD) {
+ ret = fi_ibv_rdm_req_hndl(found_request,
+ FI_IBV_EVENT_RECV_DISCARD,
+ NULL);
+
+ if (ret) {
+ goto err;
+ }
+ }
+
+ fi_ibv_rdm_cntr_inc(p->ep->recv_cntr);
+
+ if (request->comp_flags & FI_COMPLETION) {
+ request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
+ fi_ibv_rdm_move_to_cq(p->ep->fi_rcq, request);
+ } else {
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
+ }
+
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ } else {
+ ret = -FI_ENOMSG;
+ goto err;
+ }
+
+out:
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return ret;
+err:
+ request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
+
+ fi_ibv_rdm_cntr_inc_err(p->ep->recv_cntr);
+
+ if (request->comp_flags & FI_COMPLETION) {
+ fi_ibv_rdm_move_to_errcq(p->ep->fi_rcq, request, ret);
+ }
+
+ ret = FI_SUCCESS;
+ goto out;
+}
+
+static ssize_t
+fi_ibv_rdm_init_unexp_recv_request(struct fi_ibv_rdm_request *request, void *data)
+{
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
assert(request->state.eager == FI_IBV_STATE_EAGER_BEGIN);
assert(request->state.rndv == FI_IBV_STATE_RNDV_NOT_USED);
struct fi_ibv_recv_got_pkt_preprocess_data *p = data;
struct fi_ibv_rdm_buf *rbuf = p->rbuf;
+ ssize_t ret = FI_SUCCESS;
- FI_IBV_RDM_TAGGED_HANDLER_LOG();
+ FI_IBV_RDM_HNDL_REQ_LOG();
switch (p->pkt_type) {
case FI_IBV_RDM_EAGER_PKT:
- FI_IBV_RDM_TAGGED_HANDLER_LOG();
+ case FI_IBV_RDM_MSG_PKT:
+ FI_IBV_RDM_HNDL_REQ_LOG();
- request->tag = rbuf->header.tag;
- request->conn = p->conn;
+ request->minfo.conn = p->conn;
+ request->minfo.tag = rbuf->header.tag;
+ request->minfo.is_tagged = FI_IBV_RDM_EAGER_PKT ? 1 : 0;
request->len =
p->arrived_len - sizeof(struct fi_ibv_rdm_header);
+ request->comp_flags =
+ (request->minfo.is_tagged ? FI_TAGGED : FI_MSG) | FI_RECV;
assert(request->len <= p->ep->rndv_threshold);
if (request->len > 0) {
- request->unexp_rbuf = util_buf_alloc(
- fi_ibv_rdm_tagged_extra_buffers_pool);
- memcpy(request->unexp_rbuf, rbuf->payload, request->len);
+ request->unexp_rbuf =
+ util_buf_alloc(fi_ibv_rdm_extra_buffers_pool);
+ memcpy(request->unexp_rbuf, &rbuf->payload, request->len);
} else {
request->unexp_rbuf = NULL;
}
@@ -536,45 +676,44 @@ fi_ibv_rdm_tagged_init_unexp_recv_request(
request->state.eager = FI_IBV_STATE_EAGER_RECV_WAIT4RECV;
break;
case FI_IBV_RDM_RNDV_RTS_PKT:
- FI_IBV_RDM_TAGGED_HANDLER_LOG();
- assert(p->arrived_len ==
- sizeof(struct fi_ibv_rdm_tagged_rndv_header));
- struct fi_ibv_rdm_tagged_rndv_header *h =
- (struct fi_ibv_rdm_tagged_rndv_header *)rbuf;
-
- request->tag = h->base.tag;
- request->rndv.id = h->id;
+ FI_IBV_RDM_HNDL_REQ_LOG();
+ assert(p->arrived_len == sizeof(struct fi_ibv_rdm_rndv_header));
+ struct fi_ibv_rdm_rndv_header *h = (void *)&rbuf->header;
+
+ request->minfo.conn = p->conn;
+ request->minfo.tag = h->base.tag;
+ request->minfo.is_tagged = h->is_tagged;
+ request->rndv.id = (uintptr_t)h->id;
request->rndv.remote_addr = (void *)h->src_addr;
request->rndv.rkey = h->mem_key;
- request->len = h->len;
+ request->len = h->total_len;
+ request->rest_len = h->total_len;
+ request->comp_flags = (h->is_tagged ? FI_TAGGED : FI_MSG) | FI_RECV;
request->imm = p->imm_data;
- request->conn = p->conn;
request->state.eager = FI_IBV_STATE_EAGER_RECV_WAIT4RECV;
request->state.rndv = FI_IBV_STATE_RNDV_RECV_WAIT4RES;
break;
case FI_IBV_RDM_RNDV_ACK_PKT:
- FI_IBV_RDM_TAGGED_DBG_REQUEST("Unexpected RNDV ack!!!",
- request, FI_LOG_INFO);
- assert(0);
- break;
+ FI_IBV_RDM_DBG_REQUEST("Unexpected RNDV ack!!!", request,
+ FI_LOG_INFO);
default:
VERBS_INFO(FI_LOG_EP_DATA,
"Got unknown unexpected pkt: %" PRIu64 "\n",
p->pkt_type);
assert(0);
+ ret = -FI_EOTHER;
}
- fi_ibv_rdm_tagged_move_to_unexpected_queue(request);
+ fi_ibv_rdm_move_to_unexpected_queue(request);
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return ret;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_eager_recv_got_pkt(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+static ssize_t
+fi_ibv_rdm_eager_recv_got_pkt(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
struct fi_ibv_recv_got_pkt_preprocess_data *p = data;
struct fi_ibv_rdm_buf *rbuf = p->rbuf;
assert(request->state.eager == FI_IBV_STATE_EAGER_RECV_WAIT4PKT);
@@ -582,58 +721,116 @@ fi_ibv_rdm_tagged_eager_recv_got_pkt(struct fi_ibv_rdm_tagged_request *request,
switch (p->pkt_type) {
case FI_IBV_RDM_EAGER_PKT:
- assert(p->arrived_len - sizeof(rbuf->header) <= request->len);
- request->tag = rbuf->header.tag;
- request->conn = p->conn;
- request->len = p->arrived_len - sizeof(rbuf->header);
- request->exp_rbuf = rbuf->payload;
- request->imm = p->imm_data;
+ case FI_IBV_RDM_MSG_PKT:
+ {
+ const size_t data_len = p->arrived_len - sizeof(rbuf->header);
+ assert(data_len <= p->ep->rndv_threshold);
- assert(request->len <= p->ep->rndv_threshold);
+ if (request->len >= data_len) {
+ request->minfo.conn = p->conn;
+ request->minfo.tag = rbuf->header.tag;
+ request->minfo.is_tagged = FI_IBV_RDM_EAGER_PKT ? 1 : 0;
- if (request->dest_buf) {
- assert(request->exp_rbuf);
- memcpy(request->dest_buf,
- request->exp_rbuf, request->len);
+ request->len = p->arrived_len - sizeof(rbuf->header);
+ request->exp_rbuf = &rbuf->payload;
+ request->imm = p->imm_data;
+
+
+ if (request->dest_buf) {
+ assert(request->exp_rbuf);
+ memcpy(request->dest_buf,
+ request->exp_rbuf, request->len);
+ }
+
+ fi_ibv_rdm_cntr_inc(p->ep->recv_cntr);
+
+ if (request->comp_flags & FI_COMPLETION) {
+ request->state.eager =
+ FI_IBV_STATE_EAGER_READY_TO_FREE;
+ fi_ibv_rdm_move_to_cq(p->ep->fi_rcq, request);
+ } else {
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request,
+ FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool,
+ request);
+ }
+ } else {
+ VERBS_INFO(FI_LOG_EP_DATA,
+ "%s: %d RECV TRUNCATE, data_len=%d, posted_len=%d, "
+ "conn %p, tag 0x%llx, tagmask %llx\n",
+ __FUNCTION__, __LINE__, data_len,
+ request->len,
+ request->minfo.conn,
+ request->minfo.tag,
+ request->minfo.tagmask);
+
+ request->state.eager =
+ FI_IBV_STATE_EAGER_READY_TO_FREE;
+
+ fi_ibv_rdm_cntr_inc_err(p->ep->recv_cntr);
+
+ if (request->comp_flags & FI_COMPLETION) {
+ fi_ibv_rdm_move_to_errcq(p->ep->fi_rcq, request,
+ FI_ETRUNC);
+ }
}
- request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
- fi_ibv_rdm_tagged_move_to_ready_queue(request);
- FI_IBV_RDM_TAGGED_HANDLER_LOG();
+ FI_IBV_RDM_HNDL_REQ_LOG();
break;
+ }
case FI_IBV_RDM_RNDV_RTS_PKT:
- assert(p->arrived_len ==
- sizeof(struct fi_ibv_rdm_tagged_rndv_header));
+ {
+ struct fi_ibv_rdm_rndv_header *rndv_header = (void *)&rbuf->header;
+
+ assert(p->arrived_len == sizeof(*rndv_header));
- struct fi_ibv_rdm_tagged_rndv_header *rndv_header =
- (struct fi_ibv_rdm_tagged_rndv_header *)rbuf;
+ if (request->len < rndv_header->total_len) {
+ /* rndv protocol finalization requires memory
+ * deregistration, entry in errcq will be generated
+ * after acknowledgement in normal flow */
+ request->state.err = FI_ETRUNC;
+ }
- request->tag = rndv_header->base.tag;
+ request->minfo.conn = p->conn;
+ request->minfo.tag = rndv_header->base.tag;
+ request->minfo.is_tagged = rndv_header->is_tagged;
request->rndv.remote_addr = (void *)rndv_header->src_addr;
request->rndv.rkey = rndv_header->mem_key;
- request->len = rndv_header->len;
+ request->len = rndv_header->total_len;
+ request->rest_len = rndv_header->total_len;
request->imm = p->imm_data;
- request->conn = p->conn;
request->rndv.id = rndv_header->id;
- fi_ibv_rdm_tagged_move_to_postponed_queue(request);
+ fi_ibv_rdm_move_to_postponed_queue(request);
request->state.eager = FI_IBV_STATE_EAGER_RECV_END;
request->state.rndv = FI_IBV_STATE_RNDV_RECV_WAIT4RES;
- FI_IBV_RDM_TAGGED_HANDLER_LOG();
+ FI_IBV_RDM_HNDL_REQ_LOG();
break;
}
+ default:
+ VERBS_INFO(FI_LOG_EP_DATA,
+ "Got unknown unexpected pkt: %" PRIu64 "\n",
+ p->pkt_type);
+ assert(0);
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return -FI_EOTHER;
+ }
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return FI_SUCCESS;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_eager_recv_process_unexp_pkt(
- struct fi_ibv_rdm_tagged_request *request, void *data)
+static ssize_t
+fi_ibv_rdm_eager_recv_process_unexp_pkt(struct fi_ibv_rdm_request *request,
+ void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
- assert(request->state.eager == FI_IBV_STATE_EAGER_RECV_WAIT4RECV);
+ struct fi_ibv_recv_got_pkt_process_data *p = data;
+
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
+
+ assert((request->state.eager == FI_IBV_STATE_EAGER_RECV_WAIT4RECV) ||
+ (request->state.eager == FI_IBV_STATE_EAGER_RECV_CLAIMED));
assert(request->state.rndv == FI_IBV_STATE_RNDV_NOT_USED);
if (request->dest_buf && request->len != 0) {
@@ -641,130 +838,207 @@ fi_ibv_rdm_tagged_eager_recv_process_unexp_pkt(
}
if (request->unexp_rbuf) {
- util_buf_release(fi_ibv_rdm_tagged_extra_buffers_pool, request->unexp_rbuf);
+ util_buf_release(fi_ibv_rdm_extra_buffers_pool,
+ request->unexp_rbuf);
request->unexp_rbuf = NULL;
}
- fi_ibv_rdm_tagged_move_to_ready_queue(request);
- request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
+ fi_ibv_rdm_cntr_inc(p->ep->recv_cntr);
+
+ if (request->comp_flags & FI_COMPLETION) {
+ request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
+ fi_ibv_rdm_move_to_cq(p->ep->fi_rcq, request);
+ } else {
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
+ }
+
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return FI_SUCCESS;
+}
+
+static ssize_t
+fi_ibv_rdm_tagged_recv_claim(struct fi_ibv_rdm_request *request, void *data)
+{
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
+ assert(request->state.eager == FI_IBV_STATE_EAGER_RECV_WAIT4RECV);
+ assert((request->state.rndv == FI_IBV_STATE_RNDV_NOT_USED) ||
+ (request->state.rndv == FI_IBV_STATE_RNDV_RECV_WAIT4RES));
+
+ ssize_t ret = FI_SUCCESS;
+ struct fi_ibv_rdm_tagged_peek_data *peek_data = data;
+ assert(peek_data->context);
+
+ request->state.eager = FI_IBV_STATE_EAGER_RECV_CLAIMED;
+ request->context = peek_data->context;
+
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return ret;
+}
+
+static ssize_t
+fi_ibv_rdm_eager_recv_discard(struct fi_ibv_rdm_request *request, void *data)
+{
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
+ assert(request->state.eager == FI_IBV_STATE_EAGER_RECV_WAIT4RECV);
+ assert(request->state.rndv == FI_IBV_STATE_RNDV_NOT_USED);
+ assert(data == NULL);
+
+ fi_ibv_rdm_remove_from_unexp_queue(request);
+
+ if (request->unexp_rbuf) {
+ util_buf_release(fi_ibv_rdm_extra_buffers_pool,
+ request->unexp_rbuf);
+ request->unexp_rbuf = NULL;
+ }
+
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return FI_SUCCESS;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_rndv_recv_post_read(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+static ssize_t
+fi_ibv_rdm_rndv_recv_post_read(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
assert(request->state.eager == FI_IBV_STATE_EAGER_RECV_END);
assert(request->state.rndv == FI_IBV_STATE_RNDV_RECV_WAIT4RES);
struct fi_ibv_rdm_tagged_send_ready_data *p = data;
-
- fi_ibv_rdm_tagged_remove_from_postponed_queue(request);
- VERBS_DBG(FI_LOG_EP_DATA, "\t REQUEST: conn %p, tag 0x%llx, len %d, dest_buf %p, "
- "src_addr %p, rkey 0x%lx\n",
- request->conn, (long long unsigned int)request->tag,
- request->len, request->dest_buf,
- request->rndv.remote_addr,
- (long unsigned int)request->rndv.rkey);
+ const size_t offset = request->len - request->rest_len;
+ const size_t seg_cursize =
+ MIN(p->ep->rndv_seg_size, request->rest_len);
struct ibv_send_wr wr = { 0 };
struct ibv_send_wr *bad_wr = NULL;
struct ibv_sge sge;
- int ret;
- assert((request->conn->cm_role != FI_VERBS_CM_SELF) ||
+ fi_ibv_rdm_remove_from_postponed_queue(request);
+ VERBS_DBG(FI_LOG_EP_DATA,
+ "\t REQUEST: conn %p, tag 0x%llx, len %zu, rest %zu, dest_buf %p, src_addr %p, rkey 0x%lx\n",
+ request->minfo.conn, request->minfo.tag,
+ request->len, request->rest_len, request->dest_buf,
+ request->rndv.remote_addr,
+ (long unsigned int)request->rndv.rkey);
+
+ assert((request->minfo.conn->cm_role != FI_VERBS_CM_SELF) ||
(request->rndv.remote_addr != request->dest_buf));
+ /* First segment */
+ if (offset == 0) {
+ request->rndv.mr =
+ ibv_reg_mr(p->ep->domain->pd, request->dest_buf,
+ request->len,
+ IBV_ACCESS_LOCAL_WRITE |
+ IBV_ACCESS_REMOTE_WRITE);
+ if (!request->rndv.mr) {
+ VERBS_INFO_ERRNO(FI_LOG_EP_DATA, "failed ibv_reg_mr",
+ errno);
+ assert(0);
+ return -FI_ENOMEM;
+ }
+ request->post_counter = 0;
+ }
+
wr.wr_id = (uintptr_t) request;
assert(FI_IBV_RDM_CHECK_SERVICE_WR_FLAG(wr.wr_id) == 0);
wr.opcode = IBV_WR_RDMA_READ;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.send_flags = 0;
- wr.wr.rdma.remote_addr = (uintptr_t) request->rndv.remote_addr;
+ wr.wr.rdma.remote_addr = (uintptr_t)
+ ((char *)request->rndv.remote_addr + offset);
wr.wr.rdma.rkey = request->rndv.rkey;
- request->rndv.mr = ibv_reg_mr(p->ep->domain->pd,
- request->dest_buf,
- request->len, IBV_ACCESS_LOCAL_WRITE);
- assert(request->rndv.mr);
-
- sge.addr = (uintptr_t) request->dest_buf;
- sge.length = request->len;
+ sge.addr = (uintptr_t)((char *)request->dest_buf + offset);
+ sge.length = (request->state.err == FI_SUCCESS ? seg_cursize : 0);
sge.lkey = request->rndv.mr->lkey;
- FI_IBV_RDM_INC_SIG_POST_COUNTERS(request->conn, p->ep, wr.send_flags);
+ request->rest_len -= seg_cursize;
+ request->post_counter++;
+ FI_IBV_RDM_INC_SIG_POST_COUNTERS(request->minfo.conn, p->ep, wr.send_flags);
VERBS_DBG(FI_LOG_EP_DATA, "posted %d bytes, conn %p, tag 0x%llx\n",
- sge.length, request->conn, request->tag);
- ret = ibv_post_send(request->conn->qp[0], &wr, &bad_wr);
+ sge.length, request->minfo.conn, request->minfo.tag);
+ int ret = ibv_post_send(request->minfo.conn->qp[0], &wr, &bad_wr);
if (ret) {
VERBS_INFO_ERRNO(FI_LOG_EP_DATA, "ibv_post_send", errno);
assert(0);
- return FI_EP_RDM_HNDL_ERROR;
+ return -errno;
};
- request->state.eager = FI_IBV_STATE_EAGER_RECV_END;
- request->state.rndv = FI_IBV_STATE_RNDV_RECV_WAIT4LC;
+ if (request->rest_len && request->state.err == FI_SUCCESS) {
+ /* Move to postponed queue for the next iteration */
+ fi_ibv_rdm_move_to_postponed_queue(request);
+ } else {
+ request->state.eager = FI_IBV_STATE_EAGER_RECV_END;
+ request->state.rndv = FI_IBV_STATE_RNDV_RECV_WAIT4LC;
+ }
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return FI_SUCCESS;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_rndv_recv_read_lc(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+static ssize_t
+fi_ibv_rdm_rndv_recv_read_lc(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
struct fi_ibv_rdm_tagged_send_completed_data *p = data;
+ struct fi_ibv_rdm_conn *conn = request->minfo.conn;
+ struct ibv_send_wr wr = { 0 };
+ struct ibv_sge sge = { 0 };
+ struct ibv_send_wr *bad_wr = NULL;
+ struct fi_ibv_rdm_buf *sbuf = request->sbuf;
+ ssize_t ret = FI_SUCCESS;
+ const int ack_size =
+ sizeof(struct fi_ibv_rdm_header) + sizeof(request->rndv.id);
assert(request->len > (p->ep->rndv_threshold
- sizeof(struct fi_ibv_rdm_header)));
assert(request->state.eager == FI_IBV_STATE_EAGER_RECV_END);
- assert(request->state.rndv == FI_IBV_STATE_RNDV_RECV_WAIT4LC);
+ assert(request->state.rndv == FI_IBV_STATE_RNDV_RECV_WAIT4LC ||
+ request->state.rndv == FI_IBV_STATE_RNDV_RECV_WAIT4RES);
- FI_IBV_RDM_TAGGED_DEC_SEND_COUNTERS(request->conn, p->ep);
+ FI_IBV_RDM_DEC_SIG_POST_COUNTERS(conn, p->ep);
+ request->post_counter--;
- struct fi_ibv_rdm_tagged_conn *conn = request->conn;
- assert(request->sbuf);
+ if (request->rest_len || request->post_counter) {
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return ret;
+ }
- int ret = 0;
- struct ibv_send_wr wr = { 0 };
- struct ibv_sge sge = { 0 };
- struct ibv_send_wr *bad_wr = NULL;
+ assert(request->sbuf);
- struct fi_ibv_rdm_buf *sbuf = request->sbuf;
sbuf->header.tag = 0;
sbuf->header.service_tag = 0;
FI_IBV_RDM_SET_PKTTYPE(sbuf->header.service_tag,
FI_IBV_RDM_RNDV_ACK_PKT);
- memcpy(sbuf->payload, &(request->rndv.id), sizeof(request->rndv.id));
+ memcpy(&sbuf->payload, &request->rndv.id, sizeof(request->rndv.id));
wr.wr_id = ((uint64_t) (uintptr_t) (void *) request);
assert(FI_IBV_RDM_CHECK_SERVICE_WR_FLAG(wr.wr_id) == 0);
- wr.opcode = IBV_WR_RDMA_WRITE_WITH_IMM;
- wr.send_flags = IBV_SEND_INLINE;
+ wr.opcode = p->ep->eopcode;
wr.sg_list = &sge;
wr.num_sge = 1;
- wr.wr.rdma.remote_addr =
- fi_ibv_rdm_tagged_get_remote_addr(conn, request->sbuf);
+ wr.wr.rdma.remote_addr =
+ (uintptr_t) fi_ibv_rdm_get_remote_addr(conn, request->sbuf);
wr.wr.rdma.rkey = conn->remote_rbuf_rkey;
- wr.imm_data =
- fi_ibv_rdm_tagged_get_buff_service_data(request->sbuf)->seq_number;
+ wr.imm_data = 0;
sge.addr = (uintptr_t) sbuf;
- sge.length = sizeof(struct fi_ibv_rdm_header) + sizeof(request->rndv.id);
+ sge.length = ack_size + FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE;
+ wr.send_flags = (sge.length < p->ep->max_inline_rc) ? IBV_SEND_INLINE : 0;
sge.lkey = conn->s_mr->lkey;
+ sbuf->service_data.pkt_len = ack_size;
- FI_IBV_RDM_INC_SIG_POST_COUNTERS(request->conn, p->ep, wr.send_flags);
+ FI_IBV_RDM_INC_SIG_POST_COUNTERS(request->minfo.conn, p->ep, wr.send_flags);
VERBS_DBG(FI_LOG_EP_DATA,
"posted %d bytes, conn %p, tag 0x%llx, request %p\n",
- sge.length, request->conn, request->tag, request);
+ sge.length, request->minfo.conn, request->minfo.tag, request);
ret = ibv_post_send(conn->qp[0], &wr, &bad_wr);
if (ret == 0) {
assert(request->rndv.mr);
@@ -776,45 +1050,58 @@ fi_ibv_rdm_tagged_rndv_recv_read_lc(struct fi_ibv_rdm_tagged_request *request,
} else {
VERBS_INFO_ERRNO(FI_LOG_EP_DATA, "ibv_post_send", errno);
assert(0);
+ ret = -errno;
+ request->state.err = ret;
}
request->state.eager = FI_IBV_STATE_EAGER_SEND_WAIT4LC;
request->state.rndv = FI_IBV_STATE_RNDV_RECV_END;
- fi_ibv_rdm_tagged_move_to_ready_queue(request);
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ if (request->state.err == FI_SUCCESS) {
+ fi_ibv_rdm_cntr_inc(p->ep->recv_cntr);
+
+ if (request->comp_flags & FI_COMPLETION) {
+ fi_ibv_rdm_move_to_cq(p->ep->fi_rcq, request);
+ }
+ } else {
+ fi_ibv_rdm_cntr_inc_err(p->ep->recv_cntr);
+
+ if (request->comp_flags & FI_COMPLETION) {
+ fi_ibv_rdm_move_to_errcq(p->ep->fi_rcq, request,
+ request->state.err);
+ }
+ }
+
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return ret;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_rndv_recv_ack_lc(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+static ssize_t
+fi_ibv_rdm_rndv_recv_ack_lc(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
assert(request->state.eager == FI_IBV_STATE_EAGER_SEND_WAIT4LC ||
request->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE);
assert(request->state.rndv == FI_IBV_STATE_RNDV_RECV_END);
struct fi_ibv_rdm_tagged_send_completed_data *p = data;
- FI_IBV_RDM_TAGGED_DEC_SEND_COUNTERS(request->conn, p->ep);
+ FI_IBV_RDM_DEC_SIG_POST_COUNTERS(request->minfo.conn, p->ep);
if (request->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE) {
- FI_IBV_RDM_TAGGED_DBG_REQUEST("to_pool: ", request,
- FI_LOG_DEBUG);
- util_buf_release(fi_ibv_rdm_tagged_request_pool, request);
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
} else {
request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
request->state.rndv = FI_IBV_STATE_RNDV_RECV_END;
}
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return FI_SUCCESS;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_rma_init_request(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+static ssize_t
+fi_ibv_rdm_rma_init_request(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
assert(request->state.eager == FI_IBV_STATE_EAGER_BEGIN);
assert(request->state.rndv == FI_IBV_STATE_RNDV_NOT_USED);
@@ -822,8 +1109,17 @@ fi_ibv_rdm_rma_init_request(struct fi_ibv_rdm_tagged_request *request,
(struct fi_ibv_rdm_rma_start_data *)data;
request->context = p->context;
- request->conn = p->conn;
+ request->minfo.conn = p->conn;
+ request->len = p->data_len;
+ request->rest_len = p->data_len;
+ request->post_counter = 0;
+
+ request->rma.remote_addr = p->rbuf;
+ request->rma.rkey = p->rkey;
+ request->rma.lkey = p->lkey;
+ request->rma.opcode = p->op_code;
+ request->comp_flags = p->flags;
if (p->op_code == IBV_WR_RDMA_READ) {
request->dest_buf = (void*)p->lbuf;
} else {
@@ -831,25 +1127,22 @@ fi_ibv_rdm_rma_init_request(struct fi_ibv_rdm_tagged_request *request,
request->src_addr = (void*)p->lbuf;
}
- request->len = p->data_len;
-
- request->rma.remote_addr = p->rbuf;
- request->rma.rkey = p->rkey;
- request->rma.lkey = p->lkey;
- request->rma.opcode = p->op_code;
+ if (request->rmabuf && request->len >= p->ep_rdm->max_inline_rc) {
+ memcpy(&request->rmabuf->payload, request->src_addr,
+ request->len);
+ }
request->state.eager = FI_IBV_STATE_EAGER_RMA_INITIALIZED;
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ return FI_SUCCESS;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_rma_inject_request(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+static ssize_t
+fi_ibv_rdm_rma_inject_request(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
assert(request->state.eager == FI_IBV_STATE_EAGER_RMA_INJECT);
assert(request->state.rndv == FI_IBV_STATE_RNDV_NOT_USED);
@@ -857,13 +1150,12 @@ fi_ibv_rdm_rma_inject_request(struct fi_ibv_rdm_tagged_request *request,
struct ibv_sge sge = { 0 };
struct ibv_send_wr wr = { 0 };
struct ibv_send_wr *bad_wr = NULL;
- int ret = 0;
-
struct fi_ibv_rdm_rma_start_data *p = data;
- request->conn = p->conn;
+ request->minfo.conn = p->conn;
request->len = p->data_len;
- request->sbuf = NULL;
+ request->comp_flags = p->flags;
+ request->rmabuf = NULL;
wr.sg_list = &sge;
wr.num_sge = 1;
@@ -876,37 +1168,43 @@ fi_ibv_rdm_rma_inject_request(struct fi_ibv_rdm_tagged_request *request,
sge.addr = p->lbuf;
if ((request->len < p->ep_rdm->max_inline_rc) &&
- (!RMA_RESOURCES_IS_BUSY(request->conn, p->ep_rdm)))
+ (!RMA_RESOURCES_IS_BUSY(request->minfo.conn, p->ep_rdm)))
{
wr.send_flags |= IBV_SEND_INLINE;
} else if (fi_ibv_rdm_prepare_rma_request(request, p->ep_rdm)) {
- memcpy(request->rmabuf, (void*)p->lbuf, p->data_len);
- sge.addr = (uint64_t)request->rmabuf;
- sge.lkey = request->conn->rma_mr->lkey;
+ memcpy(&request->rmabuf->payload, (void*)p->lbuf, p->data_len);
+ sge.addr = (uintptr_t)&request->rmabuf->payload;
+ sge.lkey = request->minfo.conn->rma_mr->lkey;
} else {
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
- return FI_EP_RDM_HNDL_AGAIN;
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return -FI_EAGAIN;
}
- FI_IBV_RDM_INC_SIG_POST_COUNTERS(request->conn, p->ep_rdm, wr.send_flags);
- ret = ibv_post_send(request->conn->qp[0], &wr, &bad_wr);
+ FI_IBV_RDM_INC_SIG_POST_COUNTERS(request->minfo.conn, p->ep_rdm,
+ wr.send_flags);
+
+ int ret = ibv_post_send(request->minfo.conn->qp[0], &wr, &bad_wr);
request->state.eager = FI_IBV_STATE_EAGER_RMA_INJECT_WAIT4LC;
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
- return (ret == FI_EP_RDM_HNDL_SUCCESS) ?
- FI_SUCCESS : -errno;
+ return (ret == 0) ? FI_SUCCESS : -errno;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_rma_post_ready(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+static ssize_t
+fi_ibv_rdm_rma_post_ready(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
- assert(request->state.eager == FI_IBV_STATE_EAGER_RMA_INITIALIZED);
- assert(request->state.rndv == FI_IBV_STATE_RNDV_NOT_USED);
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
+ assert((request->state.eager == FI_IBV_STATE_EAGER_RMA_INITIALIZED &&
+ request->state.rndv == FI_IBV_STATE_RNDV_NOT_USED) ||
+ (request->state.eager == FI_IBV_STATE_EAGER_RMA_POSTPONED &&
+ request->state.rndv == FI_IBV_STATE_ZEROCOPY_RMA_WAIT4LC));
struct fi_ibv_rma_post_ready_data *p = data;
+ const size_t offset = request->len - request->rest_len;
+ const size_t seg_cursize =
+ MIN(p->ep_rdm->rndv_seg_size, request->rest_len);
+
struct ibv_sge sge = { 0 };
struct ibv_send_wr wr = { 0 };
struct ibv_send_wr *bad_wr = NULL;
@@ -918,285 +1216,337 @@ fi_ibv_rdm_rma_post_ready(struct fi_ibv_rdm_tagged_request *request,
wr.send_flags = 0;
wr.opcode = request->rma.opcode;
- if (request->rma.opcode == IBV_WR_RDMA_WRITE &&
- request->len < p->ep_rdm->max_inline_rc) {
- wr.send_flags |= IBV_SEND_INLINE;
+ if (request->state.eager == FI_IBV_STATE_EAGER_RMA_POSTPONED) {
+ fi_ibv_rdm_remove_from_postponed_queue(request);
+ request->state.eager = FI_IBV_STATE_EAGER_RMA_INITIALIZED;
}
/* buffered operation */
if (request->rmabuf) {
- sge.addr = (uint64_t) (request->rmabuf);
- sge.lkey = request->conn->rma_mr->lkey;
+ if (request->rma.opcode == IBV_WR_RDMA_WRITE &&
+ request->len < p->ep_rdm->max_inline_rc) {
+ wr.send_flags |= IBV_SEND_INLINE;
+ sge.addr = (uintptr_t) request->src_addr;
+ } else {
+ sge.addr = (uintptr_t) &request->rmabuf->payload;
+ sge.lkey = request->minfo.conn->rma_mr->lkey;
+ }
request->state.eager = FI_IBV_STATE_EAGER_RMA_WAIT4LC;
} else {
/* src_addr or dest_buf from an union
* for write or read properly */
- sge.addr = (uint64_t) (request->src_addr);
+ sge.addr = ((uintptr_t)request->src_addr) + offset;
sge.lkey = request->rma.lkey;
- if (request->len < p->ep_rdm->max_inline_rc) {
- request->state.eager = FI_IBV_STATE_EAGER_RMA_WAIT4LC;
- } else {
- request->state.rndv = FI_IBV_STATE_ZEROCOPY_RMA_WAIT4LC;
- }
+ request->state.rndv = FI_IBV_STATE_ZEROCOPY_RMA_WAIT4LC;
}
- sge.length = request->len;
+ sge.length = seg_cursize;
+
+ request->rest_len -= seg_cursize;
+ request->post_counter++;
+ FI_IBV_RDM_INC_SIG_POST_COUNTERS(request->minfo.conn, p->ep_rdm, wr.send_flags);
+ int ret = ibv_post_send(request->minfo.conn->qp[0], &wr, &bad_wr);
- FI_IBV_RDM_INC_SIG_POST_COUNTERS(request->conn, p->ep_rdm, wr.send_flags);
- int ret = ibv_post_send(request->conn->qp[0], &wr, &bad_wr);
+ if (request->rest_len) {
+ fi_ibv_rdm_move_to_postponed_queue(request);
+ request->state.eager = FI_IBV_STATE_EAGER_RMA_POSTPONED;
+ }
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
- return (ret == 0) ? FI_EP_RDM_HNDL_SUCCESS : FI_EP_RDM_HNDL_ERROR;
+ return (ret == 0) ? FI_SUCCESS : -errno;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_rma_inject_lc(struct fi_ibv_rdm_tagged_request *request, void *data)
+static ssize_t
+fi_ibv_rdm_rma_inject_lc(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
assert(request->state.eager == FI_IBV_STATE_EAGER_RMA_INJECT_WAIT4LC);
assert(request->state.rndv == FI_IBV_STATE_RNDV_NOT_USED);
struct fi_ibv_rdm_tagged_send_completed_data *p = data;
- FI_IBV_RDM_TAGGED_DEC_SEND_COUNTERS(request->conn, p->ep);
+ FI_IBV_RDM_DEC_SIG_POST_COUNTERS(request->minfo.conn, p->ep);
if (request->rmabuf) {
- fi_ibv_rdm_tagged_set_buffer_status(request->rmabuf,
- BUF_STATUS_FREE);
+ fi_ibv_rdm_set_buffer_status(request->rmabuf, BUF_STATUS_FREE);
} /* else inline flag was set */
- FI_IBV_RDM_TAGGED_HANDLER_LOG();
+ FI_IBV_RDM_HNDL_REQ_LOG();
- FI_IBV_RDM_TAGGED_DBG_REQUEST("to_pool: ", request,
- FI_LOG_DEBUG);
- util_buf_release(fi_ibv_rdm_tagged_request_pool, request);
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ return FI_SUCCESS;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_rma_buffered_lc(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+static ssize_t
+fi_ibv_rdm_rma_buffered_lc(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
- assert(request->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE ||
- request->state.eager == FI_IBV_STATE_EAGER_RMA_WAIT4LC);
+ assert(request->state.eager == FI_IBV_STATE_EAGER_RMA_WAIT4LC);
assert(request->state.rndv == FI_IBV_STATE_RNDV_NOT_USED);
struct fi_ibv_rdm_tagged_send_completed_data *p = data;
- FI_IBV_RDM_TAGGED_DEC_SEND_COUNTERS(request->conn, p->ep);
+ FI_IBV_RDM_DEC_SIG_POST_COUNTERS(request->minfo.conn, p->ep);
if (request->state.eager == FI_IBV_STATE_EAGER_RMA_WAIT4LC) {
if (request->rmabuf) {
if (request->rma.opcode == IBV_WR_RDMA_READ) {
- memcpy(request->dest_buf, request->rmabuf,
- request->len);
+ memcpy(request->dest_buf,
+ &request->rmabuf->payload, request->len);
}
- fi_ibv_rdm_tagged_set_buffer_status(request->rmabuf,
- BUF_STATUS_FREE);
+ fi_ibv_rdm_set_buffer_status(request->rmabuf,
+ BUF_STATUS_FREE);
+ }
+
+ if (request->rma.opcode == IBV_WR_RDMA_READ) {
+ fi_ibv_rdm_cntr_inc(p->ep->read_cntr);
+ } else if (request->rma.opcode == IBV_WR_RDMA_WRITE) {
+ fi_ibv_rdm_cntr_inc(p->ep->write_cntr);
}
- fi_ibv_rdm_tagged_move_to_ready_queue(request);
+
+ if (request->comp_flags & FI_COMPLETION) {
+ fi_ibv_rdm_move_to_cq(p->ep->fi_scq, request);
+ } else {
+ request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
+ FI_IBV_RDM_HNDL_REQ_LOG();
+ }
+ }
+
+ if (request->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE) {
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
+ } else {
request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
- } else { /* FI_IBV_STATE_EAGER_READY_TO_FREE */
- FI_IBV_RDM_TAGGED_DBG_REQUEST("to_pool: ", request,
- FI_LOG_DEBUG);
- util_buf_release(fi_ibv_rdm_tagged_request_pool, request);
}
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return FI_SUCCESS;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_rma_zerocopy_lc(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+static ssize_t
+fi_ibv_rdm_rma_zerocopy_lc(struct fi_ibv_rdm_request *request, void *data)
{
- FI_IBV_RDM_TAGGED_HANDLER_LOG_IN();
+ FI_IBV_RDM_HNDL_REQ_LOG_IN();
- assert(request->state.eager == FI_IBV_STATE_EAGER_RMA_INITIALIZED);
+ assert(request->state.eager == FI_IBV_STATE_EAGER_RMA_INITIALIZED ||
+ (request->state.eager == FI_IBV_STATE_EAGER_RMA_POSTPONED));
assert(request->state.rndv == FI_IBV_STATE_ZEROCOPY_RMA_WAIT4LC);
- VERBS_DBG(FI_LOG_EP_DATA, "conn %p, tag 0x%llx, len %d\n", request->conn,
- request->tag, request->len);
+ VERBS_DBG(FI_LOG_EP_DATA, "conn %p, tag 0x%llx, len %lu\n",
+ request->minfo.conn, request->minfo.tag, request->len);
struct fi_ibv_rdm_tagged_send_completed_data *p = data;
- FI_IBV_RDM_TAGGED_DEC_SEND_COUNTERS(request->conn, p->ep);
+ FI_IBV_RDM_DEC_SIG_POST_COUNTERS(request->minfo.conn, p->ep);
+ request->post_counter--;
+
+ if (request->rest_len == 0 && request->post_counter == 0) {
+ if (request->rma.opcode == IBV_WR_RDMA_READ) {
+ fi_ibv_rdm_cntr_inc(p->ep->read_cntr);
+ } else if (request->rma.opcode == IBV_WR_RDMA_WRITE) {
+ fi_ibv_rdm_cntr_inc(p->ep->write_cntr);
+ }
- if (request->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE) {
- assert (request->state.rndv == FI_IBV_STATE_ZEROCOPY_RMA_END);
- FI_IBV_RDM_TAGGED_DBG_REQUEST("to_pool: ", request,
- FI_LOG_DEBUG);
- util_buf_release(fi_ibv_rdm_tagged_request_pool, request);
- } else {
- fi_ibv_rdm_tagged_move_to_ready_queue(request);
- request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
- request->state.rndv = FI_IBV_STATE_ZEROCOPY_RMA_END;
+ if (request->comp_flags & FI_COMPLETION) {
+ fi_ibv_rdm_move_to_cq(p->ep->fi_scq, request);
+ request->state.eager = FI_IBV_STATE_EAGER_READY_TO_FREE;
+ request->state.rndv = FI_IBV_STATE_ZEROCOPY_RMA_END;
+ } else {
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
+ }
}
- FI_IBV_RDM_TAGGED_HANDLER_LOG_OUT();
- return FI_EP_RDM_HNDL_SUCCESS;
+ FI_IBV_RDM_HNDL_REQ_LOG_OUT();
+ return FI_SUCCESS;
}
-static enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_err_hndl(struct fi_ibv_rdm_tagged_request *request,
- void *data)
+static ssize_t
+fi_ibv_rdm_req_hndl_err(struct fi_ibv_rdm_request *request, void *data)
{
VERBS_INFO(FI_LOG_EP_DATA,
- "\t> IN\t< eager_state = %s, rndv_state = %s, len = %d\n",
- fi_ibv_rdm_tagged_req_eager_state_to_str(request->state.eager),
- fi_ibv_rdm_tagged_req_rndv_state_to_str(request->state.rndv),
+ "\t> IN\t< eager_state = %s, rndv_state = %s, len = %lu\n",
+ fi_ibv_rdm_req_eager_state_to_str(request->state.eager),
+ fi_ibv_rdm_req_rndv_state_to_str(request->state.rndv),
request->len);
assert(0);
- return FI_EP_RDM_HNDL_NOT_INIT;
+ return -FI_EOTHER;
}
-enum fi_rdm_tagged_req_hndl_ret fi_ibv_rdm_tagged_req_hndls_init(void)
+ssize_t fi_ibv_rdm_req_hndls_init(void)
{
size_t i, j, k;
for (i = 0; i < FI_IBV_STATE_EAGER_COUNT; ++i) {
for (j = 0; j < FI_IBV_STATE_RNDV_COUNT; ++j) {
for (k = 0; k < FI_IBV_EVENT_COUNT; ++k) {
- fi_ibv_rdm_tagged_hndl_arr[i][j][k] =
- fi_ibv_rdm_tagged_err_hndl;
+ fi_ibv_rdm_req_hndl_arr[i][j][k] =
+ fi_ibv_rdm_req_hndl_err;
}
}
}
// EAGER_SEND stuff
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
- [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_SEND_START] =
- fi_ibv_rdm_tagged_init_send_request;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_SEND_POSTPONED]
- [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_SEND_READY] =
- fi_ibv_rdm_tagged_eager_send_ready;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_SEND_WAIT4LC]
- [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_SEND_GOT_LC] =
- fi_ibv_rdm_tagged_eager_send_lc;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_READY_TO_FREE]
- [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_SEND_GOT_LC] =
- fi_ibv_rdm_tagged_eager_send_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_SEND_START] =
+ fi_ibv_rdm_init_send_request;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_SEND_POSTPONED]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_POST_READY] =
+ fi_ibv_rdm_eager_send_ready;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_SEND_WAIT4LC]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_eager_send_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_READY_TO_FREE]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_eager_send_lc;
// EAGER_RECV stuff
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
- [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RECV_START] =
- fi_ibv_rdm_tagged_init_recv_request;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
- [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RECV_GOT_PKT_PROCESS] =
- fi_ibv_rdm_tagged_init_unexp_recv_request;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_RECV_WAIT4PKT]
- [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RECV_GOT_PKT_PROCESS] =
- fi_ibv_rdm_tagged_eager_recv_got_pkt;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_RECV_WAIT4RECV]
- [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RECV_START] =
- fi_ibv_rdm_tagged_eager_recv_process_unexp_pkt;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RECV_START] =
+ fi_ibv_rdm_init_recv_request;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RECV_PEEK] =
+ fi_ibv_rdm_tagged_peek_request;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RECV_CLAIM] =
+ fi_ibv_rdm_tagged_recv_claim;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RECV_GOT_PKT_PROCESS] =
+ fi_ibv_rdm_init_unexp_recv_request;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RECV_WAIT4PKT]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RECV_GOT_PKT_PROCESS] =
+ fi_ibv_rdm_eager_recv_got_pkt;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RECV_WAIT4RECV]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RECV_START] =
+ fi_ibv_rdm_eager_recv_process_unexp_pkt;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RECV_WAIT4RECV]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RECV_CLAIM] =
+ fi_ibv_rdm_tagged_recv_claim;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RECV_CLAIMED]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RECV_START] =
+ fi_ibv_rdm_eager_recv_process_unexp_pkt;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RECV_WAIT4RECV]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RECV_DISCARD] =
+ fi_ibv_rdm_eager_recv_discard;
// RNDV_SEND stuff
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
- [FI_IBV_STATE_RNDV_SEND_BEGIN][FI_IBV_EVENT_SEND_START] =
- fi_ibv_rdm_tagged_init_send_request;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_SEND_POSTPONED]
- [FI_IBV_STATE_RNDV_SEND_WAIT4SEND][FI_IBV_EVENT_SEND_READY] =
- fi_ibv_rdm_tagged_rndv_rts_send_ready;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_SEND_WAIT4LC]
- [FI_IBV_STATE_RNDV_SEND_WAIT4ACK][FI_IBV_EVENT_SEND_GOT_LC] =
- fi_ibv_rdm_tagged_rndv_rts_lc;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_READY_TO_FREE]
- [FI_IBV_STATE_RNDV_SEND_END][FI_IBV_EVENT_SEND_GOT_LC] =
- fi_ibv_rdm_tagged_rndv_rts_lc;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_SEND_WAIT4LC]
- [FI_IBV_STATE_RNDV_SEND_WAIT4ACK][FI_IBV_EVENT_RECV_GOT_PKT_PROCESS]
- = fi_ibv_rdm_tagged_rndv_end;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_SEND_END]
- [FI_IBV_STATE_RNDV_SEND_WAIT4ACK][FI_IBV_EVENT_RECV_GOT_PKT_PROCESS]
- = fi_ibv_rdm_tagged_rndv_end;
-
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
+ [FI_IBV_STATE_RNDV_SEND_BEGIN][FI_IBV_EVENT_SEND_START] =
+ fi_ibv_rdm_init_send_request;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_SEND_POSTPONED]
+ [FI_IBV_STATE_RNDV_SEND_WAIT4SEND][FI_IBV_EVENT_POST_READY] =
+ fi_ibv_rdm_rndv_rts_send_ready;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_SEND_WAIT4LC]
+ [FI_IBV_STATE_RNDV_SEND_WAIT4ACK][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_rndv_rts_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_READY_TO_FREE]
+ [FI_IBV_STATE_RNDV_SEND_END][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_rndv_rts_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_SEND_WAIT4LC]
+ [FI_IBV_STATE_RNDV_SEND_END][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_rndv_rts_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_SEND_WAIT4LC]
+ [FI_IBV_STATE_RNDV_SEND_WAIT4ACK][FI_IBV_EVENT_RECV_GOT_PKT_PROCESS]
+ = fi_ibv_rdm_rndv_end;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_SEND_END]
+ [FI_IBV_STATE_RNDV_SEND_WAIT4ACK][FI_IBV_EVENT_RECV_GOT_PKT_PROCESS]
+ = fi_ibv_rdm_rndv_end;
+
// RNDV_RECV stuff
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
- [FI_IBV_STATE_RNDV_RECV_BEGIN][FI_IBV_EVENT_RECV_START] =
- fi_ibv_rdm_tagged_init_recv_request;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_RECV_END]
- [FI_IBV_STATE_RNDV_RECV_WAIT4RES][FI_IBV_EVENT_SEND_READY] =
- fi_ibv_rdm_tagged_rndv_recv_post_read;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_RECV_END]
- [FI_IBV_STATE_RNDV_RECV_WAIT4LC][FI_IBV_EVENT_SEND_GOT_LC] =
- fi_ibv_rdm_tagged_rndv_recv_read_lc;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_RECV_END]
- [FI_IBV_STATE_RNDV_RECV_WAIT4LC][FI_IBV_EVENT_SEND_READY] =
- fi_ibv_rdm_tagged_rndv_recv_read_lc;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_SEND_WAIT4LC]
- [FI_IBV_STATE_RNDV_RECV_END][FI_IBV_EVENT_SEND_GOT_LC] =
- fi_ibv_rdm_tagged_rndv_recv_ack_lc;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_READY_TO_FREE]
- [FI_IBV_STATE_RNDV_RECV_END][FI_IBV_EVENT_SEND_GOT_LC] =
- fi_ibv_rdm_tagged_rndv_recv_ack_lc;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_RECV_END]
- [FI_IBV_STATE_RNDV_RECV_WAIT4LC][FI_IBV_EVENT_SEND_GOT_LC] =
- fi_ibv_rdm_tagged_rndv_recv_read_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
+ [FI_IBV_STATE_RNDV_RECV_BEGIN][FI_IBV_EVENT_RECV_START] =
+ fi_ibv_rdm_init_recv_request;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RECV_WAIT4RECV]
+ [FI_IBV_STATE_RNDV_RECV_WAIT4RES][FI_IBV_EVENT_RECV_CLAIM] =
+ fi_ibv_rdm_tagged_recv_claim;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RECV_END]
+ [FI_IBV_STATE_RNDV_RECV_WAIT4RES][FI_IBV_EVENT_POST_READY] =
+ fi_ibv_rdm_rndv_recv_post_read;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RECV_END]
+ [FI_IBV_STATE_RNDV_RECV_WAIT4RES][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_rndv_recv_read_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RECV_END]
+ [FI_IBV_STATE_RNDV_RECV_WAIT4LC][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_rndv_recv_read_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RECV_END]
+ [FI_IBV_STATE_RNDV_RECV_WAIT4LC][FI_IBV_EVENT_POST_READY] =
+ fi_ibv_rdm_rndv_recv_read_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_SEND_WAIT4LC]
+ [FI_IBV_STATE_RNDV_RECV_END][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_rndv_recv_ack_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_READY_TO_FREE]
+ [FI_IBV_STATE_RNDV_RECV_END][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_rndv_recv_ack_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RECV_END]
+ [FI_IBV_STATE_RNDV_RECV_WAIT4LC][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_rndv_recv_read_lc;
// RMA read/write stuff
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
- [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RMA_START] =
- fi_ibv_rdm_rma_init_request;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_RMA_INJECT]
- [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RMA_START] =
- fi_ibv_rdm_rma_inject_request;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_RMA_INITIALIZED]
- [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_SEND_READY] =
- fi_ibv_rdm_rma_post_ready;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_RMA_WAIT4LC]
- [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_SEND_GOT_LC] =
- fi_ibv_rdm_rma_buffered_lc;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_RMA_INJECT_WAIT4LC]
- [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_SEND_GOT_LC] =
- fi_ibv_rdm_rma_inject_lc;
- fi_ibv_rdm_tagged_hndl_arr[FI_IBV_STATE_EAGER_RMA_INITIALIZED]
- [FI_IBV_STATE_ZEROCOPY_RMA_WAIT4LC][FI_IBV_EVENT_SEND_GOT_LC] =
- fi_ibv_rdm_rma_zerocopy_lc;
-
- return FI_EP_RDM_HNDL_SUCCESS;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_BEGIN]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RMA_START] =
+ fi_ibv_rdm_rma_init_request;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RMA_INJECT]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_RMA_START] =
+ fi_ibv_rdm_rma_inject_request;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RMA_INITIALIZED]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_POST_READY] =
+ fi_ibv_rdm_rma_post_ready;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RMA_POSTPONED]
+ [FI_IBV_STATE_ZEROCOPY_RMA_WAIT4LC][FI_IBV_EVENT_POST_READY] =
+ fi_ibv_rdm_rma_post_ready;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RMA_WAIT4LC]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_rma_buffered_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RMA_INJECT_WAIT4LC]
+ [FI_IBV_STATE_RNDV_NOT_USED][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_rma_inject_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RMA_INITIALIZED]
+ [FI_IBV_STATE_ZEROCOPY_RMA_WAIT4LC][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_rma_zerocopy_lc;
+ fi_ibv_rdm_req_hndl_arr[FI_IBV_STATE_EAGER_RMA_POSTPONED]
+ [FI_IBV_STATE_ZEROCOPY_RMA_WAIT4LC][FI_IBV_EVENT_POST_LC] =
+ fi_ibv_rdm_rma_zerocopy_lc;
+
+ return FI_SUCCESS;
}
-enum fi_rdm_tagged_req_hndl_ret fi_ibv_rdm_tagged_req_hndls_clean(void)
+ssize_t fi_ibv_rdm_req_hndls_clean(void)
{
size_t i, j, k;
for (i = 0; i < FI_IBV_STATE_EAGER_COUNT; ++i) {
for (j = 0; j < FI_IBV_STATE_RNDV_COUNT; ++j) {
for (k = 0; k < FI_IBV_EVENT_COUNT; ++k) {
- fi_ibv_rdm_tagged_hndl_arr[i][j][k] = NULL;
+ fi_ibv_rdm_req_hndl_arr[i][j][k] = NULL;
}
}
}
- return FI_EP_RDM_HNDL_SUCCESS;
+ return FI_SUCCESS;
}
-enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_req_hndl(struct fi_ibv_rdm_tagged_request * request,
- enum fi_ibv_rdm_tagged_request_event event, void *data)
+ssize_t
+fi_ibv_rdm_req_hndl(struct fi_ibv_rdm_request * request,
+ enum fi_ibv_rdm_request_event event, void *data)
{
VERBS_DBG(FI_LOG_EP_DATA, "\t%p, eager_state = %s, rndv_state = %s, event = %s\n",
- request,
- fi_ibv_rdm_tagged_req_eager_state_to_str
- (request->state.eager),
- fi_ibv_rdm_tagged_req_rndv_state_to_str(request->state.
- rndv),
- fi_ibv_rdm_tagged_req_event_to_str(event));
- assert(fi_ibv_rdm_tagged_hndl_arr[request->state.eager]
- [request->state.rndv]
- [event]);
-
- return fi_ibv_rdm_tagged_hndl_arr[request->state.eager]
- [request->state.rndv]
- [event] (request, data);
+ request,
+ fi_ibv_rdm_req_eager_state_to_str(request->state.eager),
+ fi_ibv_rdm_req_rndv_state_to_str(request->state.rndv),
+ fi_ibv_rdm_event_to_str(event));
+ assert(fi_ibv_rdm_req_hndl_arr[request->state.eager]
+ [request->state.rndv][event]);
+
+ return fi_ibv_rdm_req_hndl_arr[request->state.eager]
+ [request->state.rndv]
+ [event] (request, data);
}
-char *fi_ibv_rdm_tagged_req_eager_state_to_str(
- enum fi_ibv_rdm_tagged_request_eager_state state)
+char *
+fi_ibv_rdm_req_eager_state_to_str(enum fi_ibv_rdm_request_eager_state state)
{
switch (state) {
case FI_IBV_STATE_EAGER_BEGIN:
@@ -1215,6 +1565,8 @@ char *fi_ibv_rdm_tagged_req_eager_state_to_str(
return "STATE_EAGER_RECV_WAIT4PKT";
case FI_IBV_STATE_EAGER_RECV_WAIT4RECV:
return "STATE_EAGER_RECV_WAIT4RECV";
+ case FI_IBV_STATE_EAGER_RECV_CLAIMED:
+ return "FI_IBV_STATE_EAGER_RECV_CLAIMED";
case FI_IBV_STATE_EAGER_RECV_END:
return "STATE_EAGER_RECV_END";
@@ -1222,6 +1574,8 @@ char *fi_ibv_rdm_tagged_req_eager_state_to_str(
return "FI_IBV_STATE_EAGER_RMA_INJECT";
case FI_IBV_STATE_EAGER_RMA_INITIALIZED:
return "FI_IBV_STATE_EAGER_RMA_INITIALIZED";
+ case FI_IBV_STATE_EAGER_RMA_POSTPONED:
+ return "FI_IBV_STATE_EAGER_RMA_POSTPONED";
case FI_IBV_STATE_EAGER_RMA_WAIT4LC:
return "FI_IBV_STATE_EAGER_RMA_WAIT4LC";
case FI_IBV_STATE_EAGER_RMA_INJECT_WAIT4LC:
@@ -1239,8 +1593,7 @@ char *fi_ibv_rdm_tagged_req_eager_state_to_str(
}
}
-char *fi_ibv_rdm_tagged_req_rndv_state_to_str(
- enum fi_ibv_rdm_tagged_request_rndv_state state)
+char *fi_ibv_rdm_req_rndv_state_to_str(enum fi_ibv_rdm_request_rndv_state state)
{
switch (state) {
case FI_IBV_STATE_RNDV_NOT_USED:
@@ -1277,20 +1630,15 @@ char *fi_ibv_rdm_tagged_req_rndv_state_to_str(
}
}
-char *fi_ibv_rdm_tagged_req_event_to_str(
- enum fi_ibv_rdm_tagged_request_event event)
+char *fi_ibv_rdm_event_to_str(enum fi_ibv_rdm_request_event event)
{
switch (event) {
case FI_IBV_EVENT_SEND_START:
return "EVENT_SEND_START";
- case FI_IBV_EVENT_SEND_READY:
- return "EVENT_SEND_READY";
- case FI_IBV_EVENT_SEND_COMPLETED:
- return "EVENT_SEND_COMPLETED";
- case FI_IBV_EVENT_SEND_GOT_CTS:
- return "EVENT_SEND_GOT_CTS";
- case FI_IBV_EVENT_SEND_GOT_LC:
- return "EVENT_SEND_GOT_LC";
+ case FI_IBV_EVENT_POST_READY:
+ return "FI_IBV_EVENT_POST_READY";
+ case FI_IBV_EVENT_POST_LC:
+ return "FI_IBV_EVENT_POST_LC";
case FI_IBV_EVENT_RECV_START:
return "EVENT_RECV_START";
@@ -1300,6 +1648,12 @@ char *fi_ibv_rdm_tagged_req_event_to_str(
return "EVENT_RECV_GOT_PKT_PROCESS";
case FI_IBV_EVENT_RECV_GOT_ACK:
return "EVENT_RECV_GOT_ACK";
+ case FI_IBV_EVENT_RECV_PEEK:
+ return "FI_IBV_EVENT_RECV_PEEK";
+ case FI_IBV_EVENT_RECV_CLAIM:
+ return "FI_IBV_EVENT_RECV_CLAIM";
+ case FI_IBV_EVENT_RECV_DISCARD:
+ return "FI_IBV_EVENT_RECV_DISCARD";
case FI_IBV_EVENT_RMA_START:
return "FI_IBV_EVENT_RMA_START";
diff --git a/prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.h b/prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.h
index da499d4..ddaa682 100644
--- a/prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.h
+++ b/prov/verbs/src/ep_rdm/verbs_tagged_ep_rdm_states.h
@@ -36,9 +36,10 @@
#include <stddef.h>
#include <stdint.h>
-struct fi_ibv_rdm_tagged_request;
+struct fi_ibv_rdm_request;
+struct fi_verbs_rdm_request_minfo;
-enum fi_ibv_rdm_tagged_request_eager_state {
+enum fi_ibv_rdm_request_eager_state {
FI_IBV_STATE_EAGER_BEGIN = 0, // must be 0
FI_IBV_STATE_EAGER_SEND_POSTPONED,
@@ -48,10 +49,12 @@ enum fi_ibv_rdm_tagged_request_eager_state {
FI_IBV_STATE_EAGER_RECV_BEGIN,
FI_IBV_STATE_EAGER_RECV_WAIT4PKT,
FI_IBV_STATE_EAGER_RECV_WAIT4RECV,
+ FI_IBV_STATE_EAGER_RECV_CLAIMED,
FI_IBV_STATE_EAGER_RECV_END,
FI_IBV_STATE_EAGER_RMA_INJECT,
FI_IBV_STATE_EAGER_RMA_INITIALIZED,
+ FI_IBV_STATE_EAGER_RMA_POSTPONED,
FI_IBV_STATE_EAGER_RMA_WAIT4LC,
FI_IBV_STATE_EAGER_RMA_INJECT_WAIT4LC,
FI_IBV_STATE_EAGER_RMA_END,
@@ -61,10 +64,10 @@ enum fi_ibv_rdm_tagged_request_eager_state {
FI_IBV_STATE_EAGER_COUNT // must be last
};
-char *fi_ibv_rdm_tagged_req_eager_state_to_str(
- enum fi_ibv_rdm_tagged_request_eager_state state);
+char *
+fi_ibv_rdm_req_eager_state_to_str(enum fi_ibv_rdm_request_eager_state state);
-enum fi_ibv_rdm_tagged_request_rndv_state {
+enum fi_ibv_rdm_request_rndv_state {
FI_IBV_STATE_RNDV_NOT_USED = 0, // must be 0
FI_IBV_STATE_RNDV_SEND_BEGIN,
// FI_IBV_STATE_RNDV_SEND_WAIT4CTS, // not implemented yet
@@ -84,28 +87,28 @@ enum fi_ibv_rdm_tagged_request_rndv_state {
FI_IBV_STATE_RNDV_COUNT // must be last
};
-char *fi_ibv_rdm_tagged_req_rndv_state_to_str(
- enum fi_ibv_rdm_tagged_request_rndv_state state);
+char *
+fi_ibv_rdm_req_rndv_state_to_str(enum fi_ibv_rdm_request_rndv_state state);
-enum fi_ibv_rdm_tagged_request_event {
+enum fi_ibv_rdm_request_event {
FI_IBV_EVENT_SEND_START = 0,
- FI_IBV_EVENT_SEND_READY,
- FI_IBV_EVENT_SEND_COMPLETED,
- FI_IBV_EVENT_SEND_GOT_CTS,
- FI_IBV_EVENT_SEND_GOT_LC,
+ FI_IBV_EVENT_POST_READY,
+ FI_IBV_EVENT_POST_LC,
FI_IBV_EVENT_RECV_START,
FI_IBV_EVENT_RECV_GOT_PKT_PREPROCESS,
FI_IBV_EVENT_RECV_GOT_PKT_PROCESS,
FI_IBV_EVENT_RECV_GOT_ACK,
+ FI_IBV_EVENT_RECV_PEEK,
+ FI_IBV_EVENT_RECV_CLAIM,
+ FI_IBV_EVENT_RECV_DISCARD,
FI_IBV_EVENT_RMA_START,
FI_IBV_EVENT_COUNT // must be last
};
-char *fi_ibv_rdm_tagged_req_event_to_str(
- enum fi_ibv_rdm_tagged_request_event event);
+char *fi_ibv_rdm_event_to_str(enum fi_ibv_rdm_request_event event);
// Send service data types
@@ -117,10 +120,11 @@ enum ibv_rdm_send_type
IBV_RDM_SEND_TYPE_VEC
};
-struct fi_ibv_rdm_tagged_send_start_data {
+struct fi_ibv_rdm_send_start_data {
struct fi_ibv_rdm_ep *ep_rdm;
- struct fi_ibv_rdm_tagged_conn *conn;
+ struct fi_ibv_rdm_conn *conn;
void *context;
+ uint64_t flags;
size_t tag;
size_t data_len;
union {
@@ -128,6 +132,7 @@ struct fi_ibv_rdm_tagged_send_start_data {
struct iovec* iovec_arr;
} buf;
int iov_count;
+ int is_tagged;
unsigned int imm;
enum ibv_rdm_send_type stype;
};
@@ -143,19 +148,17 @@ struct fi_ibv_rdm_tagged_send_completed_data {
// Recv service data types
struct fi_ibv_rdm_tagged_recv_start_data {
- size_t tag;
- size_t tagmask;
+ struct fi_ibv_rdm_tagged_peek_data peek_data;
struct fi_context *context;
- void *dest_addr;
- struct fi_ibv_rdm_tagged_conn *conn;
struct fi_ibv_rdm_ep *ep;
+ void *dest_addr;
size_t data_len;
};
struct fi_ibv_recv_got_pkt_preprocess_data {
- struct fi_ibv_rdm_tagged_conn *conn;
+ struct fi_ibv_rdm_conn *conn;
struct fi_ibv_rdm_ep *ep;
- void *rbuf;
+ struct fi_ibv_rdm_buf *rbuf;
size_t arrived_len;
uint64_t pkt_type;
int imm_data;
@@ -169,9 +172,10 @@ struct fi_ibv_recv_got_pkt_process_data {
struct fi_ibv_rdm_rma_start_data {
struct fi_ibv_rdm_ep *ep_rdm;
- struct fi_ibv_rdm_tagged_conn *conn;
+ struct fi_ibv_rdm_conn *conn;
void *context;
- size_t data_len;
+ uint64_t flags;
+ uint64_t data_len;
uint64_t rbuf;
uintptr_t lbuf;
uint32_t rkey;
@@ -183,24 +187,12 @@ struct fi_ibv_rma_post_ready_data {
struct fi_ibv_rdm_ep *ep_rdm;
};
-// Return codes
-/* TODO: to replace with native OFI codes */
-
-enum fi_rdm_tagged_req_hndl_ret {
- FI_EP_RDM_HNDL_SUCCESS = 0,
- FI_EP_RDM_HNDL_DELETED_REQUEST = 1,
- FI_EP_RDM_HNDL_ERROR = 2,
- FI_EP_RDM_HNDL_AGAIN = 3,
- FI_EP_RDM_HNDL_NOT_INIT = (int)-1
-};
-
// Interfaces
-enum fi_rdm_tagged_req_hndl_ret fi_ibv_rdm_tagged_req_hndls_init();
-enum fi_rdm_tagged_req_hndl_ret fi_ibv_rdm_tagged_req_hndls_clean();
-enum fi_rdm_tagged_req_hndl_ret
-fi_ibv_rdm_tagged_req_hndl(struct fi_ibv_rdm_tagged_request *request,
- enum fi_ibv_rdm_tagged_request_event event,
- void *data);
+ssize_t fi_ibv_rdm_req_hndls_init();
+ssize_t fi_ibv_rdm_req_hndls_clean();
+ssize_t fi_ibv_rdm_req_hndl(struct fi_ibv_rdm_request *request,
+ enum fi_ibv_rdm_request_event event,
+ void *data);
#endif /* _VERBS_TAGGED_EP_RDM_STATES_H */
diff --git a/prov/verbs/src/ep_rdm/verbs_utils.c b/prov/verbs/src/ep_rdm/verbs_utils.c
index 07c1d41..e9fb5b6 100644
--- a/prov/verbs/src/ep_rdm/verbs_utils.c
+++ b/prov/verbs/src/ep_rdm/verbs_utils.c
@@ -1,4 +1,5 @@
/*
+ * Copyright (c) 2016, Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2013-2015 Intel Corporation, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -31,80 +32,141 @@
*/
#include <ifaddrs.h>
+#include <net/if.h>
+
#include <rdma/fi_errno.h>
#include "../fi_verbs.h"
#include "verbs_utils.h"
#include "verbs_rdm.h"
+#include "verbs_queuing.h"
+
+extern struct util_buf_pool *fi_ibv_rdm_request_pool;
+extern struct util_buf_pool *fi_ibv_rdm_extra_buffers_pool;
+extern struct util_buf_pool *fi_ibv_rdm_postponed_pool;
+
+size_t rdm_buffer_size(size_t buf_send_size)
+{
+ size_t size = buf_send_size + FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE +
+ sizeof(struct fi_ibv_rdm_header) + FI_IBV_RDM_BUF_ALIGNMENT;
+ size -= (size % FI_IBV_RDM_BUF_ALIGNMENT);
+ return size;
+}
-int fi_ibv_rdm_tagged_req_match(struct dlist_entry *item, const void *other)
+int fi_ibv_rdm_req_match(struct dlist_entry *item, const void *other)
{
- const struct fi_ibv_rdm_tagged_request *req = other;
+ const struct fi_ibv_rdm_request *req = other;
return (item == &req->queue_entry);
}
-int fi_ibv_rdm_tagged_req_match_by_info(struct dlist_entry *item,
- const void *info)
+int fi_ibv_rdm_req_match_by_info(struct dlist_entry *item, const void *info)
{
- struct fi_ibv_rdm_tagged_request *request =
- container_of(item, struct fi_ibv_rdm_tagged_request, queue_entry);
+ struct fi_ibv_rdm_request *request =
+ container_of(item, struct fi_ibv_rdm_request, queue_entry);
- const struct fi_verbs_rdm_tagged_request_minfo *minfo = info;
+ const struct fi_ibv_rdm_minfo *minfo = info;
- return (((request->conn == NULL) || (request->conn == minfo->conn)) &&
- ((request->tag & request->tagmask) ==
- (minfo->tag & request->tagmask)));
+ return (
+ ((request->minfo.conn == NULL) ||
+ (request->minfo.conn == minfo->conn))
+ &&
+ (request->minfo.is_tagged ?
+ ((request->minfo.tag & request->minfo.tagmask) ==
+ (minfo->tag & request->minfo.tagmask)) : 1)
+ );
}
/*
- * The same as fi_ibv_rdm_tagged_req_match_by_info but conn and tagmask fields
+ * The same as fi_ibv_rdm_req_match_by_info but conn and tagmask fields
* are used for matching instead of request's ones
*/
-int fi_ibv_rdm_tagged_req_match_by_info2(struct dlist_entry *item,
- const void *info)
+int fi_ibv_rdm_req_match_by_info2(struct dlist_entry *item, const void *info)
+{
+ struct fi_ibv_rdm_request *request =
+ container_of(item, struct fi_ibv_rdm_request, queue_entry);
+
+ const struct fi_ibv_rdm_minfo *minfo = info;
+
+ return (
+ ((minfo->conn == NULL) ||
+ (request->minfo.conn == minfo->conn))
+ &&
+ (minfo->is_tagged ?
+ ((request->minfo.tag & minfo->tagmask) ==
+ (minfo->tag & minfo->tagmask)) : 1)
+ );
+}
+
+/*
+ * The same as fi_ibv_rdm_tagged_req_match_by_info2 but context field is added
+ * to compare
+ */
+int fi_ibv_rdm_req_match_by_info3(struct dlist_entry *item, const void *info)
{
- struct fi_ibv_rdm_tagged_request *request =
- container_of(item, struct fi_ibv_rdm_tagged_request, queue_entry);
+ struct fi_ibv_rdm_request *request =
+ container_of(item, struct fi_ibv_rdm_request, queue_entry);
- const struct fi_verbs_rdm_tagged_request_minfo *minfo = info;
+ const struct fi_ibv_rdm_tagged_peek_data *peek_data = info;
+ const void *context = (peek_data->flags & FI_CLAIM) ?
+ peek_data->context : NULL;
- return (((minfo->conn == NULL) || (request->conn == minfo->conn)) &&
- ((request->tag & minfo->tagmask) ==
- (minfo->tag & minfo->tagmask)));
+ return ((request->context == context) &&
+ fi_ibv_rdm_req_match_by_info2(item, &peek_data->minfo));
}
-int fi_ibv_rdm_tagged_send_postponed_process(struct dlist_entry *postponed_item,
- const void *arg)
+int fi_ibv_rdm_postponed_process(struct dlist_entry *postponed_item,
+ const void *arg)
{
const struct fi_ibv_rdm_tagged_send_ready_data *send_data = arg;
- struct fi_ibv_rdm_tagged_postponed_entry *postponed_entry =
+ struct fi_ibv_rdm_postponed_entry *postponed_entry =
container_of(postponed_item,
- struct fi_ibv_rdm_tagged_postponed_entry, queue_entry);
+ struct fi_ibv_rdm_postponed_entry, queue_entry);
int ret = 0;
if (!dlist_empty(&postponed_entry->conn->postponed_requests_head)) {
struct dlist_entry *req_entry =
postponed_entry->conn->postponed_requests_head.next;
- struct fi_ibv_rdm_tagged_request *request =
- container_of(req_entry, struct fi_ibv_rdm_tagged_request,
- queue_entry);
+ struct fi_ibv_rdm_request *request =
+ container_of(req_entry, struct fi_ibv_rdm_request,
+ queue_entry);
- int res = fi_ibv_rdm_tagged_prepare_send_request(request,
+ int res = 0;
+ if ((request->state.eager < FI_IBV_STATE_EAGER_RMA_INJECT) &&
+ (request->sbuf == NULL))
+ {
+ res = fi_ibv_rdm_tagged_prepare_send_request(request,
send_data->ep);
+ } else {
+ /*
+ * This case is possible only for segmented RNDV msg or
+ * RMA operation (> 1GB), connection must be already
+ * established
+ */
+ assert(request->state.rndv != FI_IBV_STATE_RNDV_NOT_USED);
+ assert(fi_ibv_rdm_check_connection(request->minfo.conn,
+ send_data->ep));
+ if (request->state.eager <= FI_IBV_STATE_EAGER_RECV_END) {
+ res = !TSEND_RESOURCES_IS_BUSY(request->minfo.conn,
+ send_data->ep);
+ } else {
+ res = !RMA_RESOURCES_IS_BUSY(request->minfo.conn,
+ send_data->ep);
+ }
+ }
+
if (res) {
- fi_ibv_rdm_tagged_req_hndl(request,
- FI_IBV_EVENT_SEND_READY,
- (void *) send_data);
+ fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_POST_READY,
+ (void *) send_data);
ret++;
}
}
return ret;
}
-void fi_ibv_rdm_conn_init_cm_role(struct fi_ibv_rdm_tagged_conn *conn,
+void fi_ibv_rdm_conn_init_cm_role(struct fi_ibv_rdm_conn *conn,
struct fi_ibv_rdm_ep *ep)
{
- const int addr_cmp = memcmp(&conn->addr, &ep->cm.my_addr,
+ const int addr_cmp = memcmp(&conn->addr, &ep->my_addr,
FI_IBV_RDM_DFLT_ADDRLEN);
if (addr_cmp < 0) {
@@ -116,46 +178,53 @@ void fi_ibv_rdm_conn_init_cm_role(struct fi_ibv_rdm_tagged_conn *conn,
}
}
-/* find the IPoIB address of the device opened in the fi_domain call. The name
- * of this device is _domain->verbs->device->name. The logic of the function is:
- * iterate through all the available network interfaces, find those having "ib"
- * in the name, then try to test the IB device that correspond to each address.
- * If the name is the desired one then we're done.
+/* Find the IPoIB address of the device opened in the fi_ibv_domain call.
+ * The logic of the function is: iterate through all the available network
+ * interfaces, find those having "ib" (or iface param value if it's defined) in
+ * the name. If the name is the desired one then we're done. If user defines
+ * wrong interface, following rdma_bind_addr call will fail with corresponding
+ * propagation of error code.
*/
-int fi_ibv_rdm_tagged_find_ipoib_addr(const struct sockaddr_in *addr,
- struct fi_ibv_rdm_cm* cm)
+int fi_ibv_rdm_find_ipoib_addr(const struct sockaddr_in *addr,
+ struct sockaddr_in *ipoib_addr)
{
- struct ifaddrs *addrs, *tmp;
- struct sockaddr_in lh;
+ struct ifaddrs *addrs = NULL;
+ struct ifaddrs *tmp = NULL;
int found = 0;
- inet_pton(AF_INET, "127.0.0.1", &lh.sin_addr);
+ char iface[IFNAMSIZ];
+ char *iface_tmp = "ib";
+ size_t iface_len = 2;
+
+ if (!addr || !addr->sin_addr.s_addr) {
+ return 1;
+ }
+
+ if (fi_param_get_str(&fi_ibv_prov, "iface", &iface_tmp) == FI_SUCCESS) {
+ iface_len = strlen(iface_tmp);
+ if (iface_len > IFNAMSIZ) {
+ VERBS_INFO(FI_LOG_EP_CTRL,
+ "Too long iface name: %s, max: %d\n",
+ iface_tmp, IFNAMSIZ);
+ return 1;
+ }
+ }
+
+ strncpy(iface, iface_tmp, iface_len);
+
+ if (getifaddrs(&addrs)) {
+ return 1;
+ }
- getifaddrs(&addrs);
tmp = addrs;
while (tmp) {
if (tmp->ifa_addr && tmp->ifa_addr->sa_family == AF_INET) {
- struct sockaddr_in *paddr =
- (struct sockaddr_in *) tmp->ifa_addr;
- if (!strncmp(tmp->ifa_name, "ib", 2)) {
- int ret = 0;
-
- if (addr && addr->sin_addr.s_addr) {
- ret = !memcmp(&addr->sin_addr,
- &paddr->sin_addr,
- sizeof(addr->sin_addr)) ||
- !memcmp(&addr->sin_addr,
- &lh.sin_addr,
- sizeof(addr->sin_addr))
- ? 1 : 0;
- }
-
- if (ret == 1) {
- memcpy(&(cm->my_addr), paddr,
- sizeof(cm->my_addr));
- found = 1;
- break;
- }
+ found = !strncmp(tmp->ifa_name, iface, iface_len);
+ if (found) {
+ memcpy(ipoib_addr, tmp->ifa_addr,
+ sizeof(*ipoib_addr));
+ ipoib_addr->sin_port = addr->sin_port;
+ break;
}
}
@@ -164,8 +233,127 @@ int fi_ibv_rdm_tagged_find_ipoib_addr(const struct sockaddr_in *addr,
freeifaddrs(addrs);
- if (found) {
- assert(cm->my_addr.sin_family == AF_INET);
- }
return !found;
}
+
+void fi_ibv_rdm_clean_queues(struct fi_ibv_rdm_ep* ep)
+{
+ struct fi_ibv_rdm_request *request;
+
+ while ((request = fi_ibv_rdm_take_first_from_unexp_queue())) {
+ if (request->unexp_rbuf) {
+ util_buf_release(fi_ibv_rdm_extra_buffers_pool,
+ request->unexp_rbuf);
+ }
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
+ }
+
+ while ((request = fi_ibv_rdm_take_first_from_posted_queue())) {
+ if (request->iov_count > 0) {
+ util_buf_release(fi_ibv_rdm_extra_buffers_pool,
+ request->unexp_rbuf);
+ }
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
+ }
+
+ while ((request = fi_ibv_rdm_take_first_from_postponed_queue())) {
+ if (request->iov_count > 0) {
+ util_buf_release(fi_ibv_rdm_extra_buffers_pool,
+ request->unexp_rbuf);
+ }
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
+ }
+
+ while ((request = fi_ibv_rdm_take_first_from_cq(ep->fi_scq))) {
+ if (request->iov_count > 0) {
+ util_buf_release(fi_ibv_rdm_extra_buffers_pool,
+ request->unexp_rbuf);
+ }
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
+ }
+
+ while ((request = fi_ibv_rdm_take_first_from_cq(ep->fi_rcq))) {
+ if (request->iov_count > 0) {
+ util_buf_release(fi_ibv_rdm_extra_buffers_pool,
+ request->unexp_rbuf);
+ }
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
+ }
+
+ while ((request = fi_ibv_rdm_take_first_from_errcq(ep->fi_scq))) {
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
+ }
+
+ while ((request = fi_ibv_rdm_take_first_from_errcq(ep->fi_rcq))) {
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
+ }
+}
+
+ssize_t
+fi_ibv_rdm_send_common(struct fi_ibv_rdm_send_start_data* sdata)
+{
+ struct fi_ibv_rdm_request *request =
+ util_buf_alloc(fi_ibv_rdm_request_pool);
+ FI_IBV_RDM_DBG_REQUEST("get_from_pool: ", request, FI_LOG_DEBUG);
+
+ /* Initial state */
+ request->state.eager = FI_IBV_STATE_EAGER_BEGIN;
+ request->state.rndv = FI_IBV_STATE_RNDV_NOT_USED;
+ request->state.err = FI_SUCCESS;
+
+ const int in_order = (sdata->conn->postponed_entry) ? 0 : 1;
+ int ret = fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_SEND_START, sdata);
+
+ if (!ret && in_order &&
+ fi_ibv_rdm_tagged_prepare_send_request(request, sdata->ep_rdm))
+ {
+ struct fi_ibv_rdm_tagged_send_ready_data req_data =
+ { .ep = sdata->ep_rdm };
+ ret = fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_POST_READY,
+ &req_data);
+ }
+
+ return ret;
+}
+
+ssize_t
+rdm_trecv_second_event(struct fi_ibv_rdm_request *request,
+ struct fi_ibv_rdm_ep *ep)
+{
+ ssize_t ret = FI_SUCCESS;
+
+ switch (request->state.rndv)
+ {
+ case FI_IBV_STATE_RNDV_NOT_USED:
+ if (request->state.eager != FI_IBV_STATE_EAGER_RECV_WAIT4PKT) {
+ struct fi_ibv_recv_got_pkt_process_data data = {
+ .ep = ep
+ };
+ ret = fi_ibv_rdm_req_hndl(request,
+ FI_IBV_EVENT_RECV_START,
+ &data);
+ }
+ break;
+ case FI_IBV_STATE_RNDV_RECV_WAIT4RES:
+ if (fi_ibv_rdm_tagged_prepare_send_request(request, ep)) {
+ struct fi_ibv_rdm_tagged_send_ready_data data = {
+ .ep = ep
+ };
+ ret = fi_ibv_rdm_req_hndl(request,
+ FI_IBV_EVENT_POST_READY,
+ &data);
+ }
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
diff --git a/prov/verbs/src/ep_rdm/verbs_utils.h b/prov/verbs/src/ep_rdm/verbs_utils.h
index deee4dd..9637a1b 100644
--- a/prov/verbs/src/ep_rdm/verbs_utils.h
+++ b/prov/verbs/src/ep_rdm/verbs_utils.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013-2015 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -42,7 +43,6 @@
#include <infiniband/verbs.h>
-#include <rdma/fi_log.h>
#include "../fi_verbs.h"
#if (defined(__ICC) || defined(__INTEL_COMPILER) || \
@@ -64,21 +64,27 @@ struct fi_ibv_msg_ep;
#define FI_IBV_RDM_CM_THREAD_TIMEOUT (100)
#define FI_IBV_RDM_MEM_ALIGNMENT (64)
-#define FI_IBV_RDM_BUF_ALIGNMENT (4096)
+#define FI_IBV_RDM_BUF_ALIGNMENT (4096) /* TODO: Page or MTU size */
#define FI_IBV_RDM_TAGGED_DFLT_BUFFER_NUM (8)
+#define FI_IBV_RDM_DFLT_CQREAD_BUNCH_SIZE (FI_IBV_RDM_TAGGED_DFLT_BUFFER_NUM)
#define FI_IBV_RDM_DFLT_BUFFER_SIZE \
- ((8 * 1024 + FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE) + \
- (8 * 1024 + FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE) % \
- FI_IBV_RDM_BUF_ALIGNMENT)
+ (3 * FI_IBV_RDM_BUF_ALIGNMENT)
#define FI_IBV_RDM_DFLT_BUFFERED_SSIZE \
(FI_IBV_RDM_DFLT_BUFFER_SIZE - \
FI_IBV_RDM_BUFF_SERVICE_DATA_SIZE - \
sizeof(struct fi_ibv_rdm_header))
-#define FI_IBV_RDM_TAGGED_DFLT_RQ_SIZE (1000)
+/*
+ * calculates internal buffer size from user defined buffered send size in
+ * consideration of wired protocols, alignment, etc
+ */
+size_t rdm_buffer_size(size_t buf_send_size);
+
+/* 1GB is RC_QP limitation */
+#define FI_IBV_RDM_SEG_MAXSIZE (1024*1024*1024)
/* TODO: CQs depths increased from 100 to 1000 to prevent
* "Work Request Flushed Error" in stress tests like alltoall.
@@ -88,6 +94,14 @@ struct fi_ibv_msg_ep;
#define FI_IBV_RDM_CM_RESOLVEADDR_TIMEOUT (30000)
+#if ENABLE_DEBUG
+#define FI_IBV_RDM_CHECK_RECV_WC(wc) \
+ ((wc->status == IBV_WC_SUCCESS) && \
+ (wc->opcode == IBV_WC_RECV_RDMA_WITH_IMM || wc->opcode == IBV_WC_RECV))
+#else
+#define FI_IBV_RDM_CHECK_RECV_WC(wc) (wc->status == IBV_WC_SUCCESS)
+#endif /* ENABLE_DEBUG */
+
/* TODO: Holy macro batman, use verbs calls */
#define FI_IBV_DBG_OPCODE(wc_opcode, str) \
VERBS_DBG(FI_LOG_CQ, "CQ COMPL: "str" -> %s\n", \
@@ -103,59 +117,74 @@ struct fi_ibv_msg_ep;
#if ENABLE_DEBUG
-#define FI_IBV_RDM_TAGGED_DBG_REQUEST(prefix, request, level) \
-do { \
- const size_t max_str_len = 1024; \
- char str[max_str_len]; \
- snprintf(str, max_str_len, \
- "%s request: %p, eager_state: %s, rndv_state: %s, tag: 0x%lx, len: %lu, context: %p, connection: %p\n", \
- prefix, \
- request, \
- fi_ibv_rdm_tagged_req_eager_state_to_str(request->state.eager), \
- fi_ibv_rdm_tagged_req_rndv_state_to_str(request->state.rndv), \
- request->tag, \
- request->len, \
- request->context, \
- request->conn); \
- \
- switch (level) \
- { \
- case FI_LOG_WARN: \
- case FI_LOG_TRACE: \
- case FI_LOG_INFO: \
- VERBS_INFO(FI_LOG_EP_DATA, "%s", str); \
- break; \
- case FI_LOG_DEBUG: \
- default: \
- VERBS_DBG(FI_LOG_EP_DATA, "%s", str); \
- break; \
- } \
+#define FI_IBV_RDM_DBG_REQUEST(prefix, request, level) \
+do { \
+ const size_t max_str_len = 1024; \
+ char str[max_str_len]; \
+ snprintf(str, max_str_len, \
+ "%s request: %p, eager_state: %s, rndv_state: %s," \
+ " err_state: %ld, tag: 0x%lx, len: %lu, rest: %lu," \
+ "context: %p, connection: %p\n", \
+ prefix, \
+ request, \
+ fi_ibv_rdm_req_eager_state_to_str(request->state.eager), \
+ fi_ibv_rdm_req_rndv_state_to_str(request->state.rndv), \
+ request->state.err, \
+ request->minfo.tag, \
+ request->len, \
+ request->rest_len, \
+ request->context, \
+ request->minfo.conn); \
+ \
+ switch (level) \
+ { \
+ case FI_LOG_WARN: \
+ case FI_LOG_TRACE: \
+ case FI_LOG_INFO: \
+ VERBS_INFO(FI_LOG_EP_DATA, "%s", str); \
+ break; \
+ case FI_LOG_DEBUG: \
+ default: \
+ VERBS_DBG(FI_LOG_EP_DATA, "%s", str); \
+ break; \
+ } \
} while (0);
#else // ENABLE_DEBUG
-#define FI_IBV_RDM_TAGGED_DBG_REQUEST(prefix, request, level)
+#define FI_IBV_RDM_DBG_REQUEST(prefix, request, level)
#endif // ENABLE_DEBUG
-struct fi_verbs_rdm_tagged_request_minfo {
- struct fi_ibv_rdm_tagged_conn *conn;
- uint64_t tag;
- uint64_t tagmask;
+struct fi_ibv_rdm_minfo {
+ struct fi_ibv_rdm_conn *conn;
+ uint64_t is_tagged; /* TODO: unexpected RTS for MSG */
+ uint64_t tag;
+ uint64_t tagmask;
};
-struct fi_ibv_rdm_cm;
+struct fi_ibv_rdm_tagged_peek_data {
+ struct fi_ibv_rdm_minfo minfo;
+ void *context;
+ uint64_t flags;
+};
-int fi_ibv_rdm_tagged_req_match(struct dlist_entry *item, const void *other);
-int fi_ibv_rdm_tagged_req_match_by_info(struct dlist_entry *item,
- const void *info);
-int fi_ibv_rdm_tagged_req_match_by_info2(struct dlist_entry *item,
- const void *info);
-int fi_ibv_rdm_tagged_send_postponed_process(struct dlist_entry *item,
- const void *arg);
-void fi_ibv_rdm_conn_init_cm_role(struct fi_ibv_rdm_tagged_conn *conn,
+struct fi_ibv_rdm_cm;
+struct fi_ibv_rdm_request *request;
+struct fi_ibv_rdm_send_start_data;
+
+int fi_ibv_rdm_req_match(struct dlist_entry *item, const void *other);
+int fi_ibv_rdm_req_match_by_info(struct dlist_entry *item, const void *info);
+int fi_ibv_rdm_req_match_by_info2(struct dlist_entry *item, const void *info);
+int fi_ibv_rdm_req_match_by_info3(struct dlist_entry *item, const void *info);
+int fi_ibv_rdm_postponed_process(struct dlist_entry *item, const void *arg);
+void fi_ibv_rdm_conn_init_cm_role(struct fi_ibv_rdm_conn *conn,
struct fi_ibv_rdm_ep *ep);
-int fi_ibv_rdm_tagged_find_ipoib_addr(const struct sockaddr_in *addr,
- struct fi_ibv_rdm_cm* cm);
+int fi_ibv_rdm_find_ipoib_addr(const struct sockaddr_in *addr,
+ struct sockaddr_in *ipoib_addr);
+
+ssize_t fi_ibv_rdm_send_common(struct fi_ibv_rdm_send_start_data* sdata);
+ssize_t rdm_trecv_second_event(struct fi_ibv_rdm_request *request,
+ struct fi_ibv_rdm_ep *ep);
#endif /* _VERBS_UTILS_H */
diff --git a/prov/verbs/src/fi_verbs.c b/prov/verbs/src/fi_verbs.c
index 551d018..604a9c7 100644
--- a/prov/verbs/src/fi_verbs.c
+++ b/prov/verbs/src/fi_verbs.c
@@ -67,72 +67,59 @@ int fi_ibv_sockaddr_len(struct sockaddr *addr)
}
}
-static int fi_ibv_rdm_cm_init(struct fi_ibv_rdm_cm* cm,
- const struct rdma_addrinfo* rai)
+int fi_ibv_rdm_cm_bind_ep(struct fi_ibv_rdm_cm *cm, struct fi_ibv_rdm_ep *ep)
{
- struct sockaddr_in* src_addr = (struct sockaddr_in*)rai->ai_src_addr;
- cm->ec = rdma_create_event_channel();
-
- if (!cm->ec) {
- VERBS_INFO(FI_LOG_EP_CTRL,
- "Failed to create listener event channel: %s\n",
- strerror(errno));
- return -FI_EOTHER;
- }
+ char my_ipoib_addr_str[INET6_ADDRSTRLEN];
+ struct sockaddr_in* src_addr = (struct sockaddr_in*)ep->rai->ai_src_addr;
- if (fi_fd_nonblock(cm->ec->fd) != 0) {
- VERBS_INFO_ERRNO(FI_LOG_EP_CTRL, "fcntl", errno);
- return -FI_EOTHER;
- }
+ assert(cm->ec && cm->listener);
- if (rdma_create_id(cm->ec, &cm->listener, NULL, RDMA_PS_TCP)) {
- VERBS_INFO(FI_LOG_EP_CTRL, "Failed to create cm listener: %s\n",
- strerror(errno));
- return -FI_EOTHER;
- }
-
- if (fi_ibv_rdm_tagged_find_ipoib_addr(src_addr, cm)) {
+ if (fi_ibv_rdm_find_ipoib_addr(src_addr, &ep->my_addr)) {
VERBS_INFO(FI_LOG_EP_CTRL,
"Failed to find correct IPoIB address\n");
return -FI_ENODEV;
}
- cm->my_addr.sin_port = src_addr->sin_port;
-
- char my_ipoib_addr_str[INET6_ADDRSTRLEN];
- inet_ntop(cm->my_addr.sin_family,
- &cm->my_addr.sin_addr.s_addr,
+ inet_ntop(ep->my_addr.sin_family,
+ &ep->my_addr.sin_addr.s_addr,
my_ipoib_addr_str, INET_ADDRSTRLEN);
VERBS_INFO(FI_LOG_EP_CTRL, "My IPoIB: %s\n", my_ipoib_addr_str);
- if (rdma_bind_addr(cm->listener, (struct sockaddr *)&cm->my_addr)) {
- VERBS_INFO(FI_LOG_EP_CTRL,
- "Failed to bind cm listener to my IPoIB addr %s: %s\n",
- my_ipoib_addr_str, strerror(errno));
- return -FI_EOTHER;
+ if (!cm->is_bound) {
+ errno = 0;
+ if (rdma_bind_addr(cm->listener, (struct sockaddr *)&ep->my_addr)) {
+ VERBS_INFO(FI_LOG_EP_CTRL,
+ "Failed to bind cm listener to my IPoIB addr %s: %s\n",
+ my_ipoib_addr_str, strerror(errno));
+ return -FI_EOTHER;
+ }
+ if (rdma_listen(cm->listener, 1024)) {
+ VERBS_INFO(FI_LOG_EP_CTRL, "rdma_listen failed: %s\n",
+ strerror(errno));
+ return -FI_EOTHER;
+ }
+ cm->is_bound = 1;
}
- if (!cm->my_addr.sin_port) {
- cm->my_addr.sin_port = rdma_get_src_port(cm->listener);
+ if (!ep->my_addr.sin_port) {
+ ep->my_addr.sin_port = rdma_get_src_port(cm->listener);
}
- assert(cm->my_addr.sin_family == AF_INET);
+ assert(ep->my_addr.sin_family == AF_INET);
VERBS_INFO(FI_LOG_EP_CTRL, "My ep_addr: %s:%u\n",
- inet_ntoa(cm->my_addr.sin_addr), ntohs(cm->my_addr.sin_port));
+ inet_ntoa(ep->my_addr.sin_addr), ntohs(ep->my_addr.sin_port));
return FI_SUCCESS;
}
-int fi_ibv_create_ep(const char *node, const char *service,
- uint64_t flags, const struct fi_info *hints,
- struct rdma_addrinfo **rai, struct rdma_cm_id **id)
+int fi_ibv_get_rdma_rai(const char *node, const char *service, uint64_t flags,
+ const struct fi_info *hints, struct rdma_addrinfo **rai)
{
struct rdma_addrinfo rai_hints, *_rai;
struct rdma_addrinfo **rai_current;
- int ret;
+ int ret = fi_ibv_fi_to_rai(hints, flags, &rai_hints);
- ret = fi_ibv_fi_to_rai(hints, flags, &rai_hints);
if (ret)
goto out;
@@ -149,7 +136,9 @@ int fi_ibv_create_ep(const char *node, const char *service,
&rai_hints, &_rai);
if (ret) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "rdma_getaddrinfo", errno);
- ret = -errno;
+ if (errno) {
+ ret = -errno;
+ }
goto out;
}
@@ -171,25 +160,9 @@ int fi_ibv_create_ep(const char *node, const char *service,
}
}
- if (FI_IBV_EP_TYPE_IS_RDM(hints)) {
- struct fi_ibv_rdm_cm* cm =
- container_of(id, struct fi_ibv_rdm_cm, listener);
- fi_ibv_rdm_cm_init(cm, _rai);
- } else {
- ret = rdma_create_ep(id, _rai, NULL, NULL);
- if (ret) {
- VERBS_INFO_ERRNO(FI_LOG_FABRIC, "rdma_create_ep", errno);
- ret = -errno;
- goto err;
- }
- }
-
- if (rai) {
+ if (rai)
*rai = _rai;
- goto out;
- }
-err:
- rdma_freeaddrinfo(_rai);
+
out:
if (rai_hints.ai_src_addr)
free(rai_hints.ai_src_addr);
@@ -198,19 +171,54 @@ out:
return ret;
}
-void fi_ibv_destroy_ep(enum fi_ep_type ep_type,
- struct rdma_addrinfo *rai,
- struct rdma_cm_id **id)
+int fi_ibv_create_ep(const char *node, const char *service,
+ uint64_t flags, const struct fi_info *hints,
+ struct rdma_addrinfo **rai, struct rdma_cm_id **id)
{
- rdma_freeaddrinfo(rai);
- if (ep_type == FI_EP_RDM) {
- struct fi_ibv_rdm_cm* cm =
- container_of(id, struct fi_ibv_rdm_cm, listener);
- rdma_destroy_id(cm->listener);
- rdma_destroy_event_channel(cm->ec);
+ struct rdma_addrinfo *_rai;
+ struct sockaddr *local_addr;
+ int ret;
+
+ ret = fi_ibv_get_rdma_rai(node, service, flags, hints, &_rai);
+ if (ret) {
+ return ret;
+ }
+
+ ret = rdma_create_ep(id, _rai, NULL, NULL);
+ if (ret) {
+ VERBS_INFO_ERRNO(FI_LOG_FABRIC, "rdma_create_ep", errno);
+ ret = -errno;
+ goto err1;
+ }
+ if (rai && !_rai->ai_src_addr) {
+ local_addr = rdma_get_local_addr(*id);
+ _rai->ai_src_len = fi_ibv_sockaddr_len(local_addr);
+ if (!(_rai->ai_src_addr = malloc(_rai->ai_src_len))) {
+ ret = -FI_ENOMEM;
+ goto err2;
+ }
+ memcpy(_rai->ai_src_addr, local_addr, _rai->ai_src_len);
+ }
+
+ if (rai) {
+ *rai = _rai;
} else {
- rdma_destroy_ep(*id);
+ rdma_freeaddrinfo(_rai);
}
+
+ return ret;
+err2:
+ rdma_destroy_ep(*id);
+err1:
+ rdma_freeaddrinfo(_rai);
+
+ return ret;
+}
+
+void fi_ibv_destroy_ep(struct rdma_addrinfo *rai, struct rdma_cm_id **id)
+{
+ rdma_freeaddrinfo(rai);
+ rdma_destroy_ep(*id);
}
#define VERBS_SIGNAL_SEND(ep) \
@@ -366,5 +374,30 @@ static void fi_ibv_fini(void)
VERBS_INI
{
+ fi_param_define(&fi_ibv_prov, "iface", FI_PARAM_STRING,
+ "the prefix or the full name of the network interface "
+ "associated with the IB device (default: ib)");
+ fi_param_define(&fi_ibv_prov, "rdm_buffer_num", FI_PARAM_INT,
+ "the number of pre-registered buffers for buffered "
+ "operations between the endpoints, must be a power of 2 "
+ "(default: 8)");
+ fi_param_define(&fi_ibv_prov, "rdm_buffer_size", FI_PARAM_INT,
+ "the maximum size of a buffered operation (bytes) "
+ "(default: platform specific)");
+ fi_param_define(&fi_ibv_prov, "rdm_rndv_seg_size", FI_PARAM_INT,
+ "the segment size for zero copy protocols (bytes)"
+ "(default: 1073741824)");
+ fi_param_define(&fi_ibv_prov, "rdm_cqread_bunch_size", FI_PARAM_INT,
+ "the number of entries to be read from the verbs "
+ "completion queue at a time (default: 8)");
+ fi_param_define(&fi_ibv_prov, "rdm_thread_timeout", FI_PARAM_INT,
+ "the wake up timeout of the helper thread (usec) "
+ "(default: 100)");
+ fi_param_define(&fi_ibv_prov, "rdm_eager_send_opcode", FI_PARAM_STRING,
+ "the operation code that will be used for eager messaging. "
+ "Only IBV_WR_SEND and IBV_WR_RDMA_WRITE_WITH_IMM are supported. "
+ "The last one is not applicable for iWarp. "
+ "(default: IBV_WR_SEND)");
+
return &fi_ibv_prov;
}
diff --git a/prov/verbs/src/fi_verbs.h b/prov/verbs/src/fi_verbs.h
index e7ecc6a..f5721ce 100644
--- a/prov/verbs/src/fi_verbs.h
+++ b/prov/verbs/src/fi_verbs.h
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2013-2015 Intel Corporation, Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -57,14 +58,12 @@
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_domain.h>
-#include <rdma/fi_prov.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
#include "fi.h"
#include "fi_enosys.h"
-#include <rdma/fi_log.h>
#include "prov.h"
#include "fi_list.h"
#include "fi_signal.h"
@@ -101,6 +100,8 @@
#define VERBS_WCE_CNT 1024
#define VERBS_EPE_CNT 1024
+#define VERBS_DEF_CQ_SIZE 1024
+
extern struct fi_provider fi_ibv_prov;
@@ -137,12 +138,25 @@ struct fi_ibv_eq {
int fi_ibv_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr,
struct fid_eq **eq, void *context);
+struct fi_ibv_rdm_ep;
+
+typedef struct fi_ibv_rdm_conn *
+ (*fi_ibv_rdm_addr_to_conn_func)
+ (struct fi_ibv_rdm_ep *ep, fi_addr_t addr);
+
+typedef fi_addr_t
+ (*fi_ibv_rdm_conn_to_addr_func)
+ (struct fi_ibv_rdm_ep *ep, struct fi_ibv_rdm_conn *conn);
+
struct fi_ibv_av {
- struct fid_av av; /* TODO: rename to av_fid */
+ struct fid_av av_fid;
struct fi_ibv_domain *domain;
- struct fi_ibv_rdm_ep *ep; /* TODO: check usage */
- int type; /* TODO: AV enum? */
+ struct fi_ibv_rdm_ep *ep;
size_t count;
+ size_t used;
+ enum fi_av_type type;
+ fi_ibv_rdm_addr_to_conn_func addr_to_conn;
+ fi_ibv_rdm_conn_to_addr_func conn_to_addr;
};
int fi_ibv_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
@@ -159,12 +173,19 @@ struct fi_ibv_pep {
};
struct fi_ops_cm *fi_ibv_pep_ops_cm(struct fi_ibv_pep *pep);
+struct fi_ibv_rdm_cm;
struct fi_ibv_domain {
struct fid_domain domain_fid;
struct ibv_context *verbs;
struct ibv_pd *pd;
+ /*
+ * TODO: Currently, only 1 rdm EP can be created per rdm domain!
+ * CM logic should be separated from EP,
+ * excluding naming/addressing
+ */
int rdm;
+ struct fi_ibv_rdm_cm *rdm_cm;
struct fi_info *info;
struct fi_ibv_fabric *fab;
};
@@ -196,15 +217,22 @@ struct fi_ibv_cq {
uint64_t wr_id_mask;
fi_ibv_trywait_func trywait;
atomic_t nevents;
- /* RDM EP fields - TODO: check usage */
+};
+
+struct fi_ibv_rdm_cq {
+ struct fid_cq cq_fid;
+ struct fi_ibv_domain *domain;
struct fi_ibv_rdm_ep *ep;
- int format;
+ struct dlist_entry request_cq;
+ struct dlist_entry request_errcq;
+ uint64_t flags;
+ size_t entry_size;
+ int read_bunch_size;
+ enum fi_cq_wait_cond wait_cond;
};
int fi_ibv_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
struct fid_cq **cq, void *context);
-struct fi_ops_cq *fi_ibv_cq_ops_tagged(struct fi_ibv_cq *cq);
-
struct fi_ibv_mem_desc {
struct fid_mr mr_fid;
@@ -212,12 +240,21 @@ struct fi_ibv_mem_desc {
struct fi_ibv_domain *domain;
};
+struct fi_ibv_srq_ep {
+ struct fid_ep ep_fid;
+ struct ibv_srq *srq;
+};
+
+int fi_ibv_srq_context(struct fid_domain *domain, struct fi_rx_attr *attr,
+ struct fid_ep **rx_ep, void *context);
+
struct fi_ibv_msg_ep {
struct fid_ep ep_fid;
struct rdma_cm_id *id;
struct fi_ibv_eq *eq;
struct fi_ibv_cq *rcq;
struct fi_ibv_cq *scq;
+ struct fi_ibv_srq_ep *srq_ep;
uint64_t ep_flags;
struct fi_info *info;
atomic_t unsignaled_send_cnt;
@@ -234,21 +271,23 @@ int fi_ibv_open_ep(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context);
int fi_ibv_passive_ep(struct fid_fabric *fabric, struct fi_info *info,
struct fid_pep **pep, void *context);
-int fi_ibv_open_rdm_ep(struct fid_domain *domain, struct fi_info *info,
+int fi_ibv_rdm_open_ep(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context);
int fi_ibv_create_ep(const char *node, const char *service,
uint64_t flags, const struct fi_info *hints,
struct rdma_addrinfo **rai, struct rdma_cm_id **id);
-void fi_ibv_destroy_ep(enum fi_ep_type ep_type,
- struct rdma_addrinfo *rai,
- struct rdma_cm_id **id);
-
+void fi_ibv_destroy_ep(struct rdma_addrinfo *rai, struct rdma_cm_id **id);
+int fi_rbv_rdm_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
+ struct fid_cntr **cntr, void *context);
struct fi_ops_atomic *fi_ibv_msg_ep_ops_atomic(struct fi_ibv_msg_ep *ep);
struct fi_ops_cm *fi_ibv_msg_ep_ops_cm(struct fi_ibv_msg_ep *ep);
struct fi_ops_msg *fi_ibv_msg_ep_ops_msg(struct fi_ibv_msg_ep *ep);
struct fi_ops_rma *fi_ibv_msg_ep_ops_rma(struct fi_ibv_msg_ep *ep);
-struct fi_ops_rma *fi_ibv_rdm_ep_ops_rma(struct fi_ibv_rdm_ep *ep);
+struct fi_ops_rma *fi_ibv_rdm_ep_ops_rma();
+struct fi_ops_msg *fi_ibv_rdm_ep_ops_msg();
+
+struct fi_ops_msg *fi_ibv_msg_srq_ep_ops_msg(struct fi_ibv_msg_ep *ep);
struct fi_ibv_connreq {
struct fid handle;
@@ -266,6 +305,9 @@ struct fi_info *fi_ibv_get_verbs_info(const char *domain_name);
void fi_ibv_update_info(const struct fi_info *hints, struct fi_info *info);
int fi_ibv_fi_to_rai(const struct fi_info *fi, uint64_t flags,
struct rdma_addrinfo *rai);
+int fi_ibv_get_rdma_rai(const char *node, const char *service, uint64_t flags,
+ const struct fi_info *hints, struct rdma_addrinfo **rai);
+int fi_ibv_rdm_cm_bind_ep(struct fi_ibv_rdm_cm *cm, struct fi_ibv_rdm_ep *ep);
struct verbs_ep_domain {
char *suffix;
diff --git a/prov/verbs/src/verbs_av.c b/prov/verbs/src/verbs_av.c
index 072fca5..3e70472 100644
--- a/prov/verbs/src/verbs_av.c
+++ b/prov/verbs/src/verbs_av.c
@@ -1,4 +1,5 @@
/*
+ * Copyright (c) 2016, Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2013-2015 Intel Corporation, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -30,14 +31,17 @@
* SOFTWARE.
*/
+#include <pthread.h>
+#include <stdio.h>
+
#include <fi_enosys.h>
#include "fi_verbs.h"
-
+#include "ep_rdm/verbs_rdm.h"
static int fi_ibv_av_close(fid_t fid)
{
- struct fi_ibv_av *fid_av = container_of(fid, struct fi_ibv_av, av.fid);
- free(fid_av);
+ struct fi_ibv_av *av = container_of(fid, struct fi_ibv_av, av_fid.fid);
+ free(av);
return 0;
}
@@ -47,13 +51,47 @@ static struct fi_ops fi_ibv_fi_ops = {
.bind = fi_no_bind,
};
+static inline struct fi_ibv_rdm_conn *
+fi_ibv_rdm_av_tbl_idx_to_conn(struct fi_ibv_rdm_ep *ep, fi_addr_t addr)
+{
+ return (addr == FI_ADDR_UNSPEC) ? NULL : ep->domain->rdm_cm->conn_table[addr];
+}
+
+static inline struct fi_ibv_rdm_conn *
+fi_ibv_rdm_av_map_addr_to_conn(struct fi_ibv_rdm_ep *ep, fi_addr_t addr)
+{
+ return (struct fi_ibv_rdm_conn *)
+ (addr == FI_ADDR_UNSPEC ? NULL : (void *)addr);
+}
+
+static inline fi_addr_t
+fi_ibv_rdm_to_conn_to_av_tbl_idx(struct fi_ibv_rdm_ep *ep, struct fi_ibv_rdm_conn *conn)
+{
+ size_t i;
+ if (conn == NULL)
+ return FI_ADDR_UNSPEC;
+
+ for (i = 0; i < ep->av->used; i++) {
+ if (ep->domain->rdm_cm->conn_table[i] == conn) {
+ return i;
+ }
+ }
+
+ return FI_ADDR_UNSPEC;
+}
+
+static inline fi_addr_t
+fi_ibv_rdm_conn_to_av_map_addr(struct fi_ibv_rdm_ep *ep, struct fi_ibv_rdm_conn *conn)
+{
+ return (conn == NULL) ? FI_ADDR_UNSPEC : (fi_addr_t)(uintptr_t)conn;
+}
+
/* TODO: match rest of verbs code for variable naming */
int fi_ibv_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
- struct fid_av **av, void *context)
+ struct fid_av **av_fid, void *context)
{
struct fi_ibv_domain *fid_domain;
- struct fi_ibv_av *fid_av;
- int type = FI_AV_MAP;
+ struct fi_ibv_av *av;
size_t count = 64;
fid_domain = container_of(domain, struct fi_ibv_domain, domain_fid);
@@ -62,9 +100,10 @@ int fi_ibv_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
return -FI_EINVAL;
switch (attr->type) {
+ case FI_AV_UNSPEC:
+ attr->type = FI_AV_MAP;
case FI_AV_MAP:
case FI_AV_TABLE:
- type = attr->type;
break;
default:
return -EINVAL;
@@ -73,21 +112,39 @@ int fi_ibv_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
if (attr->count)
count = attr->count;
- fid_av = calloc(1, sizeof *fid_av);
- if (!fid_av)
+ av = calloc(1, sizeof *av);
+ if (!av)
return -ENOMEM;
- fid_av->domain = fid_domain;
- fid_av->type = type;
- fid_av->count = count;
+ assert(fid_domain->rdm);
+ av->domain = fid_domain;
+ av->type = attr->type;
+ av->count = count;
+ av->used = 0;
- fid_av->av.fid.fclass = FI_CLASS_AV;
- fid_av->av.fid.context = context;
- fid_av->av.fid.ops = &fi_ibv_fi_ops;
+ if (av->type == FI_AV_TABLE && av->count > 0) {
+ av->domain->rdm_cm->conn_table =
+ calloc(av->count, sizeof(*av->domain->rdm_cm->conn_table));
+ if (!av->domain->rdm_cm->conn_table) {
+ free(av);
+ return -ENOMEM;
+ }
+ }
- assert(fid_domain->rdm);
- fid_av->av.ops = fi_ibv_rdm_set_av_ops();
+ if (av->type == FI_AV_MAP) {
+ av->addr_to_conn = fi_ibv_rdm_av_map_addr_to_conn;
+ av->conn_to_addr = fi_ibv_rdm_conn_to_av_map_addr;
+ } else /* if (av->type == FI_AV_TABLE) */ {
+ av->addr_to_conn = fi_ibv_rdm_av_tbl_idx_to_conn;
+ av->conn_to_addr = fi_ibv_rdm_to_conn_to_av_tbl_idx;
+ }
+
+ av->av_fid.fid.fclass = FI_CLASS_AV;
+ av->av_fid.fid.context = context;
+ av->av_fid.fid.ops = &fi_ibv_fi_ops;
+
+ av->av_fid.ops = fi_ibv_rdm_set_av_ops();
- *av = &fid_av->av;
+ *av_fid = &av->av_fid;
return 0;
}
diff --git a/prov/verbs/src/verbs_cm.c b/prov/verbs/src/verbs_cm.c
index 53ac606..aa1c556 100644
--- a/prov/verbs/src/verbs_cm.c
+++ b/prov/verbs/src/verbs_cm.c
@@ -141,6 +141,9 @@ fi_ibv_msg_ep_connect(struct fid_ep *ep, const void *addr,
conn_param.retry_count = 15;
conn_param.rnr_retry_count = 7;
+ if (_ep->srq_ep)
+ conn_param.srq = 1;
+
src_addr = rdma_get_local_addr(_ep->id);
if (src_addr) {
FI_INFO(&fi_ibv_prov, FI_LOG_CORE, "src_addr: %s:%d\n",
@@ -181,6 +184,9 @@ fi_ibv_msg_ep_accept(struct fid_ep *ep, const void *param, size_t paramlen)
conn_param.flow_control = 1;
conn_param.rnr_retry_count = 7;
+ if (_ep->srq_ep)
+ conn_param.srq = 1;
+
ret = rdma_accept(_ep->id, &conn_param);
if (ret)
return -errno;
@@ -249,7 +255,7 @@ static int fi_ibv_pep_setname(fid_t pep_fid, void *addr, size_t addrlen)
FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "Unable to destroy previous rdma_cm_id\n");
return -errno;
}
- ret = rdma_create_id(NULL, &pep->id, NULL, RDMA_PS_TCP);
+ ret = rdma_create_id(NULL, &pep->id, &pep->pep_fid.fid, RDMA_PS_TCP);
if (ret) {
FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "Unable to create rdma_cm_id\n");
return -errno;
@@ -258,7 +264,7 @@ static int fi_ibv_pep_setname(fid_t pep_fid, void *addr, size_t addrlen)
ret = rdma_bind_addr(pep->id, (struct sockaddr *)addr);
if (ret) {
- FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "Unable to bind addres to rdma_cm_id\n");
+ FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "Unable to bind address to rdma_cm_id\n");
return -errno;
}
diff --git a/prov/verbs/src/verbs_cq.c b/prov/verbs/src/verbs_cq.c
index 16cca9d..02f191c 100644
--- a/prov/verbs/src/verbs_cq.c
+++ b/prov/verbs/src/verbs_cq.c
@@ -36,6 +36,40 @@
#include "fi_verbs.h"
+static uint64_t fi_ibv_comp_flags(struct ibv_wc *wc)
+{
+ uint64_t flags = 0;
+
+ if (wc->wc_flags & IBV_WC_WITH_IMM)
+ flags |= FI_REMOTE_CQ_DATA;
+
+ switch (wc->opcode) {
+ case IBV_WC_SEND:
+ flags |= FI_SEND | FI_MSG;
+ break;
+ case IBV_WC_RDMA_WRITE:
+ flags |= FI_RMA | FI_WRITE;
+ break;
+ case IBV_WC_RDMA_READ:
+ flags |= FI_RMA | FI_READ;
+ break;
+ case IBV_WC_COMP_SWAP:
+ flags |= FI_ATOMIC;
+ break;
+ case IBV_WC_FETCH_ADD:
+ flags |= FI_ATOMIC;
+ break;
+ case IBV_WC_RECV:
+ flags |= FI_RECV | FI_MSG;
+ break;
+ case IBV_WC_RECV_RDMA_WITH_IMM:
+ flags |= FI_RMA | FI_REMOTE_WRITE;
+ break;
+ default:
+ break;
+ }
+ return flags;
+}
static ssize_t
fi_ibv_cq_readerr(struct fid_cq *cq_fid, struct fi_cq_err_entry *entry,
@@ -61,7 +95,7 @@ fi_ibv_cq_readerr(struct fid_cq *cq_fid, struct fi_cq_err_entry *entry,
wce = container_of(slist_entry, struct fi_ibv_wce, entry);
entry->op_context = (void *) (uintptr_t) wce->wc.wr_id;
- entry->flags = 0;
+ entry->flags = fi_ibv_comp_flags(&wce->wc);
entry->err = EIO;
entry->prov_errno = wce->wc.status;
memcpy(&entry->err_data, &wce->wc.vendor_err,
@@ -123,7 +157,9 @@ fi_ibv_cq_sread(struct fid_cq *cq, void *buf, size_t count, const void *cond,
ssize_t ret = 0, cur;
ssize_t threshold;
struct fi_ibv_cq *_cq;
+ uint8_t *p;
+ p = buf;
_cq = container_of(cq, struct fi_ibv_cq, cq_fid);
if (!_cq->channel)
@@ -139,9 +175,9 @@ fi_ibv_cq_sread(struct fid_cq *cq, void *buf, size_t count, const void *cond,
break;
}
- ret = _cq->cq_fid.ops->read(&_cq->cq_fid, buf, count - cur);
+ ret = _cq->cq_fid.ops->read(&_cq->cq_fid, p, count - cur);
if (ret > 0) {
- buf += ret * _cq->entry_size;
+ p += ret * _cq->entry_size;
cur += ret;
if (cur >= threshold)
break;
@@ -153,41 +189,6 @@ fi_ibv_cq_sread(struct fid_cq *cq, void *buf, size_t count, const void *cond,
return cur ? cur : ret;
}
-static uint64_t fi_ibv_comp_flags(struct ibv_wc *wc)
-{
- uint64_t flags = 0;
-
- if (wc->wc_flags & IBV_WC_WITH_IMM)
- flags |= FI_REMOTE_CQ_DATA;
-
- switch (wc->opcode) {
- case IBV_WC_SEND:
- flags |= FI_SEND | FI_MSG;
- break;
- case IBV_WC_RDMA_WRITE:
- flags |= FI_RMA | FI_WRITE;
- break;
- case IBV_WC_RDMA_READ:
- flags |= FI_RMA | FI_READ;
- break;
- case IBV_WC_COMP_SWAP:
- flags |= FI_ATOMIC;
- break;
- case IBV_WC_FETCH_ADD:
- flags |= FI_ATOMIC;
- break;
- case IBV_WC_RECV:
- flags |= FI_RECV | FI_MSG;
- break;
- case IBV_WC_RECV_RDMA_WITH_IMM:
- flags |= FI_RMA | FI_REMOTE_WRITE;
- break;
- default:
- break;
- }
- return flags;
-}
-
static void fi_ibv_cq_read_context_entry(struct ibv_wc *wc, int i, void *buf)
{
struct fi_cq_entry *entry = buf;
@@ -466,10 +467,10 @@ static int fi_ibv_cq_close(fid_t fid)
}
if (cq->signal_fd[0]) {
- close(cq->signal_fd[0]);
+ ofi_close_socket(cq->signal_fd[0]);
}
if (cq->signal_fd[1]) {
- close(cq->signal_fd[1]);
+ ofi_close_socket(cq->signal_fd[1]);
}
if (cq->channel)
@@ -492,6 +493,7 @@ int fi_ibv_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
{
struct fi_ibv_cq *_cq;
int ep_cnt_bits = 0;
+ size_t size;
int ret;
_cq = calloc(1, sizeof *_cq);
@@ -499,6 +501,10 @@ int fi_ibv_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
return -FI_ENOMEM;
_cq->domain = container_of(domain, struct fi_ibv_domain, domain_fid);
+ /*
+ * RDM functionality is moved to correspond separated functions
+ */
+ assert(!_cq->domain->rdm);
switch (attr->wait_obj) {
case FI_WAIT_UNSPEC:
@@ -506,6 +512,8 @@ int fi_ibv_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
_cq->channel = ibv_create_comp_channel(_cq->domain->verbs);
if (!_cq->channel) {
ret = -errno;
+ FI_WARN(&fi_ibv_prov, FI_LOG_CQ,
+ "Unable to create completion channel\n");
goto err1;
}
@@ -530,22 +538,23 @@ int fi_ibv_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
goto err3;
}
- if (!_cq->domain->rdm) {
- _cq->cq = ibv_create_cq(_cq->domain->verbs, attr->size, _cq,
- _cq->channel, attr->signaling_vector);
+ size = attr->size ? attr->size : VERBS_DEF_CQ_SIZE;
- if (!_cq->cq) {
- ret = -errno;
- goto err3;
- }
+ _cq->cq = ibv_create_cq(_cq->domain->verbs, size, _cq, _cq->channel,
+ attr->signaling_vector);
- if (_cq->channel) {
- ret = ibv_req_notify_cq(_cq->cq, 0);
- if (ret) {
- FI_WARN(&fi_ibv_prov, FI_LOG_CQ,
- "ibv_req_notify_cq failed\n");
- goto err4;
- }
+ if (!_cq->cq) {
+ ret = -errno;
+ FI_WARN(&fi_ibv_prov, FI_LOG_CQ, "Unable to create verbs CQ\n");
+ goto err3;
+ }
+
+ if (_cq->channel) {
+ ret = ibv_req_notify_cq(_cq->cq, 0);
+ if (ret) {
+ FI_WARN(&fi_ibv_prov, FI_LOG_CQ,
+ "ibv_req_notify_cq failed\n");
+ goto err4;
}
}
@@ -557,26 +566,20 @@ int fi_ibv_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
_cq->cq_fid.ops = &fi_ibv_cq_ops;
switch (attr->format) {
+ case FI_CQ_FORMAT_UNSPEC:
case FI_CQ_FORMAT_CONTEXT:
- assert(!_cq->domain->rdm);
_cq->read_entry = fi_ibv_cq_read_context_entry;
_cq->entry_size = sizeof(struct fi_cq_entry);
break;
case FI_CQ_FORMAT_MSG:
- assert(!_cq->domain->rdm);
_cq->read_entry = fi_ibv_cq_read_msg_entry;
_cq->entry_size = sizeof(struct fi_cq_msg_entry);
break;
case FI_CQ_FORMAT_DATA:
- assert(!_cq->domain->rdm);
_cq->read_entry = fi_ibv_cq_read_data_entry;
_cq->entry_size = sizeof(struct fi_cq_data_entry);
break;
case FI_CQ_FORMAT_TAGGED:
- assert(_cq->domain->rdm);
- _cq->cq_fid.ops = fi_ibv_cq_ops_tagged(_cq);
- _cq->entry_size = sizeof(struct fi_cq_tagged_entry);
- break;
default:
ret = -FI_ENOSYS;
goto err4;
@@ -601,8 +604,8 @@ int fi_ibv_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
err4:
ibv_destroy_cq(_cq->cq);
err3:
- close(_cq->signal_fd[0]);
- close(_cq->signal_fd[1]);
+ ofi_close_socket(_cq->signal_fd[0]);
+ ofi_close_socket(_cq->signal_fd[1]);
err2:
if (_cq->channel)
ibv_destroy_comp_channel(_cq->channel);
diff --git a/prov/verbs/src/verbs_domain.c b/prov/verbs/src/verbs_domain.c
index a981fad..3cefe8b 100644
--- a/prov/verbs/src/verbs_domain.c
+++ b/prov/verbs/src/verbs_domain.c
@@ -125,6 +125,12 @@ static int fi_ibv_domain_close(fid_t fid)
int ret;
domain = container_of(fid, struct fi_ibv_domain, domain_fid.fid);
+
+ if (domain->rdm) {
+ rdma_destroy_ep(domain->rdm_cm->listener);
+ free(domain->rdm_cm);
+ }
+
if (domain->pd) {
ret = ibv_dealloc_pd(domain->pd);
if (ret)
@@ -189,16 +195,16 @@ static struct fi_ops_domain fi_ibv_domain_ops = {
.cntr_open = fi_no_cntr_open,
.poll_open = fi_no_poll_open,
.stx_ctx = fi_no_stx_context,
- .srx_ctx = fi_no_srx_context,
+ .srx_ctx = fi_ibv_srq_context,
};
static struct fi_ops_domain fi_ibv_rdm_domain_ops = {
.size = sizeof(struct fi_ops_domain),
.av_open = fi_ibv_av_open,
- .cq_open = fi_ibv_cq_open,
- .endpoint = fi_ibv_open_rdm_ep,
+ .cq_open = fi_ibv_rdm_cq_open,
+ .endpoint = fi_ibv_rdm_open_ep,
.scalable_ep = fi_no_scalable_ep,
- .cntr_open = fi_no_cntr_open,
+ .cntr_open = fi_rbv_rdm_cntr_open,
.poll_open = fi_no_poll_open,
.stx_ctx = fi_no_stx_context,
.srx_ctx = fi_no_srx_context,
@@ -229,6 +235,13 @@ fi_ibv_domain(struct fid_fabric *fabric, struct fi_info *info,
goto err1;
_domain->rdm = FI_IBV_EP_TYPE_IS_RDM(info);
+ if (_domain->rdm) {
+ _domain->rdm_cm = calloc(1, sizeof(*_domain->rdm_cm));
+ if (!_domain->rdm_cm) {
+ ret = -FI_ENOMEM;
+ goto err2;
+ }
+ }
ret = fi_ibv_open_device_by_name(_domain, info->domain_attr->name);
if (ret)
goto err2;
@@ -242,14 +255,48 @@ fi_ibv_domain(struct fid_fabric *fabric, struct fi_info *info,
_domain->domain_fid.fid.fclass = FI_CLASS_DOMAIN;
_domain->domain_fid.fid.context = context;
_domain->domain_fid.fid.ops = &fi_ibv_fid_ops;
- _domain->domain_fid.ops = _domain->rdm ? &fi_ibv_rdm_domain_ops :
- &fi_ibv_domain_ops;
_domain->domain_fid.mr = &fi_ibv_domain_mr_ops;
+ if (_domain->rdm) {
+ _domain->domain_fid.ops = &fi_ibv_rdm_domain_ops;
+
+ _domain->rdm_cm->ec = rdma_create_event_channel();
+
+ if (!_domain->rdm_cm->ec) {
+ VERBS_INFO(FI_LOG_EP_CTRL,
+ "Failed to create listener event channel: %s\n",
+ strerror(errno));
+ ret = -FI_EOTHER;
+ goto err2;
+ }
+
+ if (fi_fd_nonblock(_domain->rdm_cm->ec->fd) != 0) {
+ VERBS_INFO_ERRNO(FI_LOG_EP_CTRL, "fcntl", errno);
+ ret = -FI_EOTHER;
+ goto err3;
+ }
+
+ if (rdma_create_id(_domain->rdm_cm->ec,
+ &_domain->rdm_cm->listener, NULL, RDMA_PS_TCP))
+ {
+ VERBS_INFO(FI_LOG_EP_CTRL, "Failed to create cm listener: %s\n",
+ strerror(errno));
+ ret = -FI_EOTHER;
+ goto err3;
+ }
+ _domain->rdm_cm->is_bound = 0;
+ } else {
+ _domain->domain_fid.ops = &fi_ibv_domain_ops;
+ }
_domain->fab = container_of(fabric, struct fi_ibv_fabric, fabric_fid);
*domain = &_domain->domain_fid;
return 0;
+err3:
+ if (_domain->rdm)
+ rdma_destroy_event_channel(_domain->rdm_cm->ec);
err2:
+ if (_domain->rdm)
+ free(_domain->rdm_cm);
fi_freeinfo(_domain->info);
err1:
free(_domain);
diff --git a/prov/verbs/src/verbs_eq.c b/prov/verbs/src/verbs_eq.c
index 52f5c05..38468e7 100644
--- a/prov/verbs/src/verbs_eq.c
+++ b/prov/verbs/src/verbs_eq.c
@@ -282,7 +282,7 @@ fi_ibv_eq_sread(struct fid_eq *eq_fid, uint32_t *event,
if (ret == 0)
return -FI_EAGAIN;
else if (ret < 0)
- return ret;
+ return -errno;
};
}
diff --git a/prov/verbs/src/verbs_info.c b/prov/verbs/src/verbs_info.c
index 19e8902..1df5d7d 100644
--- a/prov/verbs/src/verbs_info.c
+++ b/prov/verbs/src/verbs_info.c
@@ -30,6 +30,11 @@
* SOFTWARE.
*/
+#include <fi_util.h>
+
+#include <ifaddrs.h>
+#include <net/if.h>
+
#include "fi_verbs.h"
#include "ep_rdm/verbs_rdm.h"
@@ -41,19 +46,23 @@
#define VERBS_MSG_CAPS (FI_MSG | FI_RMA | FI_ATOMICS | FI_READ | FI_WRITE | \
FI_SEND | FI_RECV | FI_REMOTE_READ | FI_REMOTE_WRITE)
-#define VERBS_RDM_CAPS (FI_SEND | FI_RECV | FI_TAGGED | FI_RMA | FI_READ | \
- FI_WRITE | FI_REMOTE_READ | FI_REMOTE_WRITE)
+#define VERBS_RDM_CAPS (FI_MSG | FI_RMA | FI_TAGGED | FI_READ | FI_WRITE | \
+ FI_RECV | FI_SEND | FI_REMOTE_READ | FI_REMOTE_WRITE )
#define VERBS_MODE (FI_LOCAL_MR)
#define VERBS_RDM_MODE (FI_CONTEXT)
#define VERBS_TX_OP_FLAGS (FI_INJECT | FI_COMPLETION | FI_TRANSMIT_COMPLETE)
#define VERBS_TX_OP_FLAGS_IWARP (FI_INJECT | FI_COMPLETION)
+#define VERBS_TX_OP_FLAGS_IWARP_RDM (VERBS_TX_OP_FLAGS)
#define VERBS_TX_MODE VERBS_MODE
#define VERBS_TX_RDM_MODE VERBS_RDM_MODE
#define VERBS_RX_MODE (FI_LOCAL_MR | FI_RX_CQ_DATA)
+
+#define VERBS_RX_RDM_OP_FLAGS (FI_COMPLETION)
+
#define VERBS_MSG_ORDER (FI_ORDER_RAR | FI_ORDER_RAW | FI_ORDER_RAS | \
FI_ORDER_WAW | FI_ORDER_WAS | FI_ORDER_SAW | FI_ORDER_SAS )
@@ -96,6 +105,14 @@ const struct fi_rx_attr verbs_rx_attr = {
.total_buffered_recv = 0,
};
+const struct fi_rx_attr verbs_rdm_rx_attr = {
+ .mode = VERBS_RX_MODE,
+ .op_flags = VERBS_RX_RDM_OP_FLAGS,
+ .msg_order = VERBS_MSG_ORDER,
+ .total_buffered_recv = 0,
+ .iov_limit = 1
+};
+
const struct fi_tx_attr verbs_tx_attr = {
.mode = VERBS_TX_MODE,
.op_flags = VERBS_TX_OP_FLAGS,
@@ -104,11 +121,11 @@ const struct fi_tx_attr verbs_tx_attr = {
.rma_iov_limit = 1,
};
-const struct fi_tx_attr verbs_tx_rdm_attr = {
+const struct fi_tx_attr verbs_rdm_tx_attr = {
.mode = VERBS_TX_RDM_MODE,
.op_flags = VERBS_TX_OP_FLAGS,
.msg_order = VERBS_MSG_ORDER,
- .inject_size = 0,
+ .inject_size = FI_IBV_RDM_DFLT_BUFFERED_SSIZE,
.rma_iov_limit = 1,
};
@@ -158,7 +175,7 @@ int fi_ibv_check_domain_attr(const struct fi_domain_attr *attr,
case FI_THREAD_FID:
case FI_THREAD_DOMAIN:
case FI_THREAD_COMPLETION:
- case FI_THREAD_ENDPOINT:
+ case FI_THREAD_ENDPOINT:
break;
default:
FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
@@ -297,7 +314,8 @@ int fi_ibv_check_ep_attr(const struct fi_ep_attr *attr,
return -FI_ENODATA;
}
- if (attr->rx_ctx_cnt > info->domain_attr->max_ep_rx_ctx) {
+ if ((attr->rx_ctx_cnt > info->domain_attr->max_ep_rx_ctx) &&
+ (attr->rx_ctx_cnt != FI_SHARED_CONTEXT)) {
FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
"rx_ctx_cnt exceeds supported size\n");
return -FI_ENODATA;
@@ -310,6 +328,7 @@ int fi_ibv_check_rx_attr(const struct fi_rx_attr *attr,
const struct fi_info *hints, const struct fi_info *info)
{
uint64_t compare_mode, check_mode;
+ int rm_enabled;
if (attr->caps & ~(info->rx_attr->caps)) {
FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
@@ -346,7 +365,12 @@ int fi_ibv_check_rx_attr(const struct fi_rx_attr *attr,
return -FI_ENODATA;
}
- if (attr->total_buffered_recv > info->rx_attr->total_buffered_recv) {
+ rm_enabled =(info->domain_attr &&
+ info->domain_attr->resource_mgmt == FI_RM_ENABLED);
+
+ if (!rm_enabled &&
+ (attr->total_buffered_recv > info->rx_attr->total_buffered_recv))
+ {
FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
"Given rx_attr->total_buffered_recv exceeds supported size\n");
return -FI_ENODATA;
@@ -552,8 +576,8 @@ static int fi_ibv_rai_to_fi(struct rdma_addrinfo *rai, struct fi_info *fi)
}
static inline int fi_ibv_get_qp_cap(struct ibv_context *ctx,
- struct ibv_device_attr *device_attr,
- struct fi_info *info)
+ struct ibv_device_attr *device_attr,
+ struct fi_info *info)
{
struct ibv_pd *pd;
struct ibv_cq *cq;
@@ -589,8 +613,8 @@ static inline int fi_ibv_get_qp_cap(struct ibv_context *ctx,
memset(&init_attr, 0, sizeof init_attr);
init_attr.send_cq = cq;
init_attr.recv_cq = cq;
- init_attr.cap.max_send_wr = atoi(def_tx_ctx_size);
- init_attr.cap.max_recv_wr = atoi(def_rx_ctx_size);
+ init_attr.cap.max_send_wr = MIN(atoi(def_tx_ctx_size), device_attr->max_qp_wr);
+ init_attr.cap.max_recv_wr = MIN(atoi(def_rx_ctx_size), device_attr->max_qp_wr);
init_attr.cap.max_send_sge = MIN(atoi(def_tx_iov_limit), device_attr->max_sge);
init_attr.cap.max_recv_sge = MIN(atoi(def_rx_iov_limit), device_attr->max_sge);
init_attr.cap.max_inline_data = atoi(def_inject_size);
@@ -608,7 +632,12 @@ static inline int fi_ibv_get_qp_cap(struct ibv_context *ctx,
info->tx_attr->size = init_attr.cap.max_send_wr;
info->rx_attr->iov_limit = init_attr.cap.max_recv_sge;
- info->rx_attr->size = init_attr.cap.max_recv_wr;
+ /*
+ * On some HW ibv_create_qp can increase max_recv_wr value more than
+ * it really supports. So, alignment with device capability is needed.
+ */
+ info->rx_attr->size = MIN(init_attr.cap.max_recv_wr,
+ device_attr->max_qp_wr);
ibv_destroy_qp(qp);
err2:
@@ -691,6 +720,7 @@ static int fi_ibv_alloc_info(struct ibv_context *ctx, struct fi_info **info,
union ibv_gid gid;
size_t name_len;
int ret;
+ int param;
if (!(fi = fi_allocinfo()))
return -FI_ENOMEM;
@@ -699,13 +729,14 @@ static int fi_ibv_alloc_info(struct ibv_context *ctx, struct fi_info **info,
fi->handle = NULL;
if (ep_dom->type == FI_EP_RDM) {
fi->mode = VERBS_RDM_MODE;
- *(fi->tx_attr) = verbs_tx_rdm_attr;
+ *(fi->tx_attr) = verbs_rdm_tx_attr;
} else {
fi->mode = VERBS_MODE;
*(fi->tx_attr) = verbs_tx_attr;
}
- *(fi->rx_attr) = verbs_rx_attr;
+ *(fi->rx_attr) = (ep_dom->type == FI_EP_RDM)
+ ? verbs_rdm_rx_attr : verbs_rx_attr;
*(fi->ep_attr) = verbs_ep_attr;
*(fi->domain_attr) = verbs_domain_attr;
*(fi->fabric_attr) = verbs_fabric_attr;
@@ -722,6 +753,18 @@ static int fi_ibv_alloc_info(struct ibv_context *ctx, struct fi_info **info,
fi->tx_attr->inject_size = FI_IBV_RDM_DFLT_BUFFERED_SSIZE;
fi->tx_attr->iov_limit = 1;
fi->tx_attr->rma_iov_limit = 1;
+ if (!fi_param_get_int(&fi_ibv_prov, "rdm_buffer_size", ¶m)) {
+ if (param > sizeof (struct fi_ibv_rdm_rndv_header)) {
+ fi->tx_attr->inject_size = param;
+ } else {
+ FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
+ "rdm_buffer_size too small, should be greater then %d\n",
+ sizeof (struct fi_ibv_rdm_rndv_header));
+ ret = -FI_EINVAL;
+ goto err;
+ }
+ }
+ fi->domain_attr->resource_mgmt = FI_RM_ENABLED;
}
switch (ctx->device->transport_type) {
@@ -752,9 +795,13 @@ static int fi_ibv_alloc_info(struct ibv_context *ctx, struct fi_info **info,
goto err;
}
- fi->ep_attr->protocol = (ep_dom == &verbs_msg_domain) ?
- FI_PROTO_IWARP : FI_PROTO_IWARP_RDM;
- fi->tx_attr->op_flags = VERBS_TX_OP_FLAGS_IWARP;
+ if (ep_dom == &verbs_msg_domain) {
+ fi->ep_attr->protocol = FI_PROTO_IWARP;
+ fi->tx_attr->op_flags = VERBS_TX_OP_FLAGS_IWARP;
+ } else {
+ fi->ep_attr->protocol = FI_PROTO_IWARP_RDM;
+ fi->tx_attr->op_flags = VERBS_TX_OP_FLAGS_IWARP_RDM;
+ }
break;
default:
FI_INFO(&fi_ibv_prov, FI_LOG_CORE, "Unknown transport type\n");
@@ -780,11 +827,99 @@ err:
return ret;
}
+static int fi_ibv_copy_ifaddr(const char *name, const char *service, uint64_t flags,
+ struct fi_info *info)
+{
+ struct rdma_addrinfo *rai;
+ struct fi_info *fi;
+ struct rdma_cm_id *id;
+ int ret;
+
+ ret = fi_ibv_get_rdma_rai(name, service, flags, NULL, &rai);
+ if (ret) {
+ FI_WARN(&fi_ibv_prov, FI_LOG_FABRIC,
+ "rdma_getaddrinfo failed for name:%s\n", name);
+ return ret;
+ }
+ ret = rdma_create_ep(&id, rai, NULL, NULL);
+ if (!ret) {
+ for (fi = info; fi; fi = fi->next)
+ if (!strncmp(id->verbs->device->name, fi->domain_attr->name,
+ strlen(id->verbs->device->name)))
+ break;
+ if (!fi) {
+ FI_WARN(&fi_ibv_prov, FI_LOG_FABRIC,
+ "No matching fi_info for device: "
+ "%s with address: %s\n",
+ id->verbs->device->name, name);
+ } else {
+ if (fi->src_addr) {
+ free(fi->src_addr);
+ fi->src_addr = NULL;
+ }
+ fi_ibv_rai_to_fi(rai, fi);
+ }
+ rdma_destroy_ep(id);
+ }
+ rdma_freeaddrinfo(rai);
+ return 0;
+}
+
+static int fi_ibv_getifaddrs(const char *service, uint64_t flags, struct fi_info *info)
+{
+ struct ifaddrs *ifaddr, *ifa;
+ char name[INET6_ADDRSTRLEN];
+ const char *ret_ptr;
+ int ret, num_verbs_ifs = 0;
+
+ flags |= FI_NUMERICHOST | FI_SOURCE;
+
+ ret = getifaddrs(&ifaddr);
+ if (ret) {
+ FI_WARN(&fi_ibv_prov, FI_LOG_FABRIC,
+ "Unable to get interface addresses\n");
+ return ret;
+ }
+
+ for (ifa = ifaddr; ifa; ifa = ifa->ifa_next) {
+ if (!ifa->ifa_addr || !(ifa->ifa_flags & IFF_UP) ||
+ !strcmp(ifa->ifa_name, "lo"))
+ continue;
+ switch (ifa->ifa_addr->sa_family) {
+ case AF_INET:
+ ret_ptr = inet_ntop(AF_INET, &ofi_sin_addr(ifa->ifa_addr),
+ name, INET6_ADDRSTRLEN);
+ break;
+ case AF_INET6:
+ ret_ptr = inet_ntop(AF_INET6, &ofi_sin6_addr(ifa->ifa_addr),
+ name, INET6_ADDRSTRLEN);
+ break;
+ default:
+ continue;
+ }
+ if (!ret_ptr) {
+ FI_WARN(&fi_ibv_prov, FI_LOG_FABRIC,
+ "inet_ntop failed: %s(%d)\n",
+ strerror(errno), errno);
+ goto err;
+ }
+ ret = fi_ibv_copy_ifaddr(name, service, flags, info);
+ if (ret)
+ goto err;
+ num_verbs_ifs++;
+ }
+ freeifaddrs(ifaddr);
+ return num_verbs_ifs ? 0 : -FI_ENODATA;
+err:
+ freeifaddrs(ifaddr);
+ return ret;
+}
+
int fi_ibv_init_info(void)
{
struct ibv_context **ctx_list;
struct fi_info *fi = NULL, *tail = NULL;
- int ret = 0, i, num_devices;
+ int ret = 0, i, num_devices, fork_unsafe = 0;
if (verbs_info)
return 0;
@@ -799,6 +934,21 @@ int fi_ibv_init_info(void)
goto unlock;
}
+ fi_param_get_bool(NULL, "fork_unsafe", &fork_unsafe);
+
+ if (!fork_unsafe) {
+ FI_INFO(&fi_ibv_prov, FI_LOG_CORE, "Enabling IB fork support\n");
+ ret = ibv_fork_init();
+ if (ret) {
+ FI_WARN(&fi_ibv_prov, FI_LOG_CORE,
+ "Enabling IB fork support failed: %s (%d)\n",
+ strerror(ret), ret);
+ goto unlock;
+ }
+ } else {
+ FI_INFO(&fi_ibv_prov, FI_LOG_CORE, "Not enabling IB fork support\n");
+ }
+
ctx_list = rdma_get_devices(&num_devices);
if (!num_devices) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "rdma_get_devices", errno);
@@ -935,22 +1085,13 @@ int fi_ibv_getinfo(uint32_t version, const char *node, const char *service,
{
struct rdma_cm_id *id = NULL;
struct rdma_addrinfo *rai;
- struct fi_ibv_rdm_cm rdm_cm;
int ret;
ret = fi_ibv_init_info();
if (ret)
goto out;
- if (FI_IBV_EP_TYPE_IS_RDM(hints)) {
- memset(&rdm_cm, 0, sizeof(struct fi_ibv_rdm_cm));
- ret = fi_ibv_create_ep(node, service, flags, hints, &rai,
- &(rdm_cm.listener));
- id = rdm_cm.listener;
- } else {
- ret = fi_ibv_create_ep(node, service, flags, hints, &rai, &id);
- }
-
+ ret = fi_ibv_create_ep(node, service, flags, hints, &rai, &id);
if (ret)
goto out;
@@ -959,14 +1100,23 @@ int fi_ibv_getinfo(uint32_t version, const char *node, const char *service,
hints, rai, info);
} else {
ret = fi_ibv_get_matching_info(NULL, hints, rai, info);
+ if (!ret && !(flags & FI_SOURCE) && !node
+ && (!hints || (!hints->src_addr && !hints->dest_addr))) {
+ ret = fi_ibv_getifaddrs(service, flags, *info);
+ if (ret) {
+ fi_freeinfo(*info);
+ fi_ibv_destroy_ep(rai, &id);
+ goto out;
+ }
+ }
}
- if (hints && hints->ep_attr)
- fi_ibv_destroy_ep(hints->ep_attr->type, rai,
- FI_IBV_EP_TYPE_IS_RDM(hints) ? &(rdm_cm.listener) : &id);
+ ofi_alter_info(*info, hints);
+
+ fi_ibv_destroy_ep(rai, &id);
out:
- if (!ret || ret == -FI_ENOMEM)
+ if (!ret || ret == -FI_ENOMEM || ret == -FI_ENODEV)
return ret;
else
return -FI_ENODATA;
diff --git a/prov/verbs/src/verbs_msg.c b/prov/verbs/src/verbs_msg.c
index 4010691..c71fd95 100644
--- a/prov/verbs/src/verbs_msg.c
+++ b/prov/verbs/src/verbs_msg.c
@@ -62,6 +62,7 @@ fi_ibv_msg_ep_recvmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flag
wr.num_sge = msg->iov_count;
ret = ibv_post_recv(_ep->id->qp, &wr, &bad);
+
switch (ret) {
case ENOMEM:
return -FI_EAGAIN;
@@ -222,3 +223,21 @@ struct fi_ops_msg *fi_ibv_msg_ep_ops_msg(struct fi_ibv_msg_ep *ep)
return &fi_ibv_msg_ep_msg_ops;
}
+static struct fi_ops_msg fi_ibv_msg_srq_ep_msg_ops = {
+ .size = sizeof(struct fi_ops_msg),
+ .recv = fi_no_msg_recv,
+ .recvv = fi_no_msg_recvv,
+ .recvmsg = fi_no_msg_recvmsg,
+ .send = fi_ibv_msg_ep_send,
+ .sendv = fi_ibv_msg_ep_sendv,
+ .sendmsg = fi_ibv_msg_ep_sendmsg,
+ .inject = fi_ibv_msg_ep_inject,
+ .senddata = fi_ibv_msg_ep_senddata,
+ .injectdata = fi_ibv_msg_ep_injectdata,
+};
+
+struct fi_ops_msg *fi_ibv_msg_srq_ep_ops_msg(struct fi_ibv_msg_ep *ep)
+{
+ return &fi_ibv_msg_srq_ep_msg_ops;
+}
+
diff --git a/prov/verbs/src/verbs_msg_ep.c b/prov/verbs/src/verbs_msg_ep.c
index aeb5a5e..f8eb5c5 100644
--- a/prov/verbs/src/verbs_msg_ep.c
+++ b/prov/verbs/src/verbs_msg_ep.c
@@ -127,6 +127,9 @@ static int fi_ibv_msg_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
int ret;
ep = container_of(fid, struct fi_ibv_msg_ep, ep_fid.fid);
+ ret = ofi_ep_bind_valid(&fi_ibv_prov, bfid, flags);
+ if (ret)
+ return ret;
switch (bfid->fclass) {
case FI_CLASS_CQ:
@@ -160,6 +163,9 @@ static int fi_ibv_msg_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
if (ret)
return -errno;
break;
+ case FI_CLASS_SRX_CTX:
+ ep->srq_ep = container_of(bfid, struct fi_ibv_srq_ep, ep_fid.fid);
+ break;
default:
return -EINVAL;
}
@@ -185,15 +191,15 @@ static int fi_ibv_msg_ep_enable(struct fid_ep *ep)
return -FI_ENOCQ;
}
- if (!_ep->scq && (fi_send_allowed(_ep->info->caps) ||
- fi_rma_initiate_allowed(_ep->info->caps))) {
+ if (!_ep->scq && (ofi_send_allowed(_ep->info->caps) ||
+ ofi_rma_initiate_allowed(_ep->info->caps))) {
FI_WARN(&fi_ibv_prov, FI_LOG_EP_CTRL, "Endpoint is not bound to "
"a send completion queue when it has transmit "
"capabilities enabled (FI_SEND | FI_RMA).\n");
return -FI_ENOCQ;
}
- if (!_ep->rcq && fi_recv_allowed(_ep->info->caps)) {
+ if (!_ep->rcq && ofi_recv_allowed(_ep->info->caps)) {
FI_WARN(&fi_ibv_prov, FI_LOG_EP_CTRL, "Endpoint is not bound to "
"a receive completion queue when it has receive "
"capabilities enabled. (FI_RECV)\n");
@@ -221,7 +227,16 @@ static int fi_ibv_msg_ep_enable(struct fid_ep *ep)
attr.cap.max_inline_data = _ep->info->tx_attr->inject_size;
- attr.srq = NULL;
+ if (_ep->srq_ep) {
+ attr.srq =_ep->srq_ep->srq;
+ /* Use of SRQ, no need to allocate recv_wr entries in the QP */
+ attr.cap.max_recv_wr = 0;
+
+ /* Override the default ops to prevent the user from posting WRs to a
+ * QP where a SRQ is attached to */
+ _ep->ep_fid.msg = fi_ibv_msg_srq_ep_ops_msg(_ep);
+ }
+
attr.qp_type = IBV_QPT_RC;
attr.sq_sig_all = 0;
attr.qp_context = _ep;
@@ -409,7 +424,7 @@ static int fi_ibv_pep_close(fid_t fid)
return 0;
}
-static struct fi_ops fi_ibv_pep_ops = {
+static struct fi_ops fi_ibv_pep_fi_ops = {
.size = sizeof(struct fi_ops),
.close = fi_ibv_pep_close,
.bind = fi_ibv_pep_bind,
@@ -417,6 +432,16 @@ static struct fi_ops fi_ibv_pep_ops = {
.ops_open = fi_no_ops_open,
};
+static struct fi_ops_ep fi_ibv_pep_ops = {
+ .size = sizeof(struct fi_ops_ep),
+ .getopt = fi_ibv_msg_ep_getopt,
+ .setopt = fi_no_setopt,
+ .tx_ctx = fi_no_tx_ctx,
+ .rx_ctx = fi_no_rx_ctx,
+ .rx_size_left = fi_no_rx_size_left,
+ .tx_size_left = fi_no_tx_size_left,
+};
+
int fi_ibv_passive_ep(struct fid_fabric *fabric, struct fi_info *info,
struct fid_pep **pep, void *context)
{
@@ -427,7 +452,7 @@ int fi_ibv_passive_ep(struct fid_fabric *fabric, struct fi_info *info,
if (!_pep)
return -FI_ENOMEM;
- ret = rdma_create_id(NULL, &_pep->id, NULL, RDMA_PS_TCP);
+ ret = rdma_create_id(NULL, &_pep->id, &_pep->pep_fid.fid, RDMA_PS_TCP);
if (ret) {
FI_INFO(&fi_ibv_prov, FI_LOG_DOMAIN, "Unable to create rdma_cm_id\n");
goto err1;
@@ -436,17 +461,16 @@ int fi_ibv_passive_ep(struct fid_fabric *fabric, struct fi_info *info,
if (info->src_addr) {
ret = rdma_bind_addr(_pep->id, (struct sockaddr *)info->src_addr);
if (ret) {
- FI_INFO(&fi_ibv_prov, FI_LOG_DOMAIN, "Unable to bind addres to rdma_cm_id\n");
+ FI_INFO(&fi_ibv_prov, FI_LOG_DOMAIN, "Unable to bind address to rdma_cm_id\n");
goto err2;
}
_pep->bound = 1;
}
- _pep->id->context = &_pep->pep_fid.fid;
-
_pep->pep_fid.fid.fclass = FI_CLASS_PEP;
_pep->pep_fid.fid.context = context;
- _pep->pep_fid.fid.ops = &fi_ibv_pep_ops;
+ _pep->pep_fid.fid.ops = &fi_ibv_pep_fi_ops;
+ _pep->pep_fid.ops = &fi_ibv_pep_ops;
_pep->pep_fid.cm = fi_ibv_pep_ops_cm(_pep);
_pep->src_addrlen = info->src_addrlen;
diff --git a/prov/verbs/src/verbs_rma.c b/prov/verbs/src/verbs_rma.c
index 3242bb6..e418ab0 100644
--- a/prov/verbs/src/verbs_rma.c
+++ b/prov/verbs/src/verbs_rma.c
@@ -45,7 +45,7 @@
#define VERBS_COMP_READ(ep) \
VERBS_COMP_READ_FLAGS(ep, ep->info->tx_attr->op_flags)
-extern struct util_buf_pool* fi_ibv_rdm_tagged_request_pool;
+extern struct util_buf_pool* fi_ibv_rdm_request_pool;
static ssize_t
fi_ibv_msg_ep_rma_write(struct fid_ep *ep_fid, const void *buf, size_t len,
@@ -245,7 +245,7 @@ struct fi_ops_rma *fi_ibv_msg_ep_ops_rma(struct fi_ibv_msg_ep *ep)
static inline ssize_t
fi_ibv_rdm_ep_rma_preinit(void **desc, void **raw_buf, size_t len,
- struct fi_ibv_rdm_tagged_conn *conn,
+ struct fi_ibv_rdm_conn *conn,
struct fi_ibv_rdm_ep *ep)
{
int again = 0;
@@ -276,68 +276,61 @@ fi_ibv_rdm_ep_rma_preinit(void **desc, void **raw_buf, size_t len,
}
static ssize_t
-fi_ibv_rdm_ep_rma_read(struct fid_ep *ep_fid, void *buf, size_t len,
- void *desc, fi_addr_t src_addr,
- uint64_t addr, uint64_t key, void *context)
+fi_ibv_rdm_ep_rma_readmsg(struct fid_ep *ep_fid, const struct fi_msg_rma *msg,
+ uint64_t flags)
{
- struct fi_ibv_rdm_ep *ep = container_of(ep_fid, struct fi_ibv_rdm_ep,
- ep_fid);
- struct fi_ibv_rdm_tagged_conn *conn =
- (struct fi_ibv_rdm_tagged_conn *)src_addr;
+ struct fi_ibv_rdm_ep *ep =
+ container_of(ep_fid, struct fi_ibv_rdm_ep, ep_fid);
+
+ struct fi_ibv_rdm_conn *conn = ep->av->addr_to_conn(ep, msg->addr);
+
+ struct fi_ibv_rdm_rma_start_data start_data = {
+ .ep_rdm = ep,
+ .conn = conn,
+ .context = msg->context,
+ .flags = FI_RMA | FI_READ | (ep->tx_selective_completion ?
+ (flags & FI_COMPLETION) : FI_COMPLETION),
+ .data_len = (uint64_t)msg->msg_iov[0].iov_len,
+ .rbuf = msg->rma_iov[0].addr,
+ .lbuf = (uintptr_t)msg->msg_iov[0].iov_base,
+ .rkey = (uint32_t)msg->rma_iov[0].key,
+ .lkey = (uint32_t)(uintptr_t)msg->desc[0],
+ .op_code = IBV_WR_RDMA_READ
+ };
+
+ struct fi_ibv_rma_post_ready_data post_ready_data = { .ep_rdm = ep };
+
void *raw_buf = NULL;
+ ssize_t ret = FI_SUCCESS;
- ssize_t ret = fi_ibv_rdm_ep_rma_preinit(&desc, &raw_buf, len, conn, ep);
+ if(msg->iov_count != 1 || msg->rma_iov_count != 1) {
+ assert(0);
+ return -FI_EMSGSIZE;
+ }
+
+ ret = fi_ibv_rdm_ep_rma_preinit(&msg->desc[0], &raw_buf,
+ msg->msg_iov[0].iov_len,
+ conn, ep);
if (ret) {
return ret;
}
- struct fi_ibv_rdm_tagged_request *request =
- util_buf_alloc(fi_ibv_rdm_tagged_request_pool);
- FI_IBV_RDM_TAGGED_DBG_REQUEST("get_from_pool: ", request, FI_LOG_DEBUG);
+ struct fi_ibv_rdm_request *request =
+ util_buf_alloc(fi_ibv_rdm_request_pool);
+ FI_IBV_RDM_DBG_REQUEST("get_from_pool: ", request, FI_LOG_DEBUG);
/* Initial state */
request->state.eager = FI_IBV_STATE_EAGER_BEGIN;
request->state.rndv = FI_IBV_STATE_RNDV_NOT_USED;
- request->rmabuf = raw_buf;
-
- struct fi_ibv_rdm_rma_start_data start_data = {
- .ep_rdm = container_of(ep_fid, struct fi_ibv_rdm_ep, ep_fid),
- .conn = (struct fi_ibv_rdm_tagged_conn *) src_addr,
- .context = context,
- .data_len = (uint32_t)len,
- .rbuf = addr,
- .lbuf = (uintptr_t)buf,
- .rkey = (uint32_t)key,
- .lkey = (uint32_t)(uintptr_t)desc,
- .op_code = IBV_WR_RDMA_READ
- };
-
- fi_ibv_rdm_tagged_req_hndl(request, FI_IBV_EVENT_RMA_START, &start_data);
-
- struct fi_ibv_rma_post_ready_data post_ready_data = { .ep_rdm = ep };
+ request->state.err = FI_SUCCESS;
- ret = fi_ibv_rdm_tagged_req_hndl(request, FI_IBV_EVENT_SEND_READY,
- &post_ready_data);
- return (ret == FI_EP_RDM_HNDL_SUCCESS) ? FI_SUCCESS : -FI_EOTHER;
-}
+ request->minfo.is_tagged = 0;
+ request->rmabuf = raw_buf;
-static ssize_t
-fi_ibv_rdm_ep_rma_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
- uint64_t flags)
-{
- if(msg->iov_count == 1 && msg->rma_iov_count == 1) {
- return fi_ibv_rdm_ep_rma_read(ep,
- msg->msg_iov[0].iov_base,
- msg->msg_iov[0].iov_len,
- msg->desc[0],
- msg->addr,
- msg->rma_iov[0].addr,
- msg->rma_iov[0].key,
- msg->context);
- }
+ fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_RMA_START, &start_data);
- assert(0);
- return -FI_EMSGSIZE;
+ return fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_POST_READY,
+ &post_ready_data);
}
static ssize_t
@@ -345,124 +338,151 @@ fi_ibv_rdm_ep_rma_readv(struct fid_ep *ep, const struct iovec *iov, void **desc,
size_t count, fi_addr_t src_addr, uint64_t addr, uint64_t key,
void *context)
{
+ struct fi_ibv_rdm_ep *ep_rdm =
+ container_of(ep, struct fi_ibv_rdm_ep, ep_fid);
+
struct fi_rma_iov rma_iov = {
- .addr = src_addr,
+ .addr = addr,
.len = 0,
.key = key
};
- size_t i;
- for (i = 0; i < count; i++) {
- rma_iov.len += iov[i].iov_len;
- }
-
struct fi_msg_rma msg = {
.msg_iov = iov,
.desc = desc,
.iov_count = count,
- .addr = addr,
+ .addr = src_addr,
.rma_iov = &rma_iov,
.rma_iov_count = 1,
.context = context,
.data = 0
};
- return fi_ibv_rdm_ep_rma_readmsg(ep, &msg, 0);
+ size_t i;
+ for (i = 0; i < count; i++) {
+ rma_iov.len += iov[i].iov_len;
+ }
+
+ return fi_ibv_rdm_ep_rma_readmsg(ep, &msg,
+ (ep_rdm->tx_selective_completion ? 0ULL : FI_COMPLETION));
}
static ssize_t
-fi_ibv_rdm_ep_rma_write(struct fid_ep *ep_fid, const void *buf, size_t len,
- void *desc, fi_addr_t dest_addr,
- uint64_t addr, uint64_t key, void *context)
+fi_ibv_rdm_ep_rma_read(struct fid_ep *ep_fid, void *buf, size_t len,
+ void *desc, fi_addr_t src_addr,
+ uint64_t addr, uint64_t key, void *context)
+{
+ const struct iovec iov = {
+ .iov_base = buf,
+ .iov_len = len
+ };
+
+ return fi_ibv_rdm_ep_rma_readv(ep_fid, &iov, &desc, 1, src_addr, addr,
+ key, context);
+}
+
+static ssize_t
+fi_ibv_rdm_ep_rma_writemsg(struct fid_ep *ep_fid, const struct fi_msg_rma *msg,
+ uint64_t flags)
{
struct fi_ibv_rdm_ep *ep = container_of(ep_fid, struct fi_ibv_rdm_ep,
ep_fid);
- struct fi_ibv_rdm_tagged_conn *conn =
- (struct fi_ibv_rdm_tagged_conn *) dest_addr;
+ struct fi_ibv_rdm_conn *conn = ep->av->addr_to_conn(ep, msg->addr);
+ struct fi_ibv_rdm_request *request = NULL;
void *raw_buf = NULL;
+ ssize_t ret = FI_SUCCESS;
- ssize_t ret = fi_ibv_rdm_ep_rma_preinit(&desc, &raw_buf, len, conn, ep);
+ struct fi_ibv_rdm_rma_start_data start_data = {
+ .conn = conn,
+ .ep_rdm = ep,
+ .context = msg->context,
+ .flags = FI_RMA | FI_WRITE | (ep->tx_selective_completion ?
+ (flags & FI_COMPLETION) : FI_COMPLETION),
+ .data_len = (uint64_t)msg->msg_iov[0].iov_len,
+ .rbuf = msg->rma_iov[0].addr,
+ .lbuf = (uintptr_t)msg->msg_iov[0].iov_base,
+ .rkey = (uint32_t)msg->rma_iov[0].key,
+ .lkey = (uint32_t)(uintptr_t)msg->desc[0],
+ .op_code = IBV_WR_RDMA_WRITE
+ };
+
+ if(msg->iov_count != 1 && msg->rma_iov_count != 1) {
+ assert(0);
+ return -FI_EMSGSIZE;
+ }
+
+ ret = fi_ibv_rdm_ep_rma_preinit(&msg->desc[0], &raw_buf,
+ msg->msg_iov[0].iov_len,
+ conn, ep);
if (ret) {
return ret;
}
- struct fi_ibv_rdm_tagged_request *request =
- util_buf_alloc(fi_ibv_rdm_tagged_request_pool);
- FI_IBV_RDM_TAGGED_DBG_REQUEST("get_from_pool: ", request, FI_LOG_DEBUG);
+ request = util_buf_alloc(fi_ibv_rdm_request_pool);
+ FI_IBV_RDM_DBG_REQUEST("get_from_pool: ", request, FI_LOG_DEBUG);
/* Initial state */
request->state.eager = FI_IBV_STATE_EAGER_BEGIN;
request->state.rndv = FI_IBV_STATE_RNDV_NOT_USED;
- request->rmabuf = raw_buf;
+ request->state.err = FI_SUCCESS;
- struct fi_ibv_rdm_rma_start_data start_data = {
- .conn = conn,
- .ep_rdm = ep,
- .context = context,
- .data_len = (uint32_t)len,
- .rbuf = addr,
- .lbuf = (uintptr_t)buf,
- .rkey = (uint32_t)key,
- .lkey = (uint32_t)(uintptr_t)desc,
- .op_code = IBV_WR_RDMA_WRITE
- };
+ request->minfo.is_tagged = 0;
+ request->rmabuf = raw_buf;
- fi_ibv_rdm_tagged_req_hndl(request, FI_IBV_EVENT_RMA_START, &start_data);
+ fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_RMA_START, &start_data);
struct fi_ibv_rma_post_ready_data post_ready_data = { .ep_rdm = ep };
- ret = fi_ibv_rdm_tagged_req_hndl(request, FI_IBV_EVENT_SEND_READY,
- &post_ready_data);
- return (ret == FI_EP_RDM_HNDL_SUCCESS) ? FI_SUCCESS : -FI_EOTHER;
+ return fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_POST_READY,
+ &post_ready_data);
}
static ssize_t
-fi_ibv_rdm_ep_rma_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
- uint64_t flags)
-{
- if(msg->iov_count == 1 && msg->rma_iov_count == 1) {
- return fi_ibv_rdm_ep_rma_write(ep,
- msg->msg_iov[0].iov_base,
- msg->msg_iov[0].iov_len,
- msg->desc[0],
- msg->addr,
- msg->rma_iov[0].addr,
- msg->rma_iov[0].key,
- msg->context);
- }
-
- assert(0);
- return -FI_EMSGSIZE;
-}
-
-static ssize_t
-fi_ibv_rdm_ep_rma_writev(struct fid_ep *ep, const struct iovec *iov, void **desc,
+fi_ibv_rdm_ep_rma_writev(struct fid_ep *ep_fid, const struct iovec *iov, void **desc,
size_t count, fi_addr_t dest_addr, uint64_t addr, uint64_t key,
void *context)
{
struct fi_rma_iov rma_iov = {
- .addr = dest_addr,
+ .addr = addr,
.len = 0,
.key = key
};
- size_t i;
- for (i = 0; i < count; i++) {
- rma_iov.len += iov[i].iov_len;
- }
-
struct fi_msg_rma msg = {
.msg_iov = iov,
.desc = desc,
.iov_count = count,
- .addr = addr,
+ .addr = dest_addr,
.rma_iov = &rma_iov,
.rma_iov_count = 1,
.context = context,
.data = 0
};
- return fi_ibv_rdm_ep_rma_writemsg(ep, &msg, 0);
+ size_t i;
+ for (i = 0; i < count; i++) {
+ rma_iov.len += iov[i].iov_len;
+ }
+
+ struct fi_ibv_rdm_ep *ep_rdm =
+ container_of(ep_fid, struct fi_ibv_rdm_ep, ep_fid);
+
+ return fi_ibv_rdm_ep_rma_writemsg(ep_fid, &msg,
+ (ep_rdm->tx_selective_completion ? 0ULL : FI_COMPLETION));
+}
+
+static ssize_t
+fi_ibv_rdm_ep_rma_write(struct fid_ep *ep_fid, const void *buf, size_t len,
+ void *desc, fi_addr_t dest_addr, uint64_t addr,
+ uint64_t key, void *context)
+{
+ const struct iovec iov = {
+ .iov_base = (void *)buf,
+ .iov_len = len
+ };
+
+ return fi_ibv_rdm_ep_rma_writev(ep_fid, &iov, &desc, 1, dest_addr, addr,
+ key, context);
}
static ssize_t fi_ibv_rdm_ep_rma_inject_write(struct fid_ep *ep,
@@ -472,52 +492,50 @@ static ssize_t fi_ibv_rdm_ep_rma_inject_write(struct fid_ep *ep,
{
struct fi_ibv_rdm_ep *ep_rdm = container_of(ep, struct fi_ibv_rdm_ep,
ep_fid);
- struct fi_ibv_rdm_tagged_conn *conn =
- (struct fi_ibv_rdm_tagged_conn *) dest_addr;
- struct fi_ibv_rdm_tagged_request *request = NULL;
+ struct fi_ibv_rdm_conn *conn = ep_rdm->av->addr_to_conn(ep_rdm, dest_addr);
+ struct fi_ibv_rdm_request *request = NULL;
+
+ struct fi_ibv_rdm_rma_start_data start_data = {
+ .conn = conn,
+ .ep_rdm = ep_rdm,
+ .flags = 0, /* inject does not generate completion */
+ .data_len = (uint64_t)len,
+ .rbuf = addr,
+ .lbuf = (uintptr_t)buf,
+ .rkey = (uint32_t)key,
+ .lkey = 0
+ };
ssize_t ret = fi_ibv_rdm_ep_rma_preinit(NULL, NULL, len, conn, ep_rdm);
if (ret) {
return ret;
}
- request = util_buf_alloc(fi_ibv_rdm_tagged_request_pool);
+ request = util_buf_alloc(fi_ibv_rdm_request_pool);
- FI_IBV_RDM_TAGGED_DBG_REQUEST("get_from_pool: ", request, FI_LOG_DEBUG);
+ FI_IBV_RDM_DBG_REQUEST("get_from_pool: ", request, FI_LOG_DEBUG);
/* Initial state */
request->state.eager = FI_IBV_STATE_EAGER_RMA_INJECT;
request->state.rndv = FI_IBV_STATE_RNDV_NOT_USED;
+ request->state.err = FI_SUCCESS;
- struct fi_ibv_rdm_rma_start_data start_data = {
- .conn = conn,
- .ep_rdm = ep_rdm,
- .data_len = (uint32_t)len,
- .rbuf = addr,
- .lbuf = (uintptr_t)buf,
- .rkey = (uint32_t)key,
- .lkey = 0
-
- };
-
- ret = fi_ibv_rdm_tagged_req_hndl(request, FI_IBV_EVENT_RMA_START,
- &start_data);
+ request->minfo.is_tagged = 0;
+ ret = fi_ibv_rdm_req_hndl(request, FI_IBV_EVENT_RMA_START, &start_data);
switch (ret)
{
- case FI_EP_RDM_HNDL_SUCCESS:
+ case FI_SUCCESS:
return ret;
- case FI_EP_RDM_HNDL_AGAIN:
- ret = -FI_EAGAIN;
+ case -FI_EAGAIN:
break;
default:
ret = -errno;
break;
}
- FI_IBV_RDM_TAGGED_DBG_REQUEST("to_pool: ", request,
- FI_LOG_DEBUG);
- util_buf_release(fi_ibv_rdm_tagged_request_pool, request);
+ FI_IBV_RDM_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG);
+ util_buf_release(fi_ibv_rdm_request_pool, request);
fi_ibv_rdm_tagged_poll(ep_rdm);
@@ -537,7 +555,7 @@ static struct fi_ops_rma fi_ibv_rdm_ep_rma_ops = {
.injectdata = fi_no_rma_injectdata,
};
-struct fi_ops_rma *fi_ibv_rdm_ep_ops_rma(struct fi_ibv_rdm_ep *ep)
+struct fi_ops_rma *fi_ibv_rdm_ep_ops_rma()
{
return &fi_ibv_rdm_ep_rma_ops;
}
diff --git a/prov/verbs/src/verbs_srq.c b/prov/verbs/src/verbs_srq.c
new file mode 100644
index 0000000..3c337d9
--- /dev/null
+++ b/prov/verbs/src/verbs_srq.c
@@ -0,0 +1,240 @@
+/*
+ * Copyright (c) 2013-2015 Intel Corporation, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "config.h"
+
+#include "fi_verbs.h"
+
+static struct fi_ops_ep fi_ibv_srq_ep_base_ops = {
+ .size = sizeof(struct fi_ops_ep),
+ .cancel = fi_no_cancel,
+ .getopt = fi_no_getopt,
+ .setopt = fi_no_setopt,
+ .tx_ctx = fi_no_tx_ctx,
+ .rx_ctx = fi_no_rx_ctx,
+ .rx_size_left = fi_no_rx_size_left,
+ .tx_size_left = fi_no_tx_size_left,
+};
+
+static struct fi_ops_cm fi_ibv_srq_cm_ops = {
+ .size = sizeof(struct fi_ops_cm),
+ .setname = fi_no_setname,
+ .getname = fi_no_getname,
+ .getpeer = fi_no_getpeer,
+ .connect = fi_no_connect,
+ .listen = fi_no_listen,
+ .accept = fi_no_accept,
+ .reject = fi_no_reject,
+ .shutdown = fi_no_shutdown,
+};
+
+static struct fi_ops_rma fi_ibv_srq_rma_ops = {
+ .size = sizeof(struct fi_ops_rma),
+ .read = fi_no_rma_read,
+ .readv = fi_no_rma_readv,
+ .readmsg = fi_no_rma_readmsg,
+ .write = fi_no_rma_write,
+ .writev = fi_no_rma_writev,
+ .writemsg = fi_no_rma_writemsg,
+ .inject = fi_no_rma_inject,
+ .writedata = fi_no_rma_writedata,
+ .injectdata = fi_no_rma_injectdata,
+};
+
+static struct fi_ops_atomic fi_ibv_srq_atomic_ops = {
+ .size = sizeof(struct fi_ops_atomic),
+ .write = fi_no_atomic_write,
+ .writev = fi_no_atomic_writev,
+ .writemsg = fi_no_atomic_writemsg,
+ .inject = fi_no_atomic_inject,
+ .readwrite = fi_no_atomic_readwrite,
+ .readwritev = fi_no_atomic_readwritev,
+ .readwritemsg = fi_no_atomic_readwritemsg,
+ .compwrite = fi_no_atomic_compwrite,
+ .compwritev = fi_no_atomic_compwritev,
+ .compwritemsg = fi_no_atomic_compwritemsg,
+ .writevalid = fi_no_atomic_writevalid,
+ .readwritevalid = fi_no_atomic_readwritevalid,
+ .compwritevalid = fi_no_atomic_compwritevalid,
+};
+
+static ssize_t
+fi_ibv_srq_ep_recvmsg(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags)
+{
+ struct fi_ibv_srq_ep *_ep;
+ struct ibv_recv_wr wr, *bad;
+ struct ibv_sge *sge = NULL;
+ ssize_t ret;
+ size_t i;
+
+ _ep = container_of(ep, struct fi_ibv_srq_ep, ep_fid);
+ assert(_ep->srq);
+
+ wr.wr_id = (uintptr_t) msg->context;
+ wr.next = NULL;
+ if (msg->iov_count) {
+ sge = alloca(sizeof(*sge) * msg->iov_count);
+ for (i = 0; i < msg->iov_count; i++) {
+ sge[i].addr = (uintptr_t) msg->msg_iov[i].iov_base;
+ sge[i].length = (uint32_t) msg->msg_iov[i].iov_len;
+ sge[i].lkey = (uint32_t) (uintptr_t) (msg->desc[i]);
+ }
+
+ }
+ wr.sg_list = sge;
+ wr.num_sge = msg->iov_count;
+
+ ret = ibv_post_srq_recv(_ep->srq, &wr, &bad);
+
+ switch (ret) {
+ case ENOMEM:
+ return -FI_EAGAIN;
+ case -1:
+ /* Deal with non-compliant libibverbs drivers which set errno
+ * instead of directly returning the error value */
+ return (errno == ENOMEM) ? -FI_EAGAIN : -errno;
+ default:
+ return -ret;
+ }
+}
+
+static ssize_t
+fi_ibv_srq_ep_recv(struct fid_ep *ep, void *buf, size_t len,
+ void *desc, fi_addr_t src_addr, void *context)
+{
+ struct iovec iov;
+ struct fi_msg msg;
+
+ iov.iov_base = buf;
+ iov.iov_len = len;
+
+ msg.msg_iov = &iov;
+ msg.desc = &desc;
+ msg.iov_count = 1;
+ msg.addr = src_addr;
+ msg.context = context;
+
+ return fi_ibv_srq_ep_recvmsg(ep, &msg, 0);
+}
+
+static ssize_t
+fi_ibv_srq_ep_recvv(struct fid_ep *ep, const struct iovec *iov, void **desc,
+ size_t count, fi_addr_t src_addr, void *context)
+{
+ struct fi_msg msg;
+
+ msg.msg_iov = iov;
+ msg.desc = desc;
+ msg.iov_count = count;
+ msg.addr = src_addr;
+ msg.context = context;
+
+ return fi_ibv_srq_ep_recvmsg(ep, &msg, 0);
+}
+
+static struct fi_ops_msg fi_ibv_srq_msg_ops = {
+ .size = sizeof(struct fi_ops_msg),
+ .recv = fi_ibv_srq_ep_recv,
+ .recvv = fi_ibv_srq_ep_recvv,
+ .recvmsg = fi_ibv_srq_ep_recvmsg,
+ .send = fi_no_msg_send,
+ .sendv = fi_no_msg_sendv,
+ .sendmsg = fi_no_msg_sendmsg,
+ .inject = fi_no_msg_inject,
+ .senddata = fi_no_msg_senddata,
+ .injectdata = fi_no_msg_injectdata,
+};
+
+int fi_ibv_srq_close(fid_t fid)
+{
+ struct fi_ibv_srq_ep *ep;
+ int rc;
+
+ ep = container_of(fid, struct fi_ibv_srq_ep, ep_fid.fid);
+
+ rc = ibv_destroy_srq(ep->srq);
+ if (rc)
+ FI_WARN(&fi_ibv_prov, FI_LOG_EP_CTRL, "Cannot destroy SRQ rc=%d\n", rc);
+
+ free(ep);
+
+ return 0;
+}
+
+static struct fi_ops fi_ibv_srq_ep_ops = {
+ .size = sizeof(struct fi_ops),
+ .close = fi_ibv_srq_close,
+ .bind = fi_no_bind,
+ .control = fi_no_control,
+ .ops_open = fi_no_ops_open,
+};
+
+
+int fi_ibv_srq_context(struct fid_domain *domain, struct fi_rx_attr *attr,
+ struct fid_ep **rx_ep, void *context)
+{
+ struct ibv_srq_init_attr srq_init_attr = {};
+ struct fi_ibv_domain *dom;
+ struct fi_ibv_srq_ep *_rx_ep;
+
+ if (!domain)
+ return -FI_EINVAL;
+
+ _rx_ep = calloc(1, sizeof *_rx_ep);
+ if (!_rx_ep)
+ return -FI_ENOMEM;
+
+ dom = container_of(domain, struct fi_ibv_domain, domain_fid);
+
+ _rx_ep->ep_fid.fid.fclass = FI_CLASS_SRX_CTX;
+ _rx_ep->ep_fid.fid.context = context;
+ _rx_ep->ep_fid.fid.ops = &fi_ibv_srq_ep_ops;
+ _rx_ep->ep_fid.ops = &fi_ibv_srq_ep_base_ops;
+ _rx_ep->ep_fid.msg = &fi_ibv_srq_msg_ops;
+ _rx_ep->ep_fid.cm = &fi_ibv_srq_cm_ops;
+ _rx_ep->ep_fid.rma = &fi_ibv_srq_rma_ops;
+ _rx_ep->ep_fid.atomic = &fi_ibv_srq_atomic_ops;
+
+ srq_init_attr.attr.max_wr = attr->size;
+ srq_init_attr.attr.max_sge = attr->iov_limit;
+
+ _rx_ep->srq = ibv_create_srq(dom->pd, &srq_init_attr);
+ if (!_rx_ep->srq) {
+ free(_rx_ep);
+ return -errno;
+ }
+
+ *rx_ep = &_rx_ep->ep_fid;
+
+ return 0;
+}
+
diff --git a/src/common.c b/src/common.c
index 21bacf8..b3718af 100644
--- a/src/common.c
+++ b/src/common.c
@@ -47,46 +47,15 @@
#include <sys/time.h>
#include <fi_signal.h>
+#include <rdma/providers/fi_prov.h>
#include <rdma/fi_errno.h>
#include <fi.h>
-
-int fi_wait_cond(pthread_cond_t *cond, pthread_mutex_t *mut, int timeout)
-{
- struct timespec ts;
-
- if (timeout < 0)
- return pthread_cond_wait(cond, mut);
-
- clock_gettime(CLOCK_REALTIME, &ts);
- ts.tv_sec += timeout / 1000;
- ts.tv_nsec += (timeout % 1000) * 1000000;
- return pthread_cond_timedwait(cond, mut, &ts);
-}
-
-int fi_read_file(const char *dir, const char *file, char *buf, size_t size)
-{
- char *path;
- int fd, len;
-
- if (asprintf(&path, "%s/%s", dir, file) < 0)
- return -1;
-
- fd = open(path, O_RDONLY);
- if (fd < 0) {
- free(path);
- return -1;
- }
-
- len = read(fd, buf, size);
- close(fd);
- free(path);
-
- if (len > 0 && buf[len - 1] == '\n')
- buf[--len] = '\0';
-
- return len;
-}
+struct fi_provider core_prov = {
+ .name = "core",
+ .version = 1,
+ .fi_version = FI_VERSION(FI_MAJOR_VERSION, FI_MINOR_VERSION)
+};
int fi_poll_fd(int fd, int timeout)
{
@@ -135,7 +104,7 @@ size_t fi_datatype_size(enum fi_datatype datatype)
return fi_datatype_size_table[datatype];
}
-int fi_send_allowed(uint64_t caps)
+int ofi_send_allowed(uint64_t caps)
{
if (caps & FI_MSG ||
caps & FI_TAGGED) {
@@ -149,7 +118,7 @@ int fi_send_allowed(uint64_t caps)
return 0;
}
-int fi_recv_allowed(uint64_t caps)
+int ofi_recv_allowed(uint64_t caps)
{
if (caps & FI_MSG ||
caps & FI_TAGGED) {
@@ -163,7 +132,7 @@ int fi_recv_allowed(uint64_t caps)
return 0;
}
-int fi_rma_initiate_allowed(uint64_t caps)
+int ofi_rma_initiate_allowed(uint64_t caps)
{
if (caps & FI_RMA ||
caps & FI_ATOMICS) {
@@ -179,7 +148,7 @@ int fi_rma_initiate_allowed(uint64_t caps)
return 0;
}
-int fi_rma_target_allowed(uint64_t caps)
+int ofi_rma_target_allowed(uint64_t caps)
{
if (caps & FI_RMA ||
caps & FI_ATOMICS) {
@@ -195,6 +164,37 @@ int fi_rma_target_allowed(uint64_t caps)
return 0;
}
+int ofi_ep_bind_valid(struct fi_provider *prov, struct fid *bfid, uint64_t flags)
+{
+ if (!bfid) {
+ FI_WARN(prov, FI_LOG_EP_CTRL, "NULL bind fid\n");
+ return -FI_EINVAL;
+ }
+
+ switch (bfid->fclass) {
+ case FI_CLASS_CQ:
+ if (flags & ~(FI_TRANSMIT | FI_RECV | FI_SELECTIVE_COMPLETION)) {
+ FI_WARN(prov, FI_LOG_EP_CTRL, "invalid CQ flags\n");
+ return -FI_EBADFLAGS;
+ }
+ break;
+ case FI_CLASS_CNTR:
+ if (flags & ~(FI_SEND | FI_RECV | FI_READ | FI_WRITE |
+ FI_REMOTE_READ | FI_REMOTE_WRITE)) {
+ FI_WARN(prov, FI_LOG_EP_CTRL, "invalid cntr flags\n");
+ return -FI_EBADFLAGS;
+ }
+ break;
+ default:
+ if (flags) {
+ FI_WARN(prov, FI_LOG_EP_CTRL, "invalid bind flags\n");
+ return -FI_EBADFLAGS;
+ }
+ break;
+ }
+ return FI_SUCCESS;
+}
+
uint64_t fi_gettime_ms(void)
{
struct timeval now;
@@ -203,19 +203,12 @@ uint64_t fi_gettime_ms(void)
return now.tv_sec * 1000 + now.tv_usec / 1000;
}
-int fi_fd_nonblock(int fd)
+uint64_t fi_gettime_us(void)
{
- long flags = 0;
-
- flags = fcntl(fd, F_GETFL);
- if (flags < 0) {
- return -errno;
- }
-
- if(fcntl(fd, F_SETFL, flags | O_NONBLOCK))
- return -errno;
+ struct timeval now;
- return 0;
+ gettimeofday(&now, NULL);
+ return now.tv_sec * 1000000 + now.tv_usec;
}
#ifndef HAVE_EPOLL
diff --git a/src/enosys.c b/src/enosys.c
index efdb83d..1265a7f 100644
--- a/src/enosys.c
+++ b/src/enosys.c
@@ -30,6 +30,9 @@
* SOFTWARE.
*/
+#include <pthread.h>
+#include <stdio.h>
+#include <rdma/fabric.h>
#include <rdma/fi_errno.h>
#include "fi_enosys.h"
@@ -54,7 +57,7 @@ int fi_no_ops_open(struct fid *fid, const char *name,
/*
* struct fi_ops_fabric
*/
-int fi_no_domain(struct fid_fabric *fabric, struct fi_domain_attr *attr,
+int fi_no_domain(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **dom, void *context)
{
return -FI_ENOSYS;
diff --git a/src/fabric.c b/src/fabric.c
index e533653..95f874e 100644
--- a/src/fabric.c
+++ b/src/fabric.c
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2006-2015 Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2006-2016 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2013 Intel Corp., Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -43,7 +43,6 @@
#include <rdma/fi_errno.h>
#include "fi.h"
#include "prov.h"
-#include <rdma/fi_log.h>
#ifdef HAVE_LIBDL
#include <dlfcn.h>
@@ -58,18 +57,11 @@ struct fi_prov {
static struct fi_prov *fi_getprov(const char *prov_name);
static struct fi_prov *prov_head, *prov_tail;
-int init = 0;
-static pthread_mutex_t ini_lock = PTHREAD_MUTEX_INITIALIZER;
+int ofi_init = 0;
+pthread_mutex_t ofi_ini_lock = PTHREAD_MUTEX_INITIALIZER;
static struct fi_filter prov_filter;
-struct fi_provider core_prov = {
- .name = "core",
- .version = 1,
- .fi_version = FI_VERSION(FI_MAJOR_VERSION, FI_MINOR_VERSION)
-};
-
-
static int fi_find_name(char **names, const char *name)
{
int i;
@@ -122,8 +114,11 @@ static int fi_register_provider(struct fi_provider *provider, void *dlhandle)
"registering provider: %s (%d.%d)\n", provider->name,
FI_MAJOR(provider->version), FI_MINOR(provider->version));
- if (FI_MAJOR(provider->fi_version) != FI_MAJOR_VERSION ||
- FI_MINOR(provider->fi_version) != FI_MINOR_VERSION) {
+ /* The current core implementation is not backward compatible
+ * with providers that support a release earlier than v1.3.
+ * See commit 0f4b6651.
+ */
+ if (provider->fi_version < FI_VERSION(1, 3)) {
FI_INFO(&core_prov, FI_LOG_CORE,
"provider has unsupported FI version (provider %d.%d != libfabric %d.%d); ignoring\n",
FI_MAJOR(provider->fi_version),
@@ -345,17 +340,23 @@ void fi_ini(void)
{
char *param_val = NULL;
- pthread_mutex_lock(&ini_lock);
+ pthread_mutex_lock(&ofi_ini_lock);
- if (init)
+ if (ofi_init)
goto unlock;
fi_param_init();
fi_log_init();
fi_util_init();
+ ofi_osd_init();
fi_param_define(NULL, "provider", FI_PARAM_STRING,
"Only use specified provider (default: all available)");
+ fi_param_define(NULL, "fork_unsafe", FI_PARAM_BOOL,
+ "Whether use of fork() may be unsafe for some providers"
+ " (default: no). Setting this to yes could improve"
+ " performance at the expense of making fork() potentially"
+ " unsafe");
fi_param_get_str(NULL, "provider", ¶m_val);
fi_create_filter(&prov_filter, param_val);
@@ -389,28 +390,34 @@ void fi_ini(void)
libdl_done:
#endif
- fi_register_provider(PSM_INIT, NULL);
fi_register_provider(PSM2_INIT, NULL);
+ fi_register_provider(PSM_INIT, NULL);
fi_register_provider(USNIC_INIT, NULL);
- fi_register_provider(MXM_INIT, NULL);
fi_register_provider(VERBS_INIT, NULL);
+ fi_register_provider(MXM_INIT, NULL);
fi_register_provider(GNI_INIT, NULL);
+ /* fi_register_provider(RXM_INIT, NULL); */
+ /* fi_register_provider(RXD_INIT, NULL); */
+
/* Initialize the socket(s) provider last. This will result in
it being the least preferred provider. */
fi_register_provider(UDP_INIT, NULL);
fi_register_provider(SOCKETS_INIT, NULL);
+ /* Before you add ANYTHING here, read the comment above!!! */
+
+ /* Seriously, read it! */
- init = 1;
+ ofi_init = 1;
unlock:
- pthread_mutex_unlock(&ini_lock);
+ pthread_mutex_unlock(&ofi_ini_lock);
}
-static void __attribute__((destructor)) fi_fini(void)
+FI_DESTRUCTOR(fi_fini(void))
{
struct fi_prov *prov;
- if (!init)
+ if (!ofi_init)
return;
while (prov_head) {
@@ -424,6 +431,7 @@ static void __attribute__((destructor)) fi_fini(void)
fi_log_fini();
fi_param_fini();
fi_util_fini();
+ ofi_osd_fini();
}
static struct fi_prov *fi_getprov(const char *prov_name)
@@ -513,9 +521,9 @@ int DEFAULT_SYMVER_PRE(fi_getinfo)(uint32_t version, const char *node, const cha
{
struct fi_prov *prov;
struct fi_info *tail, *cur;
- int ret = -FI_ENODATA;
+ int ret;
- if (!init)
+ if (!ofi_init)
fi_ini();
if (FI_VERSION_LT(fi_version(), version)) {
@@ -566,7 +574,7 @@ int DEFAULT_SYMVER_PRE(fi_getinfo)(uint32_t version, const char *node, const cha
tail->fabric_attr->prov_version = prov->provider->version;
}
- return *info ? 0 : ret;
+ return *info ? 0 : -FI_ENODATA;
}
DEFAULT_SYMVER(fi_getinfo_, fi_getinfo);
@@ -683,7 +691,7 @@ int DEFAULT_SYMVER_PRE(fi_fabric)(struct fi_fabric_attr *attr, struct fid_fabric
if (!attr || !attr->prov_name || !attr->name)
return -FI_EINVAL;
- if (!init)
+ if (!ofi_init)
fi_ini();
prov = fi_getprov(attr->prov_name);
@@ -713,6 +721,7 @@ static const char *const errstr[] = {
[FI_ECRC - FI_ERRNO_OFFSET] = "CRC error",
[FI_ETRUNC - FI_ERRNO_OFFSET] = "Truncation error",
[FI_ENOKEY - FI_ERRNO_OFFSET] = "Required key not available",
+ [FI_ENOAV - FI_ERRNO_OFFSET] = "Missing or unavailable address vector",
};
__attribute__((visibility ("default")))
diff --git a/src/fasthash.c b/src/fasthash.c
index 48e16ce..9252722 100644
--- a/src/fasthash.c
+++ b/src/fasthash.c
@@ -27,10 +27,13 @@
// Compression function for Merkle-Damgard construction.
// This function is generated using the framework provided.
-#define mix(h) ({ \
- (h) ^= (h) >> 23; \
- (h) *= 0x2127599bf4325c37ULL; \
- (h) ^= (h) >> 47; })
+static inline uint64_t mix(uint64_t h)
+{
+ h ^= h >> 23;
+ h *= 0x2127599bf4325c37ULL;
+ h ^= h >> 47;
+ return h;
+}
uint64_t fasthash64(const void *buf, size_t len, uint64_t seed)
{
@@ -70,6 +73,6 @@ uint32_t fasthash32(const void *buf, size_t len, uint32_t seed)
// the following trick converts the 64-bit hashcode to Fermat
// residue, which shall retain information from both the higher
// and lower parts of hashcode.
- uint64_t h = fasthash64(buf, len, seed);
+ uint64_t h = fasthash64(buf, len, seed);
return h - (h >> 32);
}
diff --git a/src/fi_tostr.c b/src/fi_tostr.c
index 4a0dbda..3d79fcf 100644
--- a/src/fi_tostr.c
+++ b/src/fi_tostr.c
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2014-2015 Intel Corp., Inc. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -41,10 +42,10 @@
#include <netinet/in.h>
#include <arpa/inet.h>
+#include "fi.h"
#include <rdma/fabric.h>
#include <rdma/fi_domain.h>
#include <rdma/fi_endpoint.h>
-#include "fi.h"
/* Print fi_info and related structs, enums, OR_able flags, addresses.
*
@@ -227,6 +228,8 @@ static void fi_tostr_protocol(char *buf, uint32_t protocol)
CASEENUMSTR(FI_PROTO_IB_RDM);
CASEENUMSTR(FI_PROTO_IWARP_RDM);
CASEENUMSTR(FI_PROTO_GNI);
+ CASEENUMSTR(FI_PROTO_RXM);
+ CASEENUMSTR(FI_PROTO_RXD);
default:
if (protocol & FI_PROV_SPECIFIC)
strcatf(buf, "Provider specific");
@@ -444,7 +447,7 @@ static void fi_tostr_domain_attr(char *buf, const struct fi_domain_attr *attr,
strcatf(buf, "%s%sdata_progress: ", prefix, TAB);
fi_tostr_progress(buf, attr->data_progress);
strcatf(buf, "\n");
- strcatf(buf, "%s%sresouce_mgmt: ", prefix, TAB);
+ strcatf(buf, "%s%sresource_mgmt: ", prefix, TAB);
fi_tostr_resource_mgmt(buf, attr->resource_mgmt);
strcatf(buf, "\n");
strcatf(buf, "%s%sav_type: ", prefix, TAB);
diff --git a/src/log.c b/src/log.c
index ddd05e1..20741e4 100644
--- a/src/log.c
+++ b/src/log.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2015-2016, Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2015, Intel Corp., Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -37,7 +37,6 @@
#include <stdlib.h>
#include <rdma/fi_errno.h>
-#include <rdma/fi_log.h>
#include "fi.h"
@@ -62,8 +61,6 @@ static const char * const log_levels[] = {
[FI_LOG_MAX] = NULL
};
-extern struct fi_provider core_prov;
-
enum {
FI_LOG_SUBSYS_OFFSET = FI_LOG_MAX,
FI_LOG_PROV_OFFSET = FI_LOG_SUBSYS_OFFSET + FI_LOG_SUBSYS_MAX,
diff --git a/src/unix/osd.c b/src/unix/osd.c
new file mode 100644
index 0000000..6a4971c
--- /dev/null
+++ b/src/unix/osd.c
@@ -0,0 +1,202 @@
+/*
+ * Copyright (c) 2013-2016 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif /* _GNU_SOURCE */
+
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <pthread.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+
+#include "fi.h"
+#include "fi_osd.h"
+#include "fi_file.h"
+
+#include "rdma/fi_errno.h"
+#include "rdma/providers/fi_log.h"
+
+int fi_fd_nonblock(int fd)
+{
+ long flags = 0;
+
+ flags = fcntl(fd, F_GETFL);
+ if (flags < 0) {
+ return -errno;
+ }
+
+ if(fcntl(fd, F_SETFL, flags | O_NONBLOCK))
+ return -errno;
+
+ return 0;
+}
+
+int fi_wait_cond(pthread_cond_t *cond, pthread_mutex_t *mut, int timeout)
+{
+ struct timespec ts;
+
+ if (timeout < 0)
+ return pthread_cond_wait(cond, mut);
+
+ clock_gettime(CLOCK_REALTIME, &ts);
+ ts.tv_sec += timeout / 1000;
+ ts.tv_nsec += (timeout % 1000) * 1000000;
+ return pthread_cond_timedwait(cond, mut, &ts);
+}
+
+int ofi_shm_map(struct util_shm *shm, const char *name, size_t size,
+ int readonly, void **mapped)
+{
+ char *fname = 0;
+ int i, ret = FI_SUCCESS;
+ int flags = O_RDWR | (readonly ? 0 : O_CREAT);
+ struct stat mapstat;
+
+ *mapped = MAP_FAILED;
+ memset(shm, 0, sizeof(*shm));
+
+ fname = calloc(1, strlen(name) + 2); /* '/' + %s + trailing 0 */
+ if (!fname)
+ return -FI_ENOMEM;
+
+ strcpy(fname, "/");
+ strcat(fname, name);
+ shm->name = fname;
+
+ for (i = 0; i < strlen(fname); i++) {
+ if (fname[i] == ' ')
+ fname[i] = '_';
+ }
+
+ FI_DBG(&core_prov, FI_LOG_CORE,
+ "Creating shm segment :%s (size: %lu)\n", fname, size);
+
+ shm->shared_fd = shm_open(fname, flags, S_IRUSR | S_IWUSR);
+ if (shm->shared_fd < 0) {
+ FI_WARN(&core_prov, FI_LOG_CORE, "shm_open failed\n");
+ ret = -FI_EINVAL;
+ goto failed;
+ }
+
+ if (fstat(shm->shared_fd, &mapstat)) {
+ FI_WARN(&core_prov, FI_LOG_CORE, "failed to do fstat: %s\n",
+ strerror(errno));
+ ret = -FI_EINVAL;
+ goto failed;
+ }
+
+ if (mapstat.st_size == 0) {
+ if (ftruncate(shm->shared_fd, size)) {
+ FI_WARN(&core_prov, FI_LOG_CORE,
+ "ftruncate failed: %s\n", strerror(errno));
+ ret = -FI_EINVAL;
+ goto failed;
+ }
+ } else if (mapstat.st_size < size) {
+ FI_WARN(&core_prov, FI_LOG_CORE, "shm file too small\n");
+ ret = -FI_EINVAL;
+ goto failed;
+ }
+
+ shm->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
+ MAP_SHARED, shm->shared_fd, 0);
+ if (shm->ptr == MAP_FAILED) {
+ FI_WARN(&core_prov, FI_LOG_CORE,
+ "mmap failed: %s\n", strerror(errno));
+ ret = -FI_EINVAL;
+ goto failed;
+ }
+
+ *mapped = shm->ptr;
+ shm->size = size;
+ return ret;
+
+failed:
+ if (shm->shared_fd >= 0) {
+ close(shm->shared_fd);
+ shm_unlink(fname);
+ }
+ if (fname)
+ free(fname);
+ memset(shm, 0, sizeof(*shm));
+ return ret;
+}
+
+int ofi_shm_unmap(struct util_shm* shm)
+{
+ if (shm->ptr && shm->ptr != MAP_FAILED) {
+ if (munmap(shm->ptr, shm->size)) {
+ FI_WARN(&core_prov, FI_LOG_CORE,
+ "munmap failed: %s\n", strerror(errno));
+ }
+ }
+
+ if (shm->shared_fd)
+ close(shm->shared_fd);
+ if (shm->name) {
+ shm_unlink(shm->name);
+ free((void*)shm->name);
+ }
+ memset(shm, 0, sizeof(*shm));
+ return FI_SUCCESS;
+}
+
+int fi_read_file(const char *dir, const char *file, char *buf, size_t size)
+{
+ char *path;
+ int fd, len;
+
+ if (asprintf(&path, "%s/%s", dir, file) < 0)
+ return -1;
+
+ fd = open(path, O_RDONLY);
+ if (fd < 0) {
+ free(path);
+ return -1;
+ }
+
+ len = read(fd, buf, size);
+ close(fd);
+ free(path);
+
+ if (len > 0 && buf[len - 1] == '\n')
+ buf[--len] = '\0';
+
+ return len;
+}
+
+
diff --git a/src/var.c b/src/var.c
index 70ac58f..2b27f03 100644
--- a/src/var.c
+++ b/src/var.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2015-2016, Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2015, Intel Corp., Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -38,17 +38,12 @@
#include <ctype.h>
#include <rdma/fi_errno.h>
-#include <rdma/fi_prov.h>
-#include <rdma/fi_log.h>
#include "fi.h"
#include "fi_list.h"
-/* When given a NULL provider pointer, use core for logging and settings. */
-extern struct fi_provider core_prov;
-
-extern int init;
+extern int ofi_init;
extern void fi_ini();
struct fi_param_entry {
@@ -92,7 +87,7 @@ int DEFAULT_SYMVER_PRE(fi_getparams)(struct fi_param **params, int *count)
int cnt, i;
char *tmp;
- if (!init)
+ if (!ofi_init)
fi_ini();
for (entry = param_list.next, cnt = 0; entry != ¶m_list;
diff --git a/util/info.c b/util/info.c
index 8d79da3..65846b0 100644
--- a/util/info.c
+++ b/util/info.c
@@ -46,13 +46,16 @@ static int verbose = 0, env = 0;
/* options and matching help strings need to be kept in sync */
static const struct option longopts[] = {
+ {"help", no_argument, NULL, 'h'},
{"node", required_argument, NULL, 'n'},
- {"port", required_argument, NULL, 'p'},
+ {"port", required_argument, NULL, 'P'},
{"caps", required_argument, NULL, 'c'},
{"mode", required_argument, NULL, 'm'},
{"ep_type", required_argument, NULL, 't'},
+ {"domain", required_argument, NULL, 'd'},
+ {"fabric", required_argument, NULL, 'f'},
{"addr_format", required_argument, NULL, 'a'},
- {"provider", required_argument, NULL, 'f'},
+ {"provider", required_argument, NULL, 'p'},
{"env", no_argument, NULL, 'e'},
{"list", no_argument, NULL, 'l'},
{"verbose", no_argument, NULL, 'v'},
@@ -61,11 +64,14 @@ static const struct option longopts[] = {
};
static const char *help_strings[][2] = {
+ {"", "\t\tdisplay this help and exit"},
{"NAME", "\t\tnode name or address"},
{"PNUM", "\t\tport number"},
{"CAP1|CAP2..", "\tone or more capabilities: FI_MSG|FI_RMA..."},
{"MOD1|MOD2..", "\tone or more modes, default all modes"},
{"EPTYPE", "\t\tspecify single endpoint type: FI_EP_MSG, FI_EP_DGRAM..."},
+ {"DOMAIN", "\t\tspecify the domain name"},
+ {"FABRIC", "\t\tspecify the fabric name"},
{"FMT", "\t\tspecify accepted address format: FI_FORMAT_UNSPEC, FI_SOCKADDR..."},
{"PROV", "\t\tspecify provider explicitly"},
{"", "\t\tprint libfabric environment variables"},
@@ -92,10 +98,15 @@ static void usage(void)
help_strings[i][1]);
}
-#define ORCASE(SYM) \
- do { if (strcmp(#SYM, inputstr) == 0) return SYM; } while (0);
+#define ORCASE(SYM) \
+ do { \
+ if (strcasecmp(#SYM, inputstr) == 0) { \
+ *value = SYM; \
+ return 0; \
+ } \
+ } while (0)
-static uint64_t str2cap(char *inputstr)
+static int str2cap(char *inputstr, uint64_t *value)
{
ORCASE(FI_MSG);
ORCASE(FI_RMA);
@@ -127,10 +138,12 @@ static uint64_t str2cap(char *inputstr)
ORCASE(FI_SOURCE);
ORCASE(FI_DIRECTED_RECV);
- return 0;
+ fprintf(stderr, "error: Unrecognized capability: %s\n", inputstr);
+
+ return -EINVAL;
}
-static uint64_t str2mode(char *inputstr)
+static int str2mode(char *inputstr, uint64_t *value)
{
ORCASE(FI_CONTEXT);
ORCASE(FI_LOCAL_MR);
@@ -138,21 +151,24 @@ static uint64_t str2mode(char *inputstr)
ORCASE(FI_ASYNC_IOV);
ORCASE(FI_RX_CQ_DATA);
- return 0;
+ fprintf(stderr, "error: Unrecognized mode: %s\n", inputstr);
+
+ return -EINVAL;
}
-static enum fi_ep_type str2ep_type(char *inputstr)
+static int str2ep_type(char *inputstr, enum fi_ep_type *value)
{
ORCASE(FI_EP_UNSPEC);
ORCASE(FI_EP_MSG);
ORCASE(FI_EP_DGRAM);
ORCASE(FI_EP_RDM);
- /* probably not the right thing to do? */
- return FI_EP_UNSPEC;
+ fprintf(stderr, "error: Unrecognized endpoint type: %s\n", inputstr);
+
+ return -EINVAL;
}
-static uint32_t str2addr_format(char *inputstr)
+static int str2addr_format(char *inputstr, uint32_t *value)
{
ORCASE(FI_FORMAT_UNSPEC);
ORCASE(FI_SOCKADDR);
@@ -161,18 +177,28 @@ static uint32_t str2addr_format(char *inputstr)
ORCASE(FI_SOCKADDR_IB);
ORCASE(FI_ADDR_PSMX);
- return FI_FORMAT_UNSPEC;
+ fprintf(stderr, "error: Unrecognized address format: %s\n", inputstr);
+
+ return -EINVAL;
}
-static uint64_t tokparse(char *caps, uint64_t (*str2flag) (char *inputstr))
+static int tokparse(char *caps,
+ int (*str2flag)(char *, uint64_t *),
+ uint64_t *flags)
{
- uint64_t flags = 0;
+ uint64_t value;
char *tok;
+ int ret;
+
+ for (tok = strtok(caps, "|"); tok != NULL; tok = strtok(NULL, "|")) {
+ ret = str2flag(tok, &value);
+ if (ret)
+ return ret;
- for (tok = strtok(caps, "|"); tok != NULL; tok = strtok(NULL, "|"))
- flags |= str2flag(tok);
+ *flags |= value;
+ }
- return flags;
+ return 0;
}
static const char *param_type(enum fi_param_type type)
@@ -230,7 +256,9 @@ static int print_providers(struct fi_info *info)
static int print_short_info(struct fi_info *info)
{
for (struct fi_info *cur = info; cur; cur = cur->next) {
- printf("%s: %s\n", cur->fabric_attr->prov_name, cur->fabric_attr->name);
+ printf("provider: %s\n", cur->fabric_attr->prov_name);
+ printf(" fabric: %s\n", cur->fabric_attr->name),
+ printf(" domain: %s\n", cur->domain_attr->name),
printf(" version: %d.%d\n", FI_MAJOR(cur->fabric_attr->prov_version),
FI_MINOR(cur->fabric_attr->prov_version));
if (!list_providers) {
@@ -286,7 +314,8 @@ int main(int argc, char **argv)
hints->mode = ~0;
- while ((op = getopt_long(argc, argv, "n:p:c:m:t:a:f:elhv", longopts, &option_index)) != -1) {
+ while ((op = getopt_long(argc, argv, "n:P:c:m:t:a:p:d:f:elhv", longopts,
+ &option_index)) != -1) {
switch (op) {
case 0:
/* If --verbose set a flag, do nothing. */
@@ -295,30 +324,51 @@ int main(int argc, char **argv)
case 'n':
node = optarg;
break;
- case 'p':
+ case 'P':
port = optarg;
break;
case 'c':
- hints->caps = tokparse(optarg, str2cap);
+ ret = tokparse(optarg, str2cap, &hints->caps);
+ if (ret)
+ goto out;
+
use_hints = 1;
break;
case 'm':
- hints->mode = tokparse(optarg, str2mode);
+ hints->mode = 0;
+ ret = tokparse(optarg, str2mode, &hints->mode);
+ if (ret)
+ goto out;
+
use_hints = 1;
break;
case 't':
- hints->ep_attr->type = str2ep_type(optarg);
+ ret = str2ep_type(optarg, &hints->ep_attr->type);
+ if (ret)
+ goto out;
+
use_hints = 1;
break;
case 'a':
- hints->addr_format = str2addr_format(optarg);
+ ret = str2addr_format(optarg, &hints->addr_format);
+ if (ret)
+ goto out;
+
use_hints = 1;
break;
- case 'f':
+ case 'p':
free(hints->fabric_attr->prov_name);
hints->fabric_attr->prov_name = strdup(optarg);
use_hints = 1;
break;
+ case 'd':
+ hints->domain_attr->name = strdup(optarg);
+ use_hints = 1;
+ break;
+ case 'f':
+ hints->fabric_attr->name = strdup(optarg);
+ use_hints = 1;
+ break;
case 'e':
env = 1;
break;
diff --git a/util/pingpong.c b/util/pingpong.c
new file mode 100644
index 0000000..269a969
--- /dev/null
+++ b/util/pingpong.c
@@ -0,0 +1,2208 @@
+/*
+ * Copyright (c) 2013-2015 Intel Corporation. All rights reserved.
+ * Copyright (c) 2014-2016, Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2015 Los Alamos Nat. Security, LLC. All rights reserved.
+ * Copyright (c) 2016 Cray Inc. All rights reserved.
+ *
+ * This software is available to you under the BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <config.h>
+
+#include <time.h>
+#include <unistd.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <assert.h>
+#include <getopt.h>
+#include <inttypes.h>
+#include <netdb.h>
+#include <poll.h>
+#include <limits.h>
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/time.h>
+
+#include <rdma/fabric.h>
+#include <rdma/fi_cm.h>
+#include <rdma/fi_domain.h>
+#include <rdma/fi_endpoint.h>
+#include <rdma/fi_eq.h>
+#include <rdma/fi_errno.h>
+
+#ifndef PP_FIVERSION
+#define PP_FIVERSION FI_VERSION(1, 4)
+#endif
+
+#ifdef __APPLE__
+#include "osx/osd.h"
+#elif defined __FreeBSD__
+#include "freebsd/osd.h"
+#endif
+
+enum precision {
+ NANO = 1,
+ MICRO = 1000,
+ MILLI = 1000000,
+};
+
+enum {
+ PP_OPT_ACTIVE = 1 << 0,
+ PP_OPT_ITER = 1 << 1,
+ PP_OPT_SIZE = 1 << 2,
+ PP_OPT_VERIFY_DATA = 1 << 3,
+};
+
+struct pp_opts {
+ uint16_t src_port;
+ uint16_t dst_port;
+ char *dst_addr;
+ int iterations;
+ int transfer_size;
+ int sizes_enabled;
+ int options;
+};
+
+#define PP_SIZE_MAX_POWER_TWO 22
+#define PP_MAX_DATA_MSG \
+ ((1 << PP_SIZE_MAX_POWER_TWO) + (1 << (PP_SIZE_MAX_POWER_TWO - 1)))
+
+#define PP_STR_LEN 32
+#define PP_MAX_CTRL_MSG 64
+#define PP_CTRL_BUF_LEN 64
+#define PP_MR_KEY 0xC0DE
+
+#define INTEG_SEED 7
+#define PP_ENABLE_ALL (~0)
+#define PP_DEFAULT_SIZE (1 << 0)
+
+#define PP_MSG_CHECK_PORT_OK "port ok"
+#define PP_MSG_LEN_PORT 5
+#define PP_MSG_CHECK_CNT_OK "cnt ok"
+#define PP_MSG_LEN_CNT 10
+#define PP_MSG_SYNC_Q "q"
+#define PP_MSG_SYNC_A "a"
+
+#define PP_PRINTERR(call, retv) \
+ fprintf(stderr, "%s(): %s:%-4d, ret=%d (%s)\n", call, __FILE__, \
+ __LINE__, (int)retv, fi_strerror((int) -retv))
+
+#define PP_ERR(fmt, ...) \
+ fprintf(stderr, "[%s] %s:%-4d: " fmt "\n", "error", __FILE__, \
+ __LINE__, ##__VA_ARGS__)
+
+int pp_debug;
+
+#define PP_DEBUG(fmt, ...) \
+ do { \
+ if (pp_debug) { \
+ fprintf(stderr, "[%s] %s:%-4d: " fmt, "debug", \
+ __FILE__, __LINE__, ##__VA_ARGS__); \
+ } \
+ } while (0)
+
+#define PP_CLOSE_FID(fd) \
+ do { \
+ int ret; \
+ if ((fd)) { \
+ ret = fi_close(&(fd)->fid); \
+ if (ret) \
+ PP_ERR("fi_close (%d) fid %d", ret, \
+ (int)(fd)->fid.fclass); \
+ fd = NULL; \
+ } \
+ } while (0)
+
+#ifndef MAX
+#define MAX(a, b) \
+ ({ \
+ typeof(a) _a = (a); \
+ typeof(b) _b = (b); \
+ _a > _b ? _a : _b; \
+ })
+#endif
+
+#ifndef MIN
+#define MIN(a, b) \
+ ({ \
+ typeof(a) _a = (a); \
+ typeof(b) _b = (b); \
+ _a < _b ? _a : _b; \
+ })
+#endif
+
+struct ct_pingpong {
+ struct fi_info *fi_pep, *fi, *hints;
+ struct fid_fabric *fabric;
+ struct fid_domain *domain;
+ struct fid_pep *pep;
+ struct fid_ep *ep;
+ struct fid_cq *txcq, *rxcq;
+ struct fid_mr *mr;
+ struct fid_av *av;
+ struct fid_eq *eq;
+
+ struct fid_mr no_mr;
+ struct fi_context tx_ctx, rx_ctx;
+ uint64_t remote_cq_data;
+
+ uint64_t tx_seq, rx_seq, tx_cq_cntr, rx_cq_cntr;
+
+ fi_addr_t remote_fi_addr;
+ void *buf, *tx_buf, *rx_buf;
+ size_t buf_size, tx_size, rx_size;
+
+ int timeout;
+ struct timespec start, end;
+
+ struct fi_av_attr av_attr;
+ struct fi_eq_attr eq_attr;
+ struct fi_cq_attr cq_attr;
+ struct pp_opts opts;
+
+ long cnt_ack_msg;
+
+ int ctrl_connfd;
+ char ctrl_buf[PP_CTRL_BUF_LEN + 1];
+ char rem_name[PP_MAX_CTRL_MSG];
+};
+
+static const char integ_alphabet[] =
+ "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+/* Size does not include trailing new line */
+static const int integ_alphabet_length =
+ (sizeof(integ_alphabet) / sizeof(*integ_alphabet)) - 1;
+
+/*******************************************************************************
+ * Compatibility methods
+ ******************************************************************************/
+
+#ifdef __APPLE__
+int clock_gettime(clockid_t clk_id, struct timespec *tp)
+{
+ int retval;
+ struct timeval tv;
+
+ retval = gettimeofday(&tv, NULL);
+
+ tp->tv_sec = tv.tv_sec;
+ tp->tv_nsec = tv.tv_usec * 1000;
+
+ return retval;
+}
+#endif
+
+/*******************************************************************************
+ * Utils
+ ******************************************************************************/
+
+long parse_ulong(char *str, long max)
+{
+ long ret;
+ char *end;
+
+ errno = 0;
+ ret = strtol(str, &end, 10);
+ if (*end != '\0' || errno != 0) {
+ if (errno == 0)
+ ret = -EINVAL;
+ else
+ ret = -errno;
+ fprintf(stderr, "Error parsing \"%s\": %s\n", str,
+ strerror(-ret));
+ return ret;
+ }
+
+ if ((ret < 0) || (max > 0 && ret > max)) {
+ ret = -ERANGE;
+ fprintf(stderr, "Error parsing \"%s\": %s\n", str,
+ strerror(-ret));
+ return ret;
+ }
+ return ret;
+}
+
+int size_to_count(int size)
+{
+ if (size >= (1 << 20))
+ return 100;
+ else if (size >= (1 << 16))
+ return 1000;
+ else
+ return 10000;
+}
+
+void pp_banner_fabric_info(struct ct_pingpong *ct)
+{
+ PP_DEBUG(
+ "Running pingpong test with the %s endpoint trough a %s provider\n",
+ fi_tostr(&ct->fi->ep_attr->type, FI_TYPE_EP_TYPE),
+ ct->fi->fabric_attr->prov_name);
+ PP_DEBUG(" * Fabric Attributes:\n");
+ PP_DEBUG(" - %-20s: %s\n", "name", ct->fi->fabric_attr->name);
+ PP_DEBUG(" - %-20s: %s\n", "prov_name",
+ ct->fi->fabric_attr->prov_name);
+ PP_DEBUG(" - %-20s: %" PRIu32 "\n", "prov_version",
+ ct->fi->fabric_attr->prov_version);
+ PP_DEBUG(" * Domain Attributes:\n");
+ PP_DEBUG(" - %-20s: %s\n", "name", ct->fi->domain_attr->name);
+ PP_DEBUG(" - %-20s: %zu\n", "cq_cnt", ct->fi->domain_attr->cq_cnt);
+ PP_DEBUG(" - %-20s: %zu\n", "cq_data_size",
+ ct->fi->domain_attr->cq_data_size);
+ PP_DEBUG(" - %-20s: %zu\n", "ep_cnt", ct->fi->domain_attr->ep_cnt);
+ PP_DEBUG(" * Endpoint Attributes:\n");
+ PP_DEBUG(" - %-20s: %s\n", "type",
+ fi_tostr(&ct->fi->ep_attr->type, FI_TYPE_EP_TYPE));
+ PP_DEBUG(" - %-20s: %" PRIu32 "\n", "protocol",
+ ct->fi->ep_attr->protocol);
+ PP_DEBUG(" - %-20s: %" PRIu32 "\n", "protocol_version",
+ ct->fi->ep_attr->protocol_version);
+ PP_DEBUG(" - %-20s: %zu\n", "max_msg_size",
+ ct->fi->ep_attr->max_msg_size);
+ PP_DEBUG(" - %-20s: %zu\n", "max_order_raw_size",
+ ct->fi->ep_attr->max_order_raw_size);
+}
+
+void pp_banner_options(struct ct_pingpong *ct)
+{
+ char size_msg[50];
+ char iter_msg[50];
+ struct pp_opts opts = ct->opts;
+
+ if ((opts.dst_addr == NULL) || (opts.dst_addr[0] == '\0'))
+ opts.dst_addr = "None";
+
+ if (opts.sizes_enabled == PP_ENABLE_ALL)
+ snprintf(size_msg, 50, "%s", "All sizes");
+ else if (opts.options & PP_OPT_SIZE)
+ snprintf(size_msg, 50, "selected size = %d",
+ opts.transfer_size);
+
+ if (opts.options & PP_OPT_ITER)
+ snprintf(iter_msg, 50, "selected iterations: %d",
+ opts.iterations);
+ else {
+ opts.iterations = size_to_count(opts.transfer_size);
+ snprintf(iter_msg, 50, "default iterations: %d",
+ opts.iterations);
+ }
+
+ PP_DEBUG(" * PingPong options:\n");
+ PP_DEBUG(" - %-20s: [%" PRIu16 "]\n", "src_port", opts.src_port);
+ PP_DEBUG(" - %-20s: [%s]\n", "dst_addr", opts.dst_addr);
+ PP_DEBUG(" - %-20s: [%" PRIu16 "]\n", "dst_port", opts.dst_port);
+ PP_DEBUG(" - %-20s: %s\n", "sizes_enabled", size_msg);
+ PP_DEBUG(" - %-20s: %s\n", "iterations", iter_msg);
+ if (ct->hints->fabric_attr->prov_name)
+ PP_DEBUG(" - %-20s: %s\n", "provider",
+ ct->hints->fabric_attr->prov_name);
+ if (ct->hints->domain_attr->name)
+ PP_DEBUG(" - %-20s: %s\n", "domain",
+ ct->hints->domain_attr->name);
+}
+
+/*******************************************************************************
+ * Control Messaging
+ ******************************************************************************/
+
+int pp_getaddrinfo(char *name, uint16_t port, struct addrinfo **results)
+{
+ int ret;
+ const char *err_msg;
+ char port_s[6];
+
+ struct addrinfo hints = {
+ .ai_family = AF_INET, /* IPv4 */
+ .ai_socktype = SOCK_STREAM, /* TCP socket */
+ .ai_protocol = IPPROTO_TCP, /* Any protocol */
+ .ai_flags = AI_NUMERICSERV /* numeric port is used */
+ };
+
+ snprintf(port_s, 6, "%" PRIu16, port);
+
+ ret = getaddrinfo(name, port_s, &hints, results);
+ if (ret != 0) {
+ err_msg = gai_strerror(ret);
+ PP_ERR("getaddrinfo : %s", err_msg);
+ ret = -EXIT_FAILURE;
+ goto out;
+ }
+ ret = EXIT_SUCCESS;
+
+out:
+ return ret;
+}
+
+static int pp_ctrl_init_client(struct ct_pingpong *ct)
+{
+ struct sockaddr_in in_addr = {0};
+ struct addrinfo *results;
+ struct addrinfo *rp;
+ int errno_save;
+ int ret;
+
+ ret = pp_getaddrinfo(ct->opts.dst_addr, ct->opts.dst_port, &results);
+ if (ret)
+ return ret;
+
+ if (!results) {
+ PP_ERR("getaddrinfo returned NULL list");
+ return -EXIT_FAILURE;
+ }
+
+ for (rp = results; rp; rp = rp->ai_next) {
+ ct->ctrl_connfd = socket(rp->ai_family, rp->ai_socktype,
+ rp->ai_protocol);
+ if (ct->ctrl_connfd == -1) {
+ errno_save = errno;
+ continue;
+ }
+
+ if (ct->opts.src_port != 0) {
+ in_addr.sin_family = AF_INET;
+ in_addr.sin_port = htons(ct->opts.src_port);
+ in_addr.sin_addr.s_addr = htonl(INADDR_ANY);
+
+ ret =
+ bind(ct->ctrl_connfd, (struct sockaddr *)&in_addr,
+ sizeof(in_addr));
+ if (ret == -1) {
+ errno_save = errno;
+ close(ct->ctrl_connfd);
+ continue;
+ }
+ }
+
+ ret = connect(ct->ctrl_connfd, rp->ai_addr, rp->ai_addrlen);
+ if (ret != -1)
+ break;
+
+ errno_save = errno;
+ close(ct->ctrl_connfd);
+ }
+
+ if (!rp || ret == -1) {
+ ret = -errno_save;
+ ct->ctrl_connfd = -1;
+ PP_ERR("failed to connect: %s", strerror(errno_save));
+ } else {
+ PP_DEBUG("CLIENT: connected\n");
+ }
+
+ freeaddrinfo(results);
+
+ return ret;
+}
+
+static int pp_ctrl_init_server(struct ct_pingpong *ct)
+{
+ struct sockaddr_in ctrl_addr = {0};
+ int optval = 1;
+ int listenfd;
+ int ret;
+
+ listenfd = socket(AF_INET, SOCK_STREAM, 0);
+ if (listenfd == -1) {
+ ret = -errno;
+ PP_PRINTERR("socket", ret);
+ return ret;
+ }
+
+ ret = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR,
+ &optval, sizeof(optval));
+ if (ret == -1) {
+ ret = -errno;
+ PP_PRINTERR("setsockopt(SO_REUSEADDR)", ret);
+ goto fail_close_socket;
+ }
+
+ ctrl_addr.sin_family = AF_INET;
+ ctrl_addr.sin_port = htons(ct->opts.src_port);
+ ctrl_addr.sin_addr.s_addr = htonl(INADDR_ANY);
+
+ ret = bind(listenfd, (struct sockaddr *)&ctrl_addr,
+ sizeof(ctrl_addr));
+ if (ret == -1) {
+ ret = -errno;
+ PP_PRINTERR("bind", ret);
+ goto fail_close_socket;
+ }
+
+ ret = listen(listenfd, 10);
+ if (ret == -1) {
+ ret = -errno;
+ PP_PRINTERR("listen", ret);
+ goto fail_close_socket;
+ }
+
+ PP_DEBUG("SERVER: waiting for connection\n");
+
+ ct->ctrl_connfd = accept(listenfd, NULL, NULL);
+ if (ct->ctrl_connfd == -1) {
+ ret = -errno;
+ PP_PRINTERR("accept", ret);
+ goto fail_close_socket;
+ }
+
+ close(listenfd);
+
+ PP_DEBUG("SERVER: connected\n");
+
+ return ret;
+
+fail_close_socket:
+ if (ct->ctrl_connfd != -1) {
+ close(ct->ctrl_connfd);
+ ct->ctrl_connfd = -1;
+ }
+
+ if (listenfd != -1)
+ close(listenfd);
+
+ return ret;
+}
+
+int pp_ctrl_init(struct ct_pingpong *ct)
+{
+ const uint32_t default_ctrl = 47592;
+ struct timeval tv = {
+ .tv_sec = 5
+ };
+ int ret;
+
+ PP_DEBUG("Initializing control messages\n");
+
+ if (ct->opts.dst_addr) {
+ if (ct->opts.dst_port == 0)
+ ct->opts.dst_port = default_ctrl;
+ ret = pp_ctrl_init_client(ct);
+ } else {
+ if (ct->opts.src_port == 0)
+ ct->opts.src_port = default_ctrl;
+ ret = pp_ctrl_init_server(ct);
+ }
+
+ if (ret)
+ return ret;
+
+ ret = setsockopt(ct->ctrl_connfd, SOL_SOCKET, SO_RCVTIMEO, &tv,
+ sizeof(struct timeval));
+ if (ret == -1) {
+ ret = -errno;
+ PP_PRINTERR("setsockopt(SO_RCVTIMEO)", ret);
+ return ret;
+ }
+
+ PP_DEBUG("Control messages initialized\n");
+
+ return ret;
+}
+
+int pp_ctrl_send(struct ct_pingpong *ct, char *buf, size_t size)
+{
+ int ret, err;
+
+ ret = send(ct->ctrl_connfd, buf, size, 0);
+ if (ret < 0) {
+ err = -errno;
+ PP_PRINTERR("ctrl/send", err);
+ return err;
+ }
+ if (ret == 0) {
+ err = -ECONNABORTED;
+ PP_ERR("ctrl/read: no data or remote connection closed");
+ return err;
+ }
+
+ return ret;
+}
+
+int pp_ctrl_recv(struct ct_pingpong *ct, char *buf, size_t size)
+{
+ int ret, err;
+
+ do {
+ PP_DEBUG("receiving\n");
+ ret = recv(ct->ctrl_connfd, buf, size, 0);
+ } while (ret == -1 && errno == EAGAIN);
+ if (ret < 0) {
+ err = -errno;
+ PP_PRINTERR("ctrl/read", err);
+ return err;
+ }
+ if (ret == 0) {
+ err = -ECONNABORTED;
+ PP_ERR("ctrl/read: no data or remote connection closed");
+ return err;
+ }
+
+ return ret;
+}
+
+int pp_send_name(struct ct_pingpong *ct, struct fid *endpoint)
+{
+ char local_name[64];
+ size_t addrlen;
+ uint32_t len;
+ int ret;
+
+ PP_DEBUG("Fetching local address\n");
+
+ addrlen = sizeof(local_name);
+ ret = fi_getname(endpoint, local_name, &addrlen);
+ if (ret) {
+ PP_PRINTERR("fi_getname", ret);
+ return ret;
+ }
+
+ if (addrlen > sizeof(local_name)) {
+ PP_DEBUG("Address exceeds control buffer length\n");
+ return -EMSGSIZE;
+ }
+
+ PP_DEBUG("Sending name length\n");
+ len = htonl(addrlen);
+ ret = pp_ctrl_send(ct, (char *) &len, sizeof(len));
+ if (ret < 0)
+ return ret;
+
+ PP_DEBUG("Sending name\n");
+ ret = pp_ctrl_send(ct, local_name, addrlen);
+ PP_DEBUG("Sent name\n");
+
+ return ret;
+}
+
+int pp_recv_name(struct ct_pingpong *ct)
+{
+ uint32_t len;
+ int ret;
+
+ PP_DEBUG("Receiving name length\n");
+ ret = pp_ctrl_recv(ct, (char *) &len, sizeof(len));
+ if (ret < 0)
+ return ret;
+
+ len = ntohl(len);
+
+ if (len > sizeof(ct->rem_name)) {
+ PP_DEBUG("Address length exceeds address storage\n");
+ return -EMSGSIZE;
+ }
+
+ PP_DEBUG("Receiving name\n");
+ ret = pp_ctrl_recv(ct, ct->rem_name, len);
+ if (ret < 0)
+ return ret;
+ PP_DEBUG("Received name\n");
+
+ ct->hints->dest_addr = malloc(len);
+ if (!ct->hints->dest_addr) {
+ PP_DEBUG("Failed to allocate memory for destination address\n");
+ return -ENOMEM;
+ }
+
+ /* fi_freeinfo will free the dest_addr field. */
+ memcpy(ct->hints->dest_addr, ct->rem_name, len);
+ ct->hints->dest_addrlen = len;
+
+ return 0;
+}
+
+int pp_ctrl_finish(struct ct_pingpong *ct)
+{
+ if (ct->ctrl_connfd != -1) {
+ close(ct->ctrl_connfd);
+ ct->ctrl_connfd = -1;
+ }
+
+ return 0;
+}
+
+int pp_ctrl_sync(struct ct_pingpong *ct)
+{
+ int ret;
+
+ PP_DEBUG("Syncing nodes\n");
+
+ if (ct->opts.dst_addr) {
+ snprintf(ct->ctrl_buf, sizeof(PP_MSG_SYNC_Q), "%s",
+ PP_MSG_SYNC_Q);
+
+ PP_DEBUG("CLIENT: syncing\n");
+ ret = pp_ctrl_send(ct, ct->ctrl_buf, sizeof(PP_MSG_SYNC_Q));
+ PP_DEBUG("CLIENT: after send / ret=%d\n", ret);
+ if (ret < 0)
+ return ret;
+ if (ret < sizeof(PP_MSG_SYNC_Q)) {
+ PP_ERR("CLIENT: bad length of sent data (len=%d/%zu)",
+ ret, sizeof(PP_MSG_SYNC_Q));
+ return -EBADMSG;
+ }
+ PP_DEBUG("CLIENT: syncing now\n");
+
+ ret = pp_ctrl_recv(ct, ct->ctrl_buf, sizeof(PP_MSG_SYNC_A));
+ PP_DEBUG("CLIENT: after recv / ret=%d\n", ret);
+ if (ret < 0)
+ return ret;
+ if (strcmp(ct->ctrl_buf, PP_MSG_SYNC_A)) {
+ ct->ctrl_buf[PP_CTRL_BUF_LEN] = '\0';
+ PP_DEBUG("CLIENT: sync error while acking A: <%s> "
+ "(len=%zu)\n",
+ ct->ctrl_buf, strlen(ct->ctrl_buf));
+ return -EBADMSG;
+ }
+ PP_DEBUG("CLIENT: synced\n");
+ } else {
+ PP_DEBUG("SERVER: syncing\n");
+ ret = pp_ctrl_recv(ct, ct->ctrl_buf, sizeof(PP_MSG_SYNC_Q));
+ PP_DEBUG("SERVER: after recv / ret=%d\n", ret);
+ if (ret < 0)
+ return ret;
+ if (strcmp(ct->ctrl_buf, PP_MSG_SYNC_Q)) {
+ ct->ctrl_buf[PP_CTRL_BUF_LEN] = '\0';
+ PP_DEBUG("SERVER: sync error while acking Q: <%s> "
+ "(len=%zu)\n",
+ ct->ctrl_buf, strlen(ct->ctrl_buf));
+ return -EBADMSG;
+ }
+
+ PP_DEBUG("SERVER: syncing now\n");
+ snprintf(ct->ctrl_buf, sizeof(PP_MSG_SYNC_A), "%s",
+ PP_MSG_SYNC_A);
+
+ ret = pp_ctrl_send(ct, ct->ctrl_buf, sizeof(PP_MSG_SYNC_A));
+ PP_DEBUG("SERVER: after send / ret=%d\n", ret);
+ if (ret < 0)
+ return ret;
+ if (ret < sizeof(PP_MSG_SYNC_A)) {
+ PP_ERR("SERVER: bad length of sent data (len=%d/%zu)",
+ ret, sizeof(PP_MSG_SYNC_A));
+ return -EBADMSG;
+ }
+ PP_DEBUG("SERVER: synced\n");
+ }
+
+ PP_DEBUG("Nodes synced\n");
+
+ return 0;
+}
+
+int pp_ctrl_txrx_msg_count(struct ct_pingpong *ct)
+{
+ int ret;
+
+ PP_DEBUG("Exchanging ack count\n");
+
+ if (ct->opts.dst_addr) {
+ memset(&ct->ctrl_buf, '\0', PP_MSG_LEN_CNT + 1);
+ snprintf(ct->ctrl_buf, PP_MSG_LEN_CNT + 1, "%ld",
+ ct->cnt_ack_msg);
+
+ PP_DEBUG("CLIENT: sending count = <%s> (len=%zu)\n",
+ ct->ctrl_buf, strlen(ct->ctrl_buf));
+ ret = pp_ctrl_send(ct, ct->ctrl_buf, PP_MSG_LEN_CNT);
+ if (ret < 0)
+ return ret;
+ if (ret < PP_MSG_LEN_CNT) {
+ PP_ERR("CLIENT: bad length of sent data (len=%d/%d)",
+ ret, PP_MSG_LEN_CNT);
+ return -EBADMSG;
+ }
+ PP_DEBUG("CLIENT: sent count\n");
+
+ ret =
+ pp_ctrl_recv(ct, ct->ctrl_buf, sizeof(PP_MSG_CHECK_CNT_OK));
+ if (ret < 0)
+ return ret;
+ if (ret < sizeof(PP_MSG_CHECK_CNT_OK)) {
+ PP_ERR(
+ "CLIENT: bad length of received data (len=%d/%zu)",
+ ret, sizeof(PP_MSG_CHECK_CNT_OK));
+ return -EBADMSG;
+ }
+
+ if (strcmp(ct->ctrl_buf, PP_MSG_CHECK_CNT_OK)) {
+ PP_DEBUG("CLIENT: error while server acking the count: "
+ "<%s> (len=%zu)\n",
+ ct->ctrl_buf, strlen(ct->ctrl_buf));
+ return ret;
+ }
+ PP_DEBUG("CLIENT: count acked by server\n");
+ } else {
+ memset(&ct->ctrl_buf, '\0', PP_MSG_LEN_CNT + 1);
+
+ PP_DEBUG("SERVER: receiving count\n");
+ ret = pp_ctrl_recv(ct, ct->ctrl_buf, PP_MSG_LEN_CNT);
+ if (ret < 0)
+ return ret;
+ if (ret < PP_MSG_LEN_CNT) {
+ PP_ERR(
+ "SERVER: bad length of received data (len=%d/%d)",
+ ret, PP_MSG_LEN_CNT);
+ return -EBADMSG;
+ }
+ ct->cnt_ack_msg = parse_ulong(ct->ctrl_buf, -1);
+ if (ct->cnt_ack_msg < 0)
+ return ret;
+ PP_DEBUG("SERVER: received count = <%ld> (len=%zu)\n",
+ ct->cnt_ack_msg, strlen(ct->ctrl_buf));
+
+ snprintf(ct->ctrl_buf, sizeof(PP_MSG_CHECK_CNT_OK), "%s",
+ PP_MSG_CHECK_CNT_OK);
+ ret =
+ pp_ctrl_send(ct, ct->ctrl_buf, sizeof(PP_MSG_CHECK_CNT_OK));
+ if (ret < 0)
+ return ret;
+ if (ret < sizeof(PP_MSG_CHECK_CNT_OK)) {
+ PP_ERR(
+ "CLIENT: bad length of received data (len=%d/%zu)",
+ ret, sizeof(PP_MSG_CHECK_CNT_OK));
+ return -EBADMSG;
+ }
+ PP_DEBUG("SERVER: acked count to client\n");
+ }
+
+ PP_DEBUG("Ack count exchanged\n");
+
+ return 0;
+}
+
+/*******************************************************************************
+ * Options
+ ******************************************************************************/
+
+static inline void pp_start(struct ct_pingpong *ct)
+{
+ PP_DEBUG("Starting test chrono\n");
+ ct->opts.options |= PP_OPT_ACTIVE;
+ clock_gettime(CLOCK_MONOTONIC, &(ct->start));
+}
+
+static inline void pp_stop(struct ct_pingpong *ct)
+{
+ clock_gettime(CLOCK_MONOTONIC, &(ct->end));
+ ct->opts.options &= ~PP_OPT_ACTIVE;
+ PP_DEBUG("Stopped test chrono\n");
+}
+
+static inline int pp_check_opts(struct ct_pingpong *ct, uint64_t flags)
+{
+ return (ct->opts.options & flags) == flags;
+}
+
+/*******************************************************************************
+ * Data Verification
+ ******************************************************************************/
+
+void pp_fill_buf(void *buf, int size)
+{
+ char *msg_buf;
+ int msg_index;
+ static unsigned int iter;
+ int i;
+
+ msg_index = ((iter++) * INTEG_SEED) % integ_alphabet_length;
+ msg_buf = (char *)buf;
+ for (i = 0; i < size; i++) {
+ PP_DEBUG("index=%d msg_index=%d\n", i, msg_index);
+ msg_buf[i] = integ_alphabet[msg_index++];
+ if (msg_index >= integ_alphabet_length)
+ msg_index = 0;
+ }
+}
+
+int pp_check_buf(void *buf, int size)
+{
+ char *recv_data;
+ char c;
+ static unsigned int iter;
+ int msg_index;
+ int i;
+
+ PP_DEBUG("Verifying buffer content\n");
+
+ msg_index = ((iter++) * INTEG_SEED) % integ_alphabet_length;
+ recv_data = (char *)buf;
+
+ for (i = 0; i < size; i++) {
+ c = integ_alphabet[msg_index++];
+ if (msg_index >= integ_alphabet_length)
+ msg_index = 0;
+ if (c != recv_data[i]) {
+ PP_DEBUG("index=%d msg_index=%d expected=%d got=%d\n",
+ i, msg_index, c, recv_data[i]);
+ break;
+ }
+ }
+ if (i != size) {
+ PP_DEBUG("Finished veryfing buffer: content is corrupted\n");
+ printf("Error at iteration=%d size=%d byte=%d\n", iter, size,
+ i);
+ return 1;
+ }
+
+ PP_DEBUG("Buffer verified\n");
+
+ return 0;
+}
+
+
+/*******************************************************************************
+ * Error handling
+ ******************************************************************************/
+
+void eq_readerr(struct fid_eq *eq)
+{
+ struct fi_eq_err_entry eq_err;
+ int rd;
+
+ rd = fi_eq_readerr(eq, &eq_err, 0);
+ if (rd != sizeof(eq_err)) {
+ PP_PRINTERR("fi_eq_readerr", rd);
+ } else {
+ PP_ERR("eq_readerr: %s",
+ fi_eq_strerror(eq, eq_err.prov_errno, eq_err.err_data,
+ NULL, 0));
+ }
+}
+
+void pp_process_eq_err(ssize_t rd, struct fid_eq *eq, const char *fn)
+{
+ if (rd == -FI_EAVAIL)
+ eq_readerr(eq);
+ else
+ PP_PRINTERR(fn, rd);
+}
+
+/*******************************************************************************
+ * Test sizes
+ ******************************************************************************/
+
+int generate_test_sizes(struct pp_opts *opts, size_t tx_size, int **sizes_)
+{
+ int defaults[6] = {64, 256, 1024, 4096, 65536, 1048576};
+ int power_of_two;
+ int half_up;
+ int n = 0;
+ int i;
+ int *sizes = NULL;
+
+ PP_DEBUG("Generating test sizes\n");
+
+ sizes = calloc(64, sizeof(*sizes));
+ if (sizes == NULL)
+ return 0;
+ *sizes_ = sizes;
+
+ if (opts->options & PP_OPT_SIZE) {
+ if (opts->transfer_size > tx_size)
+ return 0;
+
+ sizes[0] = opts->transfer_size;
+ n = 1;
+ } else if (opts->sizes_enabled != PP_ENABLE_ALL) {
+ for (i = 0; i < (sizeof(defaults) / sizeof(defaults[0])); i++) {
+ if (defaults[i] > tx_size)
+ break;
+
+ sizes[i] = defaults[i];
+ n++;
+ }
+ } else {
+ for (i = 0;; i++) {
+ power_of_two = (i == 0) ? 0 : (1 << i);
+ half_up =
+ (i == 0) ? 1 : power_of_two + (power_of_two / 2);
+
+ if (power_of_two > tx_size)
+ break;
+
+ sizes[i * 2] = power_of_two;
+ n++;
+
+ if (half_up > tx_size)
+ break;
+
+ sizes[(i * 2) + 1] = half_up;
+ n++;
+ }
+ }
+
+ PP_DEBUG("Generated %d test sizes\n", n);
+
+ return n;
+}
+
+/*******************************************************************************
+ * Performance output
+ ******************************************************************************/
+
+/* str must be an allocated buffer of PP_STR_LEN bytes */
+char *size_str(char *str, uint64_t size)
+{
+ uint64_t base, fraction = 0;
+ char mag;
+
+ memset(str, '\0', PP_STR_LEN);
+
+ if (size >= (1 << 30)) {
+ base = 1 << 30;
+ mag = 'g';
+ } else if (size >= (1 << 20)) {
+ base = 1 << 20;
+ mag = 'm';
+ } else if (size >= (1 << 10)) {
+ base = 1 << 10;
+ mag = 'k';
+ } else {
+ base = 1;
+ mag = '\0';
+ }
+
+ if (size / base < 10)
+ fraction = (size % base) * 10 / base;
+
+ if (fraction)
+ snprintf(str, PP_STR_LEN, "%" PRIu64 ".%" PRIu64 "%c",
+ size / base, fraction, mag);
+ else
+ snprintf(str, PP_STR_LEN, "%" PRIu64 "%c", size / base, mag);
+
+ return str;
+}
+
+/* str must be an allocated buffer of PP_STR_LEN bytes */
+char *cnt_str(char *str, size_t size, uint64_t cnt)
+{
+ if (cnt >= 1000000000)
+ snprintf(str, size, "%" PRIu64 "b", cnt / 1000000000);
+ else if (cnt >= 1000000)
+ snprintf(str, size, "%" PRIu64 "m", cnt / 1000000);
+ else if (cnt >= 1000)
+ snprintf(str, size, "%" PRIu64 "k", cnt / 1000);
+ else
+ snprintf(str, size, "%" PRIu64, cnt);
+
+ return str;
+}
+
+int64_t get_elapsed(const struct timespec *b, const struct timespec *a,
+ enum precision p)
+{
+ int64_t elapsed;
+
+ elapsed = difftime(a->tv_sec, b->tv_sec) * 1000 * 1000 * 1000;
+ elapsed += a->tv_nsec - b->tv_nsec;
+ return elapsed / p;
+}
+
+void show_perf(char *name, int tsize, int sent, int acked,
+ struct timespec *start, struct timespec *end, int xfers_per_iter)
+{
+ static int header = 1;
+ char str[PP_STR_LEN];
+ int64_t elapsed = get_elapsed(start, end, MICRO);
+ uint64_t bytes = (uint64_t)sent * tsize * xfers_per_iter;
+ float usec_per_xfer;
+
+ if (sent == 0)
+ return;
+
+ if (name) {
+ if (header) {
+ printf("%-50s%-8s%-8s%-9s%-8s%8s %10s%13s%13s\n",
+ "name", "bytes", "#sent", "#ack", "total",
+ "time", "MB/sec", "usec/xfer", "Mxfers/sec");
+ header = 0;
+ }
+
+ printf("%-50s", name);
+ } else {
+ if (header) {
+ printf("%-8s%-8s%-9s%-8s%8s %10s%13s%13s\n", "bytes",
+ "#sent", "#ack", "total", "time", "MB/sec",
+ "usec/xfer", "Mxfers/sec");
+ header = 0;
+ }
+ }
+
+ printf("%-8s", size_str(str, tsize));
+ printf("%-8s", cnt_str(str, sizeof(str), sent));
+
+ if (sent == acked)
+ printf("=%-8s", cnt_str(str, sizeof(str), acked));
+ else if (sent < acked)
+ printf("-%-8s", cnt_str(str, sizeof(str), acked - sent));
+ else
+ printf("+%-8s", cnt_str(str, sizeof(str), sent - acked));
+
+ printf("%-8s", size_str(str, bytes));
+
+ usec_per_xfer = ((float)elapsed / sent / xfers_per_iter);
+ printf("%8.2fs%10.2f%11.2f%11.2f\n", elapsed / 1000000.0,
+ bytes / (1.0 * elapsed), usec_per_xfer, 1.0 / usec_per_xfer);
+}
+
+/*******************************************************************************
+ * Data Messaging
+ ******************************************************************************/
+
+int pp_cq_readerr(struct fid_cq *cq)
+{
+ struct fi_cq_err_entry cq_err;
+ int ret;
+
+ ret = fi_cq_readerr(cq, &cq_err, 0);
+ if (ret < 0) {
+ PP_PRINTERR("fi_cq_readerr", ret);
+ } else {
+ PP_ERR("cq_readerr: %s",
+ fi_cq_strerror(cq, cq_err.prov_errno, cq_err.err_data,
+ NULL, 0));
+ ret = -cq_err.err;
+ }
+ return ret;
+}
+
+static int pp_get_cq_comp(struct fid_cq *cq, uint64_t *cur, uint64_t total,
+ int timeout)
+{
+ struct fi_cq_err_entry comp;
+ struct timespec a = {0}, b = {0};
+ int ret = 0;
+
+ if (timeout >= 0)
+ clock_gettime(CLOCK_MONOTONIC, &a);
+
+ while (total - *cur > 0) {
+ ret = fi_cq_read(cq, &comp, 1);
+ if (ret > 0) {
+ if (timeout >= 0)
+ clock_gettime(CLOCK_MONOTONIC, &a);
+
+ (*cur)++;
+ } else if (ret < 0 && ret != -FI_EAGAIN) {
+ if (ret == -FI_EAVAIL) {
+ ret = pp_cq_readerr(cq);
+ (*cur)++;
+ } else {
+ PP_PRINTERR("pp_get_cq_comp", ret);
+ }
+
+ return ret;
+ } else if (timeout >= 0) {
+ clock_gettime(CLOCK_MONOTONIC, &b);
+ if ((b.tv_sec - a.tv_sec) > timeout) {
+ fprintf(stderr, "%ds timeout expired\n",
+ timeout);
+ return -FI_ENODATA;
+ }
+ }
+ }
+
+ return 0;
+}
+
+int pp_get_rx_comp(struct ct_pingpong *ct, uint64_t total)
+{
+ int ret = FI_SUCCESS;
+
+ if (ct->rxcq) {
+ ret = pp_get_cq_comp(ct->rxcq, &(ct->rx_cq_cntr), total,
+ ct->timeout);
+ } else {
+ PP_ERR(
+ "Trying to get a RX completion when no RX CQ was opened");
+ ret = -FI_EOTHER;
+ }
+ return ret;
+}
+
+int pp_get_tx_comp(struct ct_pingpong *ct, uint64_t total)
+{
+ int ret;
+
+ if (ct->txcq) {
+ ret = pp_get_cq_comp(ct->txcq, &(ct->tx_cq_cntr), total, -1);
+ } else {
+ PP_ERR(
+ "Trying to get a TX completion when no TX CQ was opened");
+ ret = -FI_EOTHER;
+ }
+ return ret;
+}
+
+#define PP_POST(post_fn, comp_fn, seq, op_str, ...) \
+ do { \
+ int timeout_save; \
+ int ret, rc; \
+ \
+ while (1) { \
+ ret = post_fn(__VA_ARGS__); \
+ if (!ret) \
+ break; \
+ \
+ if (ret != -FI_EAGAIN) { \
+ PP_PRINTERR(op_str, ret); \
+ return ret; \
+ } \
+ \
+ timeout_save = ct->timeout; \
+ ct->timeout = 0; \
+ rc = comp_fn(ct, seq); \
+ ct->timeout = timeout_save; \
+ if (rc && rc != -FI_EAGAIN) { \
+ PP_ERR("Failed to get " op_str " completion"); \
+ return rc; \
+ } \
+ } \
+ seq++; \
+ } while (0)
+
+ssize_t pp_post_tx(struct ct_pingpong *ct, struct fid_ep *ep, size_t size,
+ struct fi_context *ctx)
+{
+ PP_POST(fi_send, pp_get_tx_comp, ct->tx_seq, "transmit", ep, ct->tx_buf,
+ size, fi_mr_desc(ct->mr), ct->remote_fi_addr, ctx);
+ return 0;
+}
+
+ssize_t pp_tx(struct ct_pingpong *ct, struct fid_ep *ep, size_t size)
+{
+ ssize_t ret;
+
+ if (pp_check_opts(ct, PP_OPT_VERIFY_DATA | PP_OPT_ACTIVE))
+ pp_fill_buf((char *)ct->tx_buf, size);
+
+ ret = pp_post_tx(ct, ep, size, &(ct->tx_ctx));
+ if (ret)
+ return ret;
+
+ ret = pp_get_tx_comp(ct, ct->tx_seq);
+
+ return ret;
+}
+
+ssize_t pp_post_inject(struct ct_pingpong *ct, struct fid_ep *ep, size_t size)
+{
+ PP_POST(fi_inject, pp_get_tx_comp, ct->tx_seq, "inject", ep, ct->tx_buf,
+ size, ct->remote_fi_addr);
+ ct->tx_cq_cntr++;
+ return 0;
+}
+
+ssize_t pp_inject(struct ct_pingpong *ct, struct fid_ep *ep, size_t size)
+{
+ ssize_t ret;
+
+ if (pp_check_opts(ct, PP_OPT_VERIFY_DATA | PP_OPT_ACTIVE))
+ pp_fill_buf((char *)ct->tx_buf, size);
+
+ ret = pp_post_inject(ct, ep, size);
+ if (ret)
+ return ret;
+
+ return ret;
+}
+
+ssize_t pp_post_rx(struct ct_pingpong *ct, struct fid_ep *ep, size_t size,
+ struct fi_context *ctx)
+{
+ PP_POST(fi_recv, pp_get_rx_comp, ct->rx_seq, "receive", ep, ct->rx_buf,
+ MAX(size, PP_MAX_CTRL_MSG), fi_mr_desc(ct->mr), 0, ctx);
+ return 0;
+}
+
+ssize_t pp_rx(struct ct_pingpong *ct, struct fid_ep *ep, size_t size)
+{
+ ssize_t ret;
+
+ ret = pp_get_rx_comp(ct, ct->rx_seq);
+ if (ret)
+ return ret;
+
+ if (pp_check_opts(ct, PP_OPT_VERIFY_DATA | PP_OPT_ACTIVE)) {
+ ret = pp_check_buf((char *)ct->rx_buf, size);
+ if (ret)
+ return ret;
+ }
+ /* TODO: verify CQ data, if available */
+
+ /* Ignore the size arg. Post a buffer large enough to handle all message
+ * sizes. pp_sync() makes use of pp_rx() and gets called in tests just
+ * before message size is updated. The recvs posted are always for the
+ * next incoming message.
+ */
+ ret = pp_post_rx(ct, ct->ep, ct->rx_size, &(ct->rx_ctx));
+ if (!ret)
+ ct->cnt_ack_msg++;
+
+ return ret;
+}
+
+/*******************************************************************************
+ * Initialization and allocations
+ ******************************************************************************/
+
+void init_test(struct ct_pingpong *ct, struct pp_opts *opts)
+{
+ char sstr[PP_STR_LEN];
+
+ size_str(sstr, opts->transfer_size);
+ if (!(opts->options & PP_OPT_ITER))
+ opts->iterations = size_to_count(opts->transfer_size);
+
+ ct->cnt_ack_msg = 0;
+}
+
+uint64_t pp_init_cq_data(struct fi_info *info)
+{
+ if (info->domain_attr->cq_data_size >= sizeof(uint64_t)) {
+ return 0x0123456789abcdefULL;
+ } else {
+ return 0x0123456789abcdefULL &
+ ((0x1ULL << (info->domain_attr->cq_data_size * 8)) - 1);
+ }
+}
+
+int pp_alloc_msgs(struct ct_pingpong *ct)
+{
+ int ret;
+ long alignment = 1;
+
+ ct->tx_size = ct->opts.options & PP_OPT_SIZE ? ct->opts.transfer_size
+ : PP_MAX_DATA_MSG;
+ if (ct->tx_size > ct->fi->ep_attr->max_msg_size)
+ ct->tx_size = ct->fi->ep_attr->max_msg_size;
+ ct->rx_size = ct->tx_size;
+ ct->buf_size = MAX(ct->tx_size, PP_MAX_CTRL_MSG) +
+ MAX(ct->rx_size, PP_MAX_CTRL_MSG);
+
+ alignment = sysconf(_SC_PAGESIZE);
+ if (alignment < 0) {
+ ret = -errno;
+ PP_PRINTERR("sysconf", ret);
+ return ret;
+ }
+ /* Extra alignment for the second part of the buffer */
+ ct->buf_size += alignment;
+
+ ret = posix_memalign(&(ct->buf), (size_t)alignment, ct->buf_size);
+ if (ret) {
+ PP_PRINTERR("posix_memalign", ret);
+ return ret;
+ }
+ memset(ct->buf, 0, ct->buf_size);
+ ct->rx_buf = ct->buf;
+ ct->tx_buf = (char *)ct->buf + MAX(ct->rx_size, PP_MAX_CTRL_MSG);
+ ct->tx_buf = (void *)(((uintptr_t)ct->tx_buf + alignment - 1) &
+ ~(alignment - 1));
+
+ ct->remote_cq_data = pp_init_cq_data(ct->fi);
+
+ if (ct->fi->mode & FI_LOCAL_MR) {
+ ret = fi_mr_reg(ct->domain, ct->buf, ct->buf_size,
+ FI_SEND | FI_RECV, 0, PP_MR_KEY, 0, &(ct->mr),
+ NULL);
+ if (ret) {
+ PP_PRINTERR("fi_mr_reg", ret);
+ return ret;
+ }
+ } else {
+ ct->mr = &(ct->no_mr);
+ }
+
+ return 0;
+}
+
+int pp_open_fabric_res(struct ct_pingpong *ct)
+{
+ int ret;
+
+ PP_DEBUG("Opening fabric resources: fabric, eq & domain\n");
+
+ ret = fi_fabric(ct->fi->fabric_attr, &(ct->fabric), NULL);
+ if (ret) {
+ PP_PRINTERR("fi_fabric", ret);
+ return ret;
+ }
+
+ ret = fi_eq_open(ct->fabric, &(ct->eq_attr), &(ct->eq), NULL);
+ if (ret) {
+ PP_PRINTERR("fi_eq_open", ret);
+ return ret;
+ }
+
+ ret = fi_domain(ct->fabric, ct->fi, &(ct->domain), NULL);
+ if (ret) {
+ PP_PRINTERR("fi_domain", ret);
+ return ret;
+ }
+
+ PP_DEBUG("Fabric resources opened\n");
+
+ return 0;
+}
+
+int pp_alloc_active_res(struct ct_pingpong *ct, struct fi_info *fi)
+{
+ int ret;
+
+ ret = pp_alloc_msgs(ct);
+ if (ret)
+ return ret;
+
+ if (ct->cq_attr.format == FI_CQ_FORMAT_UNSPEC)
+ ct->cq_attr.format = FI_CQ_FORMAT_CONTEXT;
+
+ ct->cq_attr.wait_obj = FI_WAIT_NONE;
+
+ ct->cq_attr.size = fi->tx_attr->size;
+ ret = fi_cq_open(ct->domain, &(ct->cq_attr), &(ct->txcq), &(ct->txcq));
+ if (ret) {
+ PP_PRINTERR("fi_cq_open", ret);
+ return ret;
+ }
+
+ ct->cq_attr.size = fi->rx_attr->size;
+ ret = fi_cq_open(ct->domain, &(ct->cq_attr), &(ct->rxcq), &(ct->rxcq));
+ if (ret) {
+ PP_PRINTERR("fi_cq_open", ret);
+ return ret;
+ }
+
+ if (fi->ep_attr->type == FI_EP_RDM ||
+ fi->ep_attr->type == FI_EP_DGRAM) {
+ if (fi->domain_attr->av_type != FI_AV_UNSPEC)
+ ct->av_attr.type = fi->domain_attr->av_type;
+
+ ret = fi_av_open(ct->domain, &(ct->av_attr), &(ct->av), NULL);
+ if (ret) {
+ PP_PRINTERR("fi_av_open", ret);
+ return ret;
+ }
+ }
+
+ ret = fi_endpoint(ct->domain, fi, &(ct->ep), NULL);
+ if (ret) {
+ PP_PRINTERR("fi_endpoint", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+int pp_getinfo(struct ct_pingpong *ct, struct fi_info *hints,
+ struct fi_info **info)
+{
+ uint64_t flags = 0;
+ int ret;
+
+ if (!hints->ep_attr->type)
+ hints->ep_attr->type = FI_EP_DGRAM;
+
+ ret = fi_getinfo(PP_FIVERSION, NULL, NULL, flags, hints, info);
+ if (ret) {
+ PP_PRINTERR("fi_getinfo", ret);
+ return ret;
+ }
+ return 0;
+}
+
+#define PP_EP_BIND(ep, fd, flags) \
+ do { \
+ int ret; \
+ if ((fd)) { \
+ ret = fi_ep_bind((ep), &(fd)->fid, (flags)); \
+ if (ret) { \
+ PP_PRINTERR("fi_ep_bind", ret); \
+ return ret; \
+ } \
+ } \
+ } while (0)
+
+int pp_init_ep(struct ct_pingpong *ct)
+{
+ int ret;
+
+ PP_DEBUG("Initializing endpoint\n");
+
+ if (ct->fi->ep_attr->type == FI_EP_MSG)
+ PP_EP_BIND(ct->ep, ct->eq, 0);
+ PP_EP_BIND(ct->ep, ct->av, 0);
+ PP_EP_BIND(ct->ep, ct->txcq, FI_TRANSMIT);
+ PP_EP_BIND(ct->ep, ct->rxcq, FI_RECV);
+
+ ret = fi_enable(ct->ep);
+ if (ret) {
+ PP_PRINTERR("fi_enable", ret);
+ return ret;
+ }
+
+ ret = pp_post_rx(ct, ct->ep, MAX(ct->rx_size, PP_MAX_CTRL_MSG),
+ &(ct->rx_ctx));
+ if (ret)
+ return ret;
+
+ PP_DEBUG("Endpoint initialzed\n");
+
+ return 0;
+}
+
+int pp_av_insert(struct fid_av *av, void *addr, size_t count,
+ fi_addr_t *fi_addr, uint64_t flags, void *context)
+{
+ int ret;
+
+ PP_DEBUG("Connection-less endpoint: inserting new address in vector\n");
+
+ ret = fi_av_insert(av, addr, count, fi_addr, flags, context);
+ if (ret < 0) {
+ PP_PRINTERR("fi_av_insert", ret);
+ return ret;
+ } else if (ret != count) {
+ PP_ERR("fi_av_insert: number of addresses inserted = %d;"
+ " number of addresses given = %zd\n",
+ ret, count);
+ return -EXIT_FAILURE;
+ }
+
+ PP_DEBUG("Connection-less endpoint: new address inserted in vector\n");
+
+ return 0;
+}
+
+int pp_exchange_names_connected(struct ct_pingpong *ct)
+{
+ int ret;
+
+ PP_DEBUG("Connection-based endpoint: setting up connection\n");
+
+ ret = pp_ctrl_sync(ct);
+ if (ret)
+ return ret;
+
+ if (ct->opts.dst_addr) {
+ ret = pp_recv_name(ct);
+ if (ret < 0)
+ return ret;
+
+ ret = pp_getinfo(ct, ct->hints, &(ct->fi));
+ if (ret)
+ return ret;
+ } else {
+ ret = pp_send_name(ct, &ct->pep->fid);
+ if (ret < 0)
+ return ret;
+ }
+
+ return 0;
+}
+
+int pp_start_server(struct ct_pingpong *ct)
+{
+ int ret;
+
+ PP_DEBUG("Connected endpoint: starting server\n");
+
+ ret = pp_getinfo(ct, ct->hints, &(ct->fi_pep));
+ if (ret)
+ return ret;
+
+ ret = fi_fabric(ct->fi_pep->fabric_attr, &(ct->fabric), NULL);
+ if (ret) {
+ PP_PRINTERR("fi_fabric", ret);
+ return ret;
+ }
+
+ ret = fi_eq_open(ct->fabric, &(ct->eq_attr), &(ct->eq), NULL);
+ if (ret) {
+ PP_PRINTERR("fi_eq_open", ret);
+ return ret;
+ }
+
+ ret = fi_passive_ep(ct->fabric, ct->fi_pep, &(ct->pep), NULL);
+ if (ret) {
+ PP_PRINTERR("fi_passive_ep", ret);
+ return ret;
+ }
+
+ ret = fi_pep_bind(ct->pep, &(ct->eq->fid), 0);
+ if (ret) {
+ PP_PRINTERR("fi_pep_bind", ret);
+ return ret;
+ }
+
+ ret = fi_listen(ct->pep);
+ if (ret) {
+ PP_PRINTERR("fi_listen", ret);
+ return ret;
+ }
+
+ PP_DEBUG("Connected endpoint: server started\n");
+
+ return 0;
+}
+
+int pp_server_connect(struct ct_pingpong *ct)
+{
+ struct fi_eq_cm_entry entry;
+ uint32_t event;
+ ssize_t rd;
+ int ret;
+
+ PP_DEBUG("Connected endpoint: connecting server\n");
+
+ ret = pp_exchange_names_connected(ct);
+ if (ret)
+ goto err;
+
+ ret = pp_ctrl_sync(ct);
+ if (ret)
+ goto err;
+
+ /* Listen */
+ rd = fi_eq_sread(ct->eq, &event, &entry, sizeof(entry), -1, 0);
+ if (rd != sizeof(entry)) {
+ pp_process_eq_err(rd, ct->eq, "fi_eq_sread");
+ return (int)rd;
+ }
+
+ ct->fi = entry.info;
+ if (event != FI_CONNREQ) {
+ fprintf(stderr, "Unexpected CM event %d\n", event);
+ ret = -FI_EOTHER;
+ goto err;
+ }
+
+ ret = fi_domain(ct->fabric, ct->fi, &(ct->domain), NULL);
+ if (ret) {
+ PP_PRINTERR("fi_domain", ret);
+ goto err;
+ }
+
+ ret = pp_alloc_active_res(ct, ct->fi);
+ if (ret)
+ goto err;
+
+ ret = pp_init_ep(ct);
+ if (ret)
+ goto err;
+
+ PP_DEBUG("accepting\n");
+
+ ret = fi_accept(ct->ep, NULL, 0);
+ if (ret) {
+ PP_PRINTERR("fi_accept", ret);
+ goto err;
+ }
+
+ ret = pp_ctrl_sync(ct);
+ if (ret)
+ goto err;
+
+ /* Accept */
+ rd = fi_eq_sread(ct->eq, &event, &entry, sizeof(entry), -1, 0);
+ if (rd != sizeof(entry)) {
+ pp_process_eq_err(rd, ct->eq, "fi_eq_sread");
+ ret = (int)rd;
+ goto err;
+ }
+
+ if (event != FI_CONNECTED || entry.fid != &(ct->ep->fid)) {
+ fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
+ event, entry.fid, ct->ep);
+ ret = -FI_EOTHER;
+ goto err;
+ }
+
+ PP_DEBUG("Connected endpoint: server connected\n");
+
+ return 0;
+err:
+ fi_reject(ct->pep, ct->fi->handle, NULL, 0);
+ return ret;
+}
+
+int pp_client_connect(struct ct_pingpong *ct)
+{
+ struct fi_eq_cm_entry entry;
+ uint32_t event;
+ ssize_t rd;
+ int ret;
+
+ ret = pp_exchange_names_connected(ct);
+ if (ret)
+ return ret;
+
+ /* Check that the remote is still up */
+ ret = pp_ctrl_sync(ct);
+ if (ret)
+ return ret;
+
+ ret = pp_open_fabric_res(ct);
+ if (ret)
+ return ret;
+
+ ret = pp_alloc_active_res(ct, ct->fi);
+ if (ret)
+ return ret;
+
+ ret = pp_init_ep(ct);
+ if (ret)
+ return ret;
+
+ ret = fi_connect(ct->ep, ct->rem_name, NULL, 0);
+ if (ret) {
+ PP_PRINTERR("fi_connect", ret);
+ return ret;
+ }
+
+ ret = pp_ctrl_sync(ct);
+ if (ret)
+ return ret;
+
+ /* Connect */
+ rd = fi_eq_sread(ct->eq, &event, &entry, sizeof(entry), -1, 0);
+ if (rd != sizeof(entry)) {
+ pp_process_eq_err(rd, ct->eq, "fi_eq_sread");
+ ret = (int)rd;
+ return ret;
+ }
+
+ if (event != FI_CONNECTED || entry.fid != &(ct->ep->fid)) {
+ fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
+ event, entry.fid, ct->ep);
+ ret = -FI_EOTHER;
+ return ret;
+ }
+
+ return 0;
+}
+
+int pp_init_fabric(struct ct_pingpong *ct)
+{
+ int ret;
+
+ ret = pp_ctrl_init(ct);
+ if (ret)
+ return ret;
+
+ PP_DEBUG("Initializing fabric\n");
+
+ PP_DEBUG("Connection-less endpoint: initializing address vector\n");
+
+ if (ct->opts.dst_addr) {
+ ret = pp_recv_name(ct);
+ if (ret < 0)
+ return ret;
+
+ ret = pp_getinfo(ct, ct->hints, &(ct->fi));
+ if (ret)
+ return ret;
+
+ ret = pp_open_fabric_res(ct);
+ if (ret)
+ return ret;
+
+ ret = pp_alloc_active_res(ct, ct->fi);
+ if (ret)
+ return ret;
+
+ ret = pp_init_ep(ct);
+ if (ret)
+ return ret;
+
+ ret = pp_send_name(ct, &ct->ep->fid);
+ } else {
+ PP_DEBUG("SERVER: getinfo\n");
+ ret = pp_getinfo(ct, ct->hints, &(ct->fi));
+ if (ret)
+ return ret;
+
+ PP_DEBUG("SERVER: open fabric resources\n");
+ ret = pp_open_fabric_res(ct);
+ if (ret)
+ return ret;
+
+ PP_DEBUG("SERVER: allocate active resource\n");
+ ret = pp_alloc_active_res(ct, ct->fi);
+ if (ret)
+ return ret;
+
+ PP_DEBUG("SERVER: initialize endpoint\n");
+ ret = pp_init_ep(ct);
+ if (ret)
+ return ret;
+
+ ret = pp_send_name(ct, &ct->ep->fid);
+ if (ret < 0)
+ return ret;
+
+ ret = pp_recv_name(ct);
+ }
+
+ if (ret < 0)
+ return ret;
+
+ ret = pp_av_insert(ct->av, ct->rem_name, 1, &(ct->remote_fi_addr), 0,
+ NULL);
+ if (ret)
+ return ret;
+ PP_DEBUG("Connection-less endpoint: address vector initialized\n");
+
+ PP_DEBUG("Fabric Initialized\n");
+
+ return 0;
+}
+
+/*******************************************************************************
+ * Deallocations and Final
+ ******************************************************************************/
+
+void pp_free_res(struct ct_pingpong *ct)
+{
+ PP_DEBUG("Freeing resources of test suite\n");
+
+ if (ct->mr != &(ct->no_mr))
+ PP_CLOSE_FID(ct->mr);
+ PP_CLOSE_FID(ct->ep);
+ PP_CLOSE_FID(ct->pep);
+ PP_CLOSE_FID(ct->rxcq);
+ PP_CLOSE_FID(ct->txcq);
+ PP_CLOSE_FID(ct->av);
+ PP_CLOSE_FID(ct->eq);
+ PP_CLOSE_FID(ct->domain);
+ PP_CLOSE_FID(ct->fabric);
+
+ if (ct->buf) {
+ free(ct->buf);
+ ct->buf = ct->rx_buf = ct->tx_buf = NULL;
+ ct->buf_size = ct->rx_size = ct->tx_size = 0;
+ }
+ if (ct->fi_pep) {
+ fi_freeinfo(ct->fi_pep);
+ ct->fi_pep = NULL;
+ }
+ if (ct->fi) {
+ fi_freeinfo(ct->fi);
+ ct->fi = NULL;
+ }
+ if (ct->hints) {
+ fi_freeinfo(ct->hints);
+ ct->hints = NULL;
+ }
+
+ PP_DEBUG("Resources of test suite freed\n");
+}
+
+int pp_finalize(struct ct_pingpong *ct)
+{
+ struct iovec iov;
+ int ret;
+ struct fi_context ctx;
+ struct fi_msg msg;
+
+ PP_DEBUG("Terminating test\n");
+
+ strcpy(ct->tx_buf, "fin");
+ iov.iov_base = ct->tx_buf;
+ iov.iov_len = 4;
+
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_iov = &iov;
+ msg.iov_count = 1;
+ msg.addr = ct->remote_fi_addr;
+ msg.context = &ctx;
+
+ ret = fi_sendmsg(ct->ep, &msg, FI_INJECT | FI_TRANSMIT_COMPLETE);
+ if (ret) {
+ PP_PRINTERR("transmit", ret);
+ return ret;
+ }
+
+ ret = pp_get_tx_comp(ct, ++ct->tx_seq);
+ if (ret)
+ return ret;
+
+ ret = pp_get_rx_comp(ct, ct->rx_seq);
+ if (ret)
+ return ret;
+
+ ret = pp_ctrl_finish(ct);
+ if (ret)
+ return ret;
+
+ PP_DEBUG("Test terminated\n");
+
+ return 0;
+}
+
+/*******************************************************************************
+ * CLI: Usage and Options parsing
+ ******************************************************************************/
+
+void pp_pingpong_usage(char *name, char *desc)
+{
+ fprintf(stderr, "Usage:\n");
+ fprintf(stderr, " %s [OPTIONS]\t\tstart server\n", name);
+ fprintf(stderr, " %s [OPTIONS] <srv_addr>\tconnect to server\n", name);
+
+ if (desc)
+ fprintf(stderr, "\n%s\n", desc);
+
+ fprintf(stderr, "\nOptions:\n");
+
+ fprintf(stderr, " %-20s %s\n", "-B <src_port>",
+ "source control port number (server: 47592, client: auto)");
+ fprintf(stderr, " %-20s %s\n", "-P <dst_port>",
+ "destination control port number (client: 47592)");
+
+ fprintf(stderr, " %-20s %s\n", "-d <domain>", "domain name");
+ fprintf(stderr, " %-20s %s\n", "-p <provider>",
+ "specific provider name eg sockets, verbs");
+ fprintf(stderr, " %-20s %s\n", "-e <ep_type>",
+ "endpoint type: msg|rdm|dgram (dgram)");
+
+ fprintf(stderr, " %-20s %s\n", "-I <number>",
+ "number of iterations (1000)");
+ fprintf(stderr, " %-20s %s\n", "-S <size>",
+ "specific transfer size or 'all' (all)");
+
+ fprintf(stderr, " %-20s %s\n", "-c", "enables data_integrity checks");
+
+ fprintf(stderr, " %-20s %s\n", "-h", "display this help output");
+ fprintf(stderr, " %-20s %s\n", "-v", "enable debugging output");
+}
+
+void pp_parse_opts(struct ct_pingpong *ct, int op, char *optarg)
+{
+ switch (op) {
+
+ /* Domain */
+ case 'd':
+ ct->hints->domain_attr->name = strdup(optarg);
+ break;
+
+ /* Provider */
+ case 'p':
+ /* The provider name will be checked during the fabric
+ * initialization.
+ */
+ ct->hints->fabric_attr->prov_name = strdup(optarg);
+ break;
+
+ /* Endpoint */
+ case 'e':
+ if (!strncasecmp("msg", optarg, 3) && (strlen(optarg) == 3)) {
+ ct->hints->ep_attr->type = FI_EP_MSG;
+ } else if (!strncasecmp("rdm", optarg, 3) &&
+ (strlen(optarg) == 3)) {
+ ct->hints->ep_attr->type = FI_EP_RDM;
+ } else if (!strncasecmp("dgram", optarg, 5) &&
+ (strlen(optarg) == 5)) {
+ ct->hints->ep_attr->type = FI_EP_DGRAM;
+ } else {
+ fprintf(stderr, "Unknown endpoint : %s\n", optarg);
+ exit(EXIT_FAILURE);
+ }
+ break;
+
+ /* Iterations */
+ case 'I':
+ ct->opts.options |= PP_OPT_ITER;
+ ct->opts.iterations = (int)parse_ulong(optarg, INT_MAX);
+ if (ct->opts.iterations < 0)
+ ct->opts.iterations = 0;
+ break;
+
+ /* Message Size */
+ case 'S':
+ if (!strncasecmp("all", optarg, 3) && (strlen(optarg) == 3)) {
+ ct->opts.sizes_enabled = PP_ENABLE_ALL;
+ } else {
+ ct->opts.options |= PP_OPT_SIZE;
+ ct->opts.transfer_size =
+ (int)parse_ulong(optarg, INT_MAX);
+ }
+ break;
+
+ /* Check data */
+ case 'c':
+ ct->opts.options |= PP_OPT_VERIFY_DATA;
+ break;
+
+ /* Source Port */
+ case 'B':
+ ct->opts.src_port = parse_ulong(optarg, UINT16_MAX);
+ break;
+
+ /* Destination Port */
+ case 'P':
+ ct->opts.dst_port = parse_ulong(optarg, UINT16_MAX);
+ break;
+
+ /* Debug */
+ case 'v':
+ pp_debug = 1;
+ break;
+ default:
+ /* let getopt handle unknown opts*/
+ break;
+ }
+}
+
+/*******************************************************************************
+ * PingPong core and implemenations for endpoints
+ ******************************************************************************/
+
+int pingpong(struct ct_pingpong *ct)
+{
+ int ret, i;
+
+ ret = pp_ctrl_sync(ct);
+ if (ret)
+ return ret;
+
+ pp_start(ct);
+ if (ct->opts.dst_addr) {
+ for (i = 0; i < ct->opts.iterations; i++) {
+
+ if (ct->opts.transfer_size <
+ ct->fi->tx_attr->inject_size)
+ ret = pp_inject(ct, ct->ep,
+ ct->opts.transfer_size);
+ else
+ ret = pp_tx(ct, ct->ep, ct->opts.transfer_size);
+ if (ret)
+ return ret;
+
+ ret = pp_rx(ct, ct->ep, ct->opts.transfer_size);
+ if (ret)
+ return ret;
+ }
+ } else {
+ for (i = 0; i < ct->opts.iterations; i++) {
+
+ ret = pp_rx(ct, ct->ep, ct->opts.transfer_size);
+ if (ret)
+ return ret;
+
+ if (ct->opts.transfer_size <
+ ct->fi->tx_attr->inject_size)
+ ret = pp_inject(ct, ct->ep,
+ ct->opts.transfer_size);
+ else
+ ret = pp_tx(ct, ct->ep, ct->opts.transfer_size);
+ if (ret)
+ return ret;
+ }
+ }
+ pp_stop(ct);
+
+ ret = pp_ctrl_txrx_msg_count(ct);
+ if (ret)
+ return ret;
+
+ PP_DEBUG("Results:\n");
+ show_perf(NULL, ct->opts.transfer_size, ct->opts.iterations,
+ ct->cnt_ack_msg, &(ct->start), &(ct->end), 2);
+
+ return 0;
+}
+
+int run_suite_pingpong(struct ct_pingpong *ct)
+{
+ int i, sizes_cnt;
+ int ret = 0;
+ int *sizes = NULL;
+
+ pp_banner_fabric_info(ct);
+
+ sizes_cnt = generate_test_sizes(&ct->opts, ct->tx_size, &sizes);
+
+ PP_DEBUG("Count of sizes to test: %d\n", sizes_cnt);
+
+ for (i = 0; i < sizes_cnt; i++) {
+ ct->opts.transfer_size = sizes[i];
+ init_test(ct, &(ct->opts));
+ ret = pingpong(ct);
+ if (ret)
+ goto out;
+ }
+
+out:
+ free(sizes);
+ return ret;
+}
+
+static int run_pingpong_dgram(struct ct_pingpong *ct)
+{
+ int ret;
+
+ PP_DEBUG("Selected endpoint: DGRAM\n");
+
+ ret = pp_init_fabric(ct);
+ if (ret)
+ return ret;
+
+ /* Post an extra receive to avoid lacking a posted receive in the
+ * finalize.
+ */
+ ret = fi_recv(ct->ep, ct->rx_buf, ct->rx_size, fi_mr_desc(ct->mr), 0,
+ &ct->rx_ctx);
+
+ ret = run_suite_pingpong(ct);
+ if (ret)
+ return ret;
+
+ return pp_finalize(ct);
+}
+
+static int run_pingpong_rdm(struct ct_pingpong *ct)
+{
+ int ret;
+
+ PP_DEBUG("Selected endpoint: RDM\n");
+
+ ret = pp_init_fabric(ct);
+ if (ret)
+ return ret;
+
+ ret = run_suite_pingpong(ct);
+ if (ret)
+ return ret;
+
+ return pp_finalize(ct);
+}
+
+static int run_pingpong_msg(struct ct_pingpong *ct)
+{
+ int ret;
+
+ PP_DEBUG("Selected endpoint: MSG\n");
+
+ ret = pp_ctrl_init(ct);
+ if (ret)
+ return ret;
+
+ if (!ct->opts.dst_addr) {
+ ret = pp_start_server(ct);
+ if (ret)
+ return ret;
+ }
+
+ if (ct->opts.dst_addr) {
+ ret = pp_client_connect(ct);
+ PP_DEBUG("CLIENT: client_connect=%s\n", ret ? "KO" : "OK");
+ } else {
+ ret = pp_server_connect(ct);
+ PP_DEBUG("SERVER: server_connect=%s\n", ret ? "KO" : "OK");
+ }
+
+ if (ret)
+ return ret;
+
+ ret = run_suite_pingpong(ct);
+ if (ret)
+ goto out;
+
+ ret = pp_finalize(ct);
+out:
+ fi_shutdown(ct->ep, 0);
+ return ret;
+}
+
+int main(int argc, char **argv)
+{
+ int ret, op;
+
+ ret = EXIT_SUCCESS;
+
+ struct ct_pingpong ct = {
+ .timeout = -1,
+ .ctrl_connfd = -1,
+ .opts = {
+ .iterations = 1000,
+ .transfer_size = 1024,
+ .sizes_enabled = PP_DEFAULT_SIZE
+ },
+ .eq_attr.wait_obj = FI_WAIT_UNSPEC,
+ };
+
+ ct.hints = fi_allocinfo();
+ if (!ct.hints)
+ return EXIT_FAILURE;
+ ct.hints->ep_attr->type = FI_EP_DGRAM;
+ ct.hints->caps = FI_MSG;
+ ct.hints->mode = FI_CONTEXT | FI_LOCAL_MR;
+
+ while ((op = getopt(argc, argv, "hvd:p:e:I:S:B:P:c")) != -1) {
+ switch (op) {
+ default:
+ pp_parse_opts(&ct, op, optarg);
+ break;
+ case '?':
+ case 'h':
+ pp_pingpong_usage(argv[0],
+ "Ping pong client and server");
+ return EXIT_FAILURE;
+ }
+ }
+
+ if (optind < argc)
+ ct.opts.dst_addr = argv[optind];
+
+ pp_banner_options(&ct);
+
+ switch (ct.hints->ep_attr->type) {
+ case FI_EP_DGRAM:
+ if (ct.opts.options & PP_OPT_SIZE)
+ ct.hints->ep_attr->max_msg_size = ct.opts.transfer_size;
+ ret = run_pingpong_dgram(&ct);
+ break;
+ case FI_EP_RDM:
+ ret = run_pingpong_rdm(&ct);
+ break;
+ case FI_EP_MSG:
+ ret = run_pingpong_msg(&ct);
+ break;
+ default:
+ fprintf(stderr, "Endpoint unsupported: %d\n",
+ ct.hints->ep_attr->type);
+ ret = EXIT_FAILURE;
+ }
+
+ pp_free_res(&ct);
+ return -ret;
+}
diff --git a/util/strerror.c b/util/strerror.c
new file mode 100644
index 0000000..ed8d338
--- /dev/null
+++ b/util/strerror.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2013-2014 Intel Corporation. All rights reserved.
+ * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
+ *
+ * This software is available to you under the BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <limits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <rdma/fi_errno.h>
+
+static void usage(const char *argv0)
+{
+ printf("Usage: %s FI_ERROR_CODE\n", argv0);
+ printf("\n");
+ printf("Displays the error string for the given numeric FI_ERROR_CODE.\n");
+ printf("FI_ERROR_CODE may be a hexadecimal, octal, or decimal constant.\n");
+ printf("For convenience, the absolute value of FI_ERROR_CODE will be used.\n");
+}
+
+int main(int argc, char *argv[])
+{
+ char *endptr;
+ long err;
+
+ if (argc != 2) {
+ usage(argv[0]);
+ return EXIT_FAILURE;
+ }
+
+ if (strcmp(argv[1], "-h") == 0) {
+ usage(argv[0]);
+ return EXIT_SUCCESS;
+ }
+
+ errno = 0;
+ endptr = NULL;
+ err = strtol(argv[1], &endptr, 0);
+ if (errno || endptr == argv[1] || *endptr != '\0') {
+ printf("ERROR: unable to parse '%s'\n\n", argv[1]);
+ usage(argv[0]);
+ return EXIT_FAILURE;
+ }
+
+ if (err < (long)INT_MIN || err > (long)INT_MAX) {
+ printf("ERROR: '%s' is out of range\n\n", argv[1]);
+ usage(argv[0]);
+ return EXIT_FAILURE;
+ }
+
+ printf("%s\n", fi_strerror(err >= 0 ? (int)err : (int)-err));
+
+ return EXIT_SUCCESS;
+}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-ofed/libfabric.git
More information about the Pkg-ofed-commits
mailing list