[Pkg-ofed-commits] [ibacm] 01/11: Imported Upstream version 1.1.0
Ana Beatriz Guerrero López
ana at moszumanska.debian.org
Tue Mar 29 22:24:10 UTC 2016
This is an automated email from the git hooks/post-receive script.
ana pushed a commit to branch master
in repository ibacm.
commit 8d1cd641635ac84cbea4327398dd5817c2215fc0
Author: Ana Beatriz Guerrero Lopez <ana at debian.org>
Date: Tue Mar 29 16:27:25 2016 +0200
Imported Upstream version 1.1.0
---
Makefile.am | 31 +-
Makefile.in | 253 ++-
config.h.in | 9 +
configure | 142 +-
configure.ac | 25 +-
ibacm.spec | 4 +-
ibacm.spec.in | 4 +-
{src => include}/acm_mad.h | 0
include/infiniband/acm.h | 23 +-
include/infiniband/acm_prov.h | 119 ++
linux/osd.h | 11 +-
man/ib_acme.1 | 15 +-
man/ibacm.1 | 63 +-
man/ibacm.7 | 4 +-
man/ibacm_prov.7 | 80 +
prov/acmp/src/acmp.c | 2917 ++++++++++++++++++++++++++++++
prov/acmp/src/libibacmp.map | 5 +
src/acm.c | 4014 ++++++++++++++---------------------------
src/acm_util.c | 18 +-
src/acm_util.h | 2 +-
src/acme.c | 228 ++-
src/libacm.c | 110 +-
src/libacm.h | 7 +-
23 files changed, 5226 insertions(+), 2858 deletions(-)
diff --git a/Makefile.am b/Makefile.am
index 9070a5a..7690cb9 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,25 +1,46 @@
AM_CPPFLAGS = -I$(srcdir)/include -I$(srcdir)/linux -I$(srcdir)/src
-AM_CFLAGS = -g -Wall -D_GNU_SOURCE -DSYSCONFDIR=\"$(sysconfdir)\" -DBINDIR=\"$(bindir)\" -DRDMADIR=\"@rdmadir@\"
+AM_CFLAGS = -g -Wall -D_GNU_SOURCE
+AM_LDFLAGS = -lpthread -ldl
+
+if HAVE_LD_VERSION_SCRIPT
+ libibacmp_version_script = -Wl,--version-script=$(srcdir)/prov/acmp/src/libibacmp.map
+else
+ libibacmp_version_script =
+endif
bin_PROGRAMS = util/ib_acme
sbin_PROGRAMS = svc/ibacm
svc_ibacm_SOURCES = src/acm.c src/acm_util.c
util_ib_acme_SOURCES = src/acme.c src/libacm.c linux/libacm_linux.c src/parse.c
svc_ibacm_CFLAGS = $(AM_CFLAGS)
+svc_ibacm_LDFLAGS = -rdynamic $(AM_LDFLAGS)
util_ib_acme_CFLAGS = $(AM_CFLAGS) -DACME_PRINTS
+pkglib_LTLIBRARIES = lib/libibacmp.la
+lib_libibacmp_la_CFLAGS = $(AM_CFLAGS)
+lib_libibacmp_la_SOURCES = prov/acmp/src/acmp.c
+lib_libibacmp_la_LDFLAGS = -version-info 1 -export-dynamic \
+ $(libibacmp_version_script) $(AM_LDFLAGS)
+lib_libibacmp_la_DEPENDENCIES = $(srcdir)/prov/acmp/src/libibacmp.map
+
ibacmincludedir = $(includedir)/infiniband
-ibacminclude_HEADERS = include/infiniband/acm.h
+ibacminclude_HEADERS = include/infiniband/acm.h include/infiniband/acm_prov.h
+
+libibacmpincludedir = $(includedir)/infiniband
+libibacmpinclude_HEADERS = include/infiniband/acm.h include/infiniband/acm_prov.h
man_MANS = \
man/ib_acme.1 \
man/ibacm.1 \
- man/ibacm.7
+ man/ibacm.7 \
+ man/ibacm_prov.7
-EXTRA_DIST = src/acm_util.h src/acm_mad.h src/libacm.h ibacm.init.in \
- linux/osd.h linux/dlist.h ibacm.spec.in $(man_MANS) ibacm_hosts.data
+EXTRA_DIST = src/acm_util.h prov/acmp/src/libibacmp.map \
+ include/acm_mad.h src/libacm.h ibacm.init.in \
+ linux/osd.h linux/dlist.h ibacm.spec.in \
+ $(man_MANS) ibacm_hosts.data
install-exec-hook:
install -D -m 755 ibacm.init $(DESTDIR)$(sysconfdir)/init.d/ibacm;
diff --git a/Makefile.in b/Makefile.in
index df4848f..389dffb 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -16,6 +16,7 @@
@SET_MAKE@
+
VPATH = @srcdir@
pkgdatadir = $(datadir)/@PACKAGE@
pkgincludedir = $(includedir)/@PACKAGE@
@@ -39,12 +40,13 @@ bin_PROGRAMS = util/ib_acme$(EXEEXT)
sbin_PROGRAMS = svc/ibacm$(EXEEXT)
subdir = .
DIST_COMMON = README $(am__configure_deps) $(ibacminclude_HEADERS) \
- $(srcdir)/Makefile.am $(srcdir)/Makefile.in \
- $(srcdir)/config.h.in $(srcdir)/ibacm.init.in \
- $(srcdir)/ibacm.spec.in $(top_srcdir)/configure AUTHORS \
- COPYING ChangeLog INSTALL NEWS config/compile \
- config/config.guess config/config.sub config/depcomp \
- config/install-sh config/ltmain.sh config/missing
+ $(libibacmpinclude_HEADERS) $(srcdir)/Makefile.am \
+ $(srcdir)/Makefile.in $(srcdir)/config.h.in \
+ $(srcdir)/ibacm.init.in $(srcdir)/ibacm.spec.in \
+ $(top_srcdir)/configure AUTHORS COPYING ChangeLog INSTALL NEWS \
+ config/compile config/config.guess config/config.sub \
+ config/depcomp config/install-sh config/ltmain.sh \
+ config/missing
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
am__aclocal_m4_deps = $(top_srcdir)/configure.ac
am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
@@ -55,21 +57,51 @@ mkinstalldirs = $(install_sh) -d
CONFIG_HEADER = config.h
CONFIG_CLEAN_FILES = ibacm.spec ibacm.init
CONFIG_CLEAN_VPATH_FILES =
-am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(sbindir)" \
- "$(DESTDIR)$(man1dir)" "$(DESTDIR)$(man7dir)" \
- "$(DESTDIR)$(ibacmincludedir)"
-PROGRAMS = $(bin_PROGRAMS) $(sbin_PROGRAMS)
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+ $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+ *) f=$$p;; \
+ esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+ srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+ for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+ for p in $$list; do echo "$$p $$p"; done | \
+ sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+ $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+ if (++n[$$2] == $(am__install_max)) \
+ { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+ END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+ sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+ sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__installdirs = "$(DESTDIR)$(pkglibdir)" "$(DESTDIR)$(bindir)" \
+ "$(DESTDIR)$(sbindir)" "$(DESTDIR)$(man1dir)" \
+ "$(DESTDIR)$(man7dir)" "$(DESTDIR)$(ibacmincludedir)" \
+ "$(DESTDIR)$(libibacmpincludedir)"
+LTLIBRARIES = $(pkglib_LTLIBRARIES)
+lib_libibacmp_la_LIBADD =
am__dirstamp = $(am__leading_dot)dirstamp
+am_lib_libibacmp_la_OBJECTS = prov/acmp/src/lib_libibacmp_la-acmp.lo
+lib_libibacmp_la_OBJECTS = $(am_lib_libibacmp_la_OBJECTS)
+AM_V_lt = $(am__v_lt_$(V))
+am__v_lt_ = $(am__v_lt_$(AM_DEFAULT_VERBOSITY))
+am__v_lt_0 = --silent
+lib_libibacmp_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+ $(lib_libibacmp_la_CFLAGS) $(CFLAGS) \
+ $(lib_libibacmp_la_LDFLAGS) $(LDFLAGS) -o $@
+PROGRAMS = $(bin_PROGRAMS) $(sbin_PROGRAMS)
am_svc_ibacm_OBJECTS = src/svc_ibacm-acm.$(OBJEXT) \
src/svc_ibacm-acm_util.$(OBJEXT)
svc_ibacm_OBJECTS = $(am_svc_ibacm_OBJECTS)
svc_ibacm_LDADD = $(LDADD)
-AM_V_lt = $(am__v_lt_$(V))
-am__v_lt_ = $(am__v_lt_$(AM_DEFAULT_VERBOSITY))
-am__v_lt_0 = --silent
svc_ibacm_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
$(LIBTOOLFLAGS) --mode=link $(CCLD) $(svc_ibacm_CFLAGS) \
- $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
+ $(CFLAGS) $(svc_ibacm_LDFLAGS) $(LDFLAGS) -o $@
am_util_ib_acme_OBJECTS = src/util_ib_acme-acme.$(OBJEXT) \
src/util_ib_acme-libacm.$(OBJEXT) \
linux/util_ib_acme-libacm_linux.$(OBJEXT) \
@@ -105,34 +137,15 @@ am__v_CCLD_0 = @echo " CCLD " $@;
AM_V_GEN = $(am__v_GEN_$(V))
am__v_GEN_ = $(am__v_GEN_$(AM_DEFAULT_VERBOSITY))
am__v_GEN_0 = @echo " GEN " $@;
-SOURCES = $(svc_ibacm_SOURCES) $(util_ib_acme_SOURCES)
-DIST_SOURCES = $(svc_ibacm_SOURCES) $(util_ib_acme_SOURCES)
-am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
-am__vpath_adj = case $$p in \
- $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
- *) f=$$p;; \
- esac;
-am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
-am__install_max = 40
-am__nobase_strip_setup = \
- srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
-am__nobase_strip = \
- for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
-am__nobase_list = $(am__nobase_strip_setup); \
- for p in $$list; do echo "$$p $$p"; done | \
- sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
- $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
- if (++n[$$2] == $(am__install_max)) \
- { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
- END { for (dir in files) print dir, files[dir] }'
-am__base_list = \
- sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
- sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+SOURCES = $(lib_libibacmp_la_SOURCES) $(svc_ibacm_SOURCES) \
+ $(util_ib_acme_SOURCES)
+DIST_SOURCES = $(lib_libibacmp_la_SOURCES) $(svc_ibacm_SOURCES) \
+ $(util_ib_acme_SOURCES)
man1dir = $(mandir)/man1
man7dir = $(mandir)/man7
NROFF = nroff
MANS = $(man_MANS)
-HEADERS = $(ibacminclude_HEADERS)
+HEADERS = $(ibacminclude_HEADERS) $(libibacmpinclude_HEADERS)
ETAGS = etags
CTAGS = ctags
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
@@ -171,6 +184,9 @@ EGREP = @EGREP@
EXEEXT = @EXEEXT@
FGREP = @FGREP@
GREP = @GREP@
+IBACM_BIN_PATH = @IBACM_BIN_PATH@
+IBACM_CONFIG_PATH = @IBACM_CONFIG_PATH@
+IBACM_LIB_PATH = @IBACM_LIB_PATH@
INSTALL = @INSTALL@
INSTALL_DATA = @INSTALL_DATA@
INSTALL_PROGRAM = @INSTALL_PROGRAM@
@@ -261,20 +277,36 @@ top_build_prefix = @top_build_prefix@
top_builddir = @top_builddir@
top_srcdir = @top_srcdir@
AM_CPPFLAGS = -I$(srcdir)/include -I$(srcdir)/linux -I$(srcdir)/src
-AM_CFLAGS = -g -Wall -D_GNU_SOURCE -DSYSCONFDIR=\"$(sysconfdir)\" -DBINDIR=\"$(bindir)\" -DRDMADIR=\"@rdmadir@\"
+AM_CFLAGS = -g -Wall -D_GNU_SOURCE
+AM_LDFLAGS = -lpthread -ldl
+ at HAVE_LD_VERSION_SCRIPT_FALSE@libibacmp_version_script =
+ at HAVE_LD_VERSION_SCRIPT_TRUE@libibacmp_version_script = -Wl,--version-script=$(srcdir)/prov/acmp/src/libibacmp.map
svc_ibacm_SOURCES = src/acm.c src/acm_util.c
util_ib_acme_SOURCES = src/acme.c src/libacm.c linux/libacm_linux.c src/parse.c
svc_ibacm_CFLAGS = $(AM_CFLAGS)
+svc_ibacm_LDFLAGS = -rdynamic $(AM_LDFLAGS)
util_ib_acme_CFLAGS = $(AM_CFLAGS) -DACME_PRINTS
+pkglib_LTLIBRARIES = lib/libibacmp.la
+lib_libibacmp_la_CFLAGS = $(AM_CFLAGS)
+lib_libibacmp_la_SOURCES = prov/acmp/src/acmp.c
+lib_libibacmp_la_LDFLAGS = -version-info 1 -export-dynamic \
+ $(libibacmp_version_script) $(AM_LDFLAGS)
+
+lib_libibacmp_la_DEPENDENCIES = $(srcdir)/prov/acmp/src/libibacmp.map
ibacmincludedir = $(includedir)/infiniband
-ibacminclude_HEADERS = include/infiniband/acm.h
+ibacminclude_HEADERS = include/infiniband/acm.h include/infiniband/acm_prov.h
+libibacmpincludedir = $(includedir)/infiniband
+libibacmpinclude_HEADERS = include/infiniband/acm.h include/infiniband/acm_prov.h
man_MANS = \
man/ib_acme.1 \
man/ibacm.1 \
- man/ibacm.7
+ man/ibacm.7 \
+ man/ibacm_prov.7
-EXTRA_DIST = src/acm_util.h src/acm_mad.h src/libacm.h ibacm.init.in \
- linux/osd.h linux/dlist.h ibacm.spec.in $(man_MANS) ibacm_hosts.data
+EXTRA_DIST = src/acm_util.h prov/acmp/src/libibacmp.map \
+ include/acm_mad.h src/libacm.h ibacm.init.in \
+ linux/osd.h linux/dlist.h ibacm.spec.in \
+ $(man_MANS) ibacm_hosts.data
all: config.h
$(MAKE) $(AM_MAKEFLAGS) all-am
@@ -336,6 +368,50 @@ ibacm.spec: $(top_builddir)/config.status $(srcdir)/ibacm.spec.in
cd $(top_builddir) && $(SHELL) ./config.status $@
ibacm.init: $(top_builddir)/config.status $(srcdir)/ibacm.init.in
cd $(top_builddir) && $(SHELL) ./config.status $@
+install-pkglibLTLIBRARIES: $(pkglib_LTLIBRARIES)
+ @$(NORMAL_INSTALL)
+ test -z "$(pkglibdir)" || $(MKDIR_P) "$(DESTDIR)$(pkglibdir)"
+ @list='$(pkglib_LTLIBRARIES)'; test -n "$(pkglibdir)" || list=; \
+ list2=; for p in $$list; do \
+ if test -f $$p; then \
+ list2="$$list2 $$p"; \
+ else :; fi; \
+ done; \
+ test -z "$$list2" || { \
+ echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(pkglibdir)'"; \
+ $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(pkglibdir)"; \
+ }
+
+uninstall-pkglibLTLIBRARIES:
+ @$(NORMAL_UNINSTALL)
+ @list='$(pkglib_LTLIBRARIES)'; test -n "$(pkglibdir)" || list=; \
+ for p in $$list; do \
+ $(am__strip_dir) \
+ echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(pkglibdir)/$$f'"; \
+ $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(pkglibdir)/$$f"; \
+ done
+
+clean-pkglibLTLIBRARIES:
+ -test -z "$(pkglib_LTLIBRARIES)" || rm -f $(pkglib_LTLIBRARIES)
+ @list='$(pkglib_LTLIBRARIES)'; for p in $$list; do \
+ dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \
+ test "$$dir" != "$$p" || dir=.; \
+ echo "rm -f \"$${dir}/so_locations\""; \
+ rm -f "$${dir}/so_locations"; \
+ done
+prov/acmp/src/$(am__dirstamp):
+ @$(MKDIR_P) prov/acmp/src
+ @: > prov/acmp/src/$(am__dirstamp)
+prov/acmp/src/$(DEPDIR)/$(am__dirstamp):
+ @$(MKDIR_P) prov/acmp/src/$(DEPDIR)
+ @: > prov/acmp/src/$(DEPDIR)/$(am__dirstamp)
+prov/acmp/src/lib_libibacmp_la-acmp.lo: prov/acmp/src/$(am__dirstamp) \
+ prov/acmp/src/$(DEPDIR)/$(am__dirstamp)
+lib/$(am__dirstamp):
+ @$(MKDIR_P) lib
+ @: > lib/$(am__dirstamp)
+lib/libibacmp.la: $(lib_libibacmp_la_OBJECTS) $(lib_libibacmp_la_DEPENDENCIES) lib/$(am__dirstamp)
+ $(AM_V_CCLD)$(lib_libibacmp_la_LINK) -rpath $(pkglibdir) $(lib_libibacmp_la_OBJECTS) $(lib_libibacmp_la_LIBADD) $(LIBS)
install-binPROGRAMS: $(bin_PROGRAMS)
@$(NORMAL_INSTALL)
test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)"
@@ -462,6 +538,8 @@ util/ib_acme$(EXEEXT): $(util_ib_acme_OBJECTS) $(util_ib_acme_DEPENDENCIES) util
mostlyclean-compile:
-rm -f *.$(OBJEXT)
-rm -f linux/util_ib_acme-libacm_linux.$(OBJEXT)
+ -rm -f prov/acmp/src/lib_libibacmp_la-acmp.$(OBJEXT)
+ -rm -f prov/acmp/src/lib_libibacmp_la-acmp.lo
-rm -f src/svc_ibacm-acm.$(OBJEXT)
-rm -f src/svc_ibacm-acm_util.$(OBJEXT)
-rm -f src/util_ib_acme-acme.$(OBJEXT)
@@ -472,6 +550,7 @@ distclean-compile:
-rm -f *.tab.c
@AMDEP_TRUE@@am__include@ @am__quote at linux/$(DEPDIR)/util_ib_acme-libacm_linux.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at prov/acmp/src/$(DEPDIR)/lib_libibacmp_la-acmp.Plo at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/svc_ibacm-acm.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/svc_ibacm-acm_util.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at src/$(DEPDIR)/util_ib_acme-acme.Po at am__quote@
@@ -505,6 +584,14 @@ distclean-compile:
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCC_FALSE@ $(LTCOMPILE) -c -o $@ $<
+prov/acmp/src/lib_libibacmp_la-acmp.lo: prov/acmp/src/acmp.c
+ at am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_libibacmp_la_CFLAGS) $(CFLAGS) -MT prov/acmp/src/lib_libibacmp_la-acmp.lo -MD -MP -MF prov/acmp/src/$(DEPDIR)/lib_libibacmp_la-acmp.Tpo -c -o prov/acmp/src/lib_libibacmp_la-acmp.lo `test -f 'prov/acmp/src/acmp.c' || echo '$(srcdir)/'`prov/acmp/src/acmp.c
+ at am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) prov/acmp/src/$(DEPDIR)/lib_libibacmp_la-acmp.Tpo prov/acmp/src/$(DEPDIR)/lib_libibacmp_la-acmp.Plo
+ at am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ source='prov/acmp/src/acmp.c' object='prov/acmp/src/lib_libibacmp_la-acmp.lo' libtool=yes @AMDEPBACKSLASH@
+ at AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+ at am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_libibacmp_la_CFLAGS) $(CFLAGS) -c -o prov/acmp/src/lib_libibacmp_la-acmp.lo `test -f 'prov/acmp/src/acmp.c' || echo '$(srcdir)/'`prov/acmp/src/acmp.c
+
src/svc_ibacm-acm.o: src/acm.c
@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(svc_ibacm_CFLAGS) $(CFLAGS) -MT src/svc_ibacm-acm.o -MD -MP -MF src/$(DEPDIR)/svc_ibacm-acm.Tpo -c -o src/svc_ibacm-acm.o `test -f 'src/acm.c' || echo '$(srcdir)/'`src/acm.c
@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) src/$(DEPDIR)/svc_ibacm-acm.Tpo src/$(DEPDIR)/svc_ibacm-acm.Po
@@ -606,6 +693,8 @@ mostlyclean-libtool:
clean-libtool:
-rm -rf .libs _libs
+ -rm -rf lib/.libs lib/_libs
+ -rm -rf prov/acmp/src/.libs prov/acmp/src/_libs
-rm -rf svc/.libs svc/_libs
-rm -rf util/.libs util/_libs
@@ -707,6 +796,26 @@ uninstall-ibacmincludeHEADERS:
test -n "$$files" || exit 0; \
echo " ( cd '$(DESTDIR)$(ibacmincludedir)' && rm -f" $$files ")"; \
cd "$(DESTDIR)$(ibacmincludedir)" && rm -f $$files
+install-libibacmpincludeHEADERS: $(libibacmpinclude_HEADERS)
+ @$(NORMAL_INSTALL)
+ test -z "$(libibacmpincludedir)" || $(MKDIR_P) "$(DESTDIR)$(libibacmpincludedir)"
+ @list='$(libibacmpinclude_HEADERS)'; test -n "$(libibacmpincludedir)" || list=; \
+ for p in $$list; do \
+ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; \
+ done | $(am__base_list) | \
+ while read files; do \
+ echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(libibacmpincludedir)'"; \
+ $(INSTALL_HEADER) $$files "$(DESTDIR)$(libibacmpincludedir)" || exit $$?; \
+ done
+
+uninstall-libibacmpincludeHEADERS:
+ @$(NORMAL_UNINSTALL)
+ @list='$(libibacmpinclude_HEADERS)'; test -n "$(libibacmpincludedir)" || list=; \
+ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+ test -n "$$files" || exit 0; \
+ echo " ( cd '$(DESTDIR)$(libibacmpincludedir)' && rm -f" $$files ")"; \
+ cd "$(DESTDIR)$(libibacmpincludedir)" && rm -f $$files
ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
@@ -925,9 +1034,10 @@ distcleancheck: distclean
exit 1; } >&2
check-am: all-am
check: check-am
-all-am: Makefile $(PROGRAMS) $(MANS) $(HEADERS) config.h
+all-am: Makefile $(LTLIBRARIES) $(PROGRAMS) $(MANS) $(HEADERS) \
+ config.h
installdirs:
- for dir in "$(DESTDIR)$(bindir)" "$(DESTDIR)$(sbindir)" "$(DESTDIR)$(man1dir)" "$(DESTDIR)$(man7dir)" "$(DESTDIR)$(ibacmincludedir)"; do \
+ for dir in "$(DESTDIR)$(pkglibdir)" "$(DESTDIR)$(bindir)" "$(DESTDIR)$(sbindir)" "$(DESTDIR)$(man1dir)" "$(DESTDIR)$(man7dir)" "$(DESTDIR)$(ibacmincludedir)" "$(DESTDIR)$(libibacmpincludedir)"; do \
test -z "$$dir" || $(MKDIR_P) "$$dir"; \
done
install: install-am
@@ -951,8 +1061,11 @@ clean-generic:
distclean-generic:
-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+ -rm -f lib/$(am__dirstamp)
-rm -f linux/$(DEPDIR)/$(am__dirstamp)
-rm -f linux/$(am__dirstamp)
+ -rm -f prov/acmp/src/$(DEPDIR)/$(am__dirstamp)
+ -rm -f prov/acmp/src/$(am__dirstamp)
-rm -f src/$(DEPDIR)/$(am__dirstamp)
-rm -f src/$(am__dirstamp)
-rm -f svc/$(am__dirstamp)
@@ -964,11 +1077,11 @@ maintainer-clean-generic:
clean: clean-am
clean-am: clean-binPROGRAMS clean-generic clean-libtool \
- clean-sbinPROGRAMS mostlyclean-am
+ clean-pkglibLTLIBRARIES clean-sbinPROGRAMS mostlyclean-am
distclean: distclean-am
-rm -f $(am__CONFIG_DISTCLEAN_FILES)
- -rm -rf linux/$(DEPDIR) src/$(DEPDIR)
+ -rm -rf linux/$(DEPDIR) prov/acmp/src/$(DEPDIR) src/$(DEPDIR)
-rm -f Makefile
distclean-am: clean-am distclean-compile distclean-generic \
distclean-hdr distclean-libtool distclean-tags
@@ -985,13 +1098,15 @@ info: info-am
info-am:
-install-data-am: install-ibacmincludeHEADERS install-man
+install-data-am: install-ibacmincludeHEADERS \
+ install-libibacmpincludeHEADERS install-man
install-dvi: install-dvi-am
install-dvi-am:
-install-exec-am: install-binPROGRAMS install-sbinPROGRAMS
+install-exec-am: install-binPROGRAMS install-pkglibLTLIBRARIES \
+ install-sbinPROGRAMS
@$(NORMAL_INSTALL)
$(MAKE) $(AM_MAKEFLAGS) install-exec-hook
install-html: install-html-am
@@ -1017,7 +1132,7 @@ installcheck-am:
maintainer-clean: maintainer-clean-am
-rm -f $(am__CONFIG_DISTCLEAN_FILES)
-rm -rf $(top_srcdir)/autom4te.cache
- -rm -rf linux/$(DEPDIR) src/$(DEPDIR)
+ -rm -rf linux/$(DEPDIR) prov/acmp/src/$(DEPDIR) src/$(DEPDIR)
-rm -f Makefile
maintainer-clean-am: distclean-am maintainer-clean-generic
@@ -1035,7 +1150,8 @@ ps: ps-am
ps-am:
uninstall-am: uninstall-binPROGRAMS uninstall-ibacmincludeHEADERS \
- uninstall-man uninstall-sbinPROGRAMS
+ uninstall-libibacmpincludeHEADERS uninstall-man \
+ uninstall-pkglibLTLIBRARIES uninstall-sbinPROGRAMS
uninstall-man: uninstall-man1 uninstall-man7
@@ -1043,23 +1159,26 @@ uninstall-man: uninstall-man1 uninstall-man7
.PHONY: CTAGS GTAGS all all-am am--refresh check check-am clean \
clean-binPROGRAMS clean-generic clean-libtool \
- clean-sbinPROGRAMS ctags dist dist-all dist-bzip2 dist-gzip \
- dist-hook dist-lzma dist-shar dist-tarZ dist-xz dist-zip \
- distcheck distclean distclean-compile distclean-generic \
- distclean-hdr distclean-libtool distclean-tags distcleancheck \
- distdir distuninstallcheck dvi dvi-am html html-am info \
- info-am install install-am install-binPROGRAMS install-data \
- install-data-am install-dvi install-dvi-am install-exec \
- install-exec-am install-exec-hook install-html install-html-am \
- install-ibacmincludeHEADERS install-info install-info-am \
+ clean-pkglibLTLIBRARIES clean-sbinPROGRAMS ctags dist dist-all \
+ dist-bzip2 dist-gzip dist-hook dist-lzma dist-shar dist-tarZ \
+ dist-xz dist-zip distcheck distclean distclean-compile \
+ distclean-generic distclean-hdr distclean-libtool \
+ distclean-tags distcleancheck distdir distuninstallcheck dvi \
+ dvi-am html html-am info info-am install install-am \
+ install-binPROGRAMS install-data install-data-am install-dvi \
+ install-dvi-am install-exec install-exec-am install-exec-hook \
+ install-html install-html-am install-ibacmincludeHEADERS \
+ install-info install-info-am install-libibacmpincludeHEADERS \
install-man install-man1 install-man7 install-pdf \
- install-pdf-am install-ps install-ps-am install-sbinPROGRAMS \
- install-strip installcheck installcheck-am installdirs \
- maintainer-clean maintainer-clean-generic mostlyclean \
- mostlyclean-compile mostlyclean-generic mostlyclean-libtool \
- pdf pdf-am ps ps-am tags uninstall uninstall-am \
- uninstall-binPROGRAMS uninstall-ibacmincludeHEADERS \
- uninstall-man uninstall-man1 uninstall-man7 \
+ install-pdf-am install-pkglibLTLIBRARIES install-ps \
+ install-ps-am install-sbinPROGRAMS install-strip installcheck \
+ installcheck-am installdirs maintainer-clean \
+ maintainer-clean-generic mostlyclean mostlyclean-compile \
+ mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \
+ tags uninstall uninstall-am uninstall-binPROGRAMS \
+ uninstall-ibacmincludeHEADERS \
+ uninstall-libibacmpincludeHEADERS uninstall-man uninstall-man1 \
+ uninstall-man7 uninstall-pkglibLTLIBRARIES \
uninstall-sbinPROGRAMS
diff --git a/config.h.in b/config.h.in
index 8559f48..283fa5c 100644
--- a/config.h.in
+++ b/config.h.in
@@ -39,6 +39,15 @@
/* Define to 1 if you have the <unistd.h> header file. */
#undef HAVE_UNISTD_H
+/* Define the path to bin directory */
+#undef IBACM_BIN_PATH
+
+/* Define the path to configurations */
+#undef IBACM_CONFIG_PATH
+
+/* Define the path to the provider lib directory */
+#undef IBACM_LIB_PATH
+
/* Define to the sub-directory in which libtool stores uninstalled libraries.
*/
#undef LT_OBJDIR
diff --git a/configure b/configure
index 84eed24..7883c75 100755
--- a/configure
+++ b/configure
@@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.67 for ibacm 1.0.9.
+# Generated by GNU Autoconf 2.67 for ibacm 1.1.0.
#
# Report bugs to <linux-rdma at vger.kernel.org>.
#
@@ -701,8 +701,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='ibacm'
PACKAGE_TARNAME='ibacm'
-PACKAGE_VERSION='1.0.9'
-PACKAGE_STRING='ibacm 1.0.9'
+PACKAGE_VERSION='1.1.0'
+PACKAGE_STRING='ibacm 1.1.0'
PACKAGE_BUGREPORT='linux-rdma at vger.kernel.org'
PACKAGE_URL=''
@@ -751,6 +751,9 @@ rdmascript
rdmadir
HAVE_LD_VERSION_SCRIPT_FALSE
HAVE_LD_VERSION_SCRIPT_TRUE
+IBACM_LIB_PATH
+IBACM_BIN_PATH
+IBACM_CONFIG_PATH
CPP
OTOOL64
OTOOL
@@ -862,8 +865,8 @@ ac_subst_files=''
ac_user_opts='
enable_option_checking
enable_silent_rules
-enable_shared
enable_static
+enable_shared
with_pic
enable_fast_install
enable_dependency_tracking
@@ -1424,7 +1427,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-\`configure' configures ibacm 1.0.9 to adapt to many kinds of systems.
+\`configure' configures ibacm 1.1.0 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1494,7 +1497,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of ibacm 1.0.9:";;
+ short | recursive ) echo "Configuration of ibacm 1.1.0:";;
esac
cat <<\_ACEOF
@@ -1504,8 +1507,8 @@ Optional Features:
--enable-FEATURE[=ARG] include FEATURE [ARG=yes]
--enable-silent-rules less verbose build output (undo: `make V=1')
--disable-silent-rules verbose build output (undo: `make V=0')
+ --enable-static[=PKGS] build static libraries [default=no]
--enable-shared[=PKGS] build shared libraries [default=yes]
- --enable-static[=PKGS] build static libraries [default=yes]
--enable-fast-install[=PKGS]
optimize for fast installation [default=yes]
--disable-dependency-tracking speeds up one-time build
@@ -1598,7 +1601,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
-ibacm configure 1.0.9
+ibacm configure 1.1.0
generated by GNU Autoconf 2.67
Copyright (C) 2010 Free Software Foundation, Inc.
@@ -2145,7 +2148,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by ibacm $as_me 1.0.9, which was
+It was created by ibacm $as_me 1.1.0, which was
generated by GNU Autoconf 2.67. Invocation command line was
$ $0 $@
@@ -2963,7 +2966,7 @@ fi
# Define the identity of the package.
PACKAGE='ibacm'
- VERSION='1.0.9'
+ VERSION='1.1.0'
cat >>confdefs.h <<_ACEOF
@@ -4675,13 +4678,13 @@ if test "${lt_cv_nm_interface+set}" = set; then :
else
lt_cv_nm_interface="BSD nm"
echo "int some_variable = 0;" > conftest.$ac_ext
- (eval echo "\"\$as_me:4678: $ac_compile\"" >&5)
+ (eval echo "\"\$as_me:4681: $ac_compile\"" >&5)
(eval "$ac_compile" 2>conftest.err)
cat conftest.err >&5
- (eval echo "\"\$as_me:4681: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+ (eval echo "\"\$as_me:4684: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
(eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
cat conftest.err >&5
- (eval echo "\"\$as_me:4684: output\"" >&5)
+ (eval echo "\"\$as_me:4687: output\"" >&5)
cat conftest.out >&5
if $GREP 'External.*some_variable' conftest.out > /dev/null; then
lt_cv_nm_interface="MS dumpbin"
@@ -5887,7 +5890,7 @@ ia64-*-hpux*)
;;
*-*-irix6*)
# Find out which ABI we are using.
- echo '#line 5890 "configure"' > conftest.$ac_ext
+ echo '#line 5893 "configure"' > conftest.$ac_ext
if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
(eval $ac_compile) 2>&5
ac_status=$?
@@ -6917,36 +6920,27 @@ done
# Set options
-
-
-
- enable_dlopen=no
-
-
- enable_win32_dll=no
-
-
- # Check whether --enable-shared was given.
-if test "${enable_shared+set}" = set; then :
- enableval=$enable_shared; p=${PACKAGE-default}
+# Check whether --enable-static was given.
+if test "${enable_static+set}" = set; then :
+ enableval=$enable_static; p=${PACKAGE-default}
case $enableval in
- yes) enable_shared=yes ;;
- no) enable_shared=no ;;
+ yes) enable_static=yes ;;
+ no) enable_static=no ;;
*)
- enable_shared=no
+ enable_static=no
# Look at the argument we got. We use all the common list separators.
lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR,"
for pkg in $enableval; do
IFS="$lt_save_ifs"
if test "X$pkg" = "X$p"; then
- enable_shared=yes
+ enable_static=yes
fi
done
IFS="$lt_save_ifs"
;;
esac
else
- enable_shared=yes
+ enable_static=no
fi
@@ -6957,27 +6951,34 @@ fi
- # Check whether --enable-static was given.
-if test "${enable_static+set}" = set; then :
- enableval=$enable_static; p=${PACKAGE-default}
+
+ enable_dlopen=no
+
+
+ enable_win32_dll=no
+
+
+ # Check whether --enable-shared was given.
+if test "${enable_shared+set}" = set; then :
+ enableval=$enable_shared; p=${PACKAGE-default}
case $enableval in
- yes) enable_static=yes ;;
- no) enable_static=no ;;
+ yes) enable_shared=yes ;;
+ no) enable_shared=no ;;
*)
- enable_static=no
+ enable_shared=no
# Look at the argument we got. We use all the common list separators.
lt_save_ifs="$IFS"; IFS="${IFS}$PATH_SEPARATOR,"
for pkg in $enableval; do
IFS="$lt_save_ifs"
if test "X$pkg" = "X$p"; then
- enable_static=yes
+ enable_shared=yes
fi
done
IFS="$lt_save_ifs"
;;
esac
else
- enable_static=yes
+ enable_shared=yes
fi
@@ -6989,6 +6990,7 @@ fi
+
# Check whether --with-pic was given.
if test "${with_pic+set}" = set; then :
withval=$with_pic; pic_mode="$withval"
@@ -7412,11 +7414,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:7415: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:7417: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
- echo "$as_me:7419: \$? = $ac_status" >&5
+ echo "$as_me:7421: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -7751,11 +7753,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:7754: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:7756: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
- echo "$as_me:7758: \$? = $ac_status" >&5
+ echo "$as_me:7760: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -7856,11 +7858,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:7859: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:7861: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
- echo "$as_me:7863: \$? = $ac_status" >&5
+ echo "$as_me:7865: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -7911,11 +7913,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:7914: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:7916: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
- echo "$as_me:7918: \$? = $ac_status" >&5
+ echo "$as_me:7920: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -10281,7 +10283,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
-#line 10284 "configure"
+#line 10286 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@@ -10377,7 +10379,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
-#line 10380 "configure"
+#line 10382 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@@ -11493,6 +11495,36 @@ fi
fi
+IBACM_CONFIG_PATH_TMP1="`eval echo ${sysconfdir}`"
+IBACM_CONFIG_PATH_TMP2="`echo $IBACM_CONFIG_PATH_TMP1 | sed 's/^NONE/$ac_default_prefix/'`"
+IBACM_CONFIG_PATH="`eval echo $IBACM_CONFIG_PATH_TMP2`/rdma"
+
+
+cat >>confdefs.h <<_ACEOF
+#define IBACM_CONFIG_PATH "$IBACM_CONFIG_PATH"
+_ACEOF
+
+
+IBACM_BIN_PATH_TMP1="`eval echo ${bindir}`"
+IBACM_BIN_PATH_TMP2="`echo $IBACM_BIN_PATH_TMP1 | sed 's/^NONE/$ac_default_prefix/'`"
+IBACM_BIN_PATH="`eval echo $IBACM_BIN_PATH_TMP2`"
+
+
+cat >>confdefs.h <<_ACEOF
+#define IBACM_BIN_PATH "$IBACM_BIN_PATH"
+_ACEOF
+
+
+IBACM_LIB_PATH_TMP1="`eval echo ${libdir}`"
+IBACM_LIB_PATH_TMP2="`echo $IBACM_LIB_PATH_TMP1 | sed 's/^NONE/$ac_default_prefix/'`"
+IBACM_LIB_PATH="`eval echo $IBACM_LIB_PATH_TMP2`/ibacm"
+
+
+cat >>confdefs.h <<_ACEOF
+#define IBACM_LIB_PATH "$IBACM_LIB_PATH"
+_ACEOF
+
+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for ANSI C header files" >&5
$as_echo_n "checking for ANSI C header files... " >&6; }
if test "${ac_cv_header_stdc+set}" = set; then :
@@ -12225,7 +12257,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
-This file was extended by ibacm $as_me 1.0.9, which was
+This file was extended by ibacm $as_me 1.1.0, which was
generated by GNU Autoconf 2.67. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -12291,7 +12323,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
-ibacm config.status 1.0.9
+ibacm config.status 1.1.0
configured by $0, generated by GNU Autoconf 2.67,
with options \\"\$ac_cs_config\\"
@@ -12422,8 +12454,8 @@ double_quote_subst='$double_quote_subst'
delay_variable_subst='$delay_variable_subst'
macro_version='`$ECHO "X$macro_version" | $Xsed -e "$delay_single_quote_subst"`'
macro_revision='`$ECHO "X$macro_revision" | $Xsed -e "$delay_single_quote_subst"`'
-enable_shared='`$ECHO "X$enable_shared" | $Xsed -e "$delay_single_quote_subst"`'
enable_static='`$ECHO "X$enable_static" | $Xsed -e "$delay_single_quote_subst"`'
+enable_shared='`$ECHO "X$enable_shared" | $Xsed -e "$delay_single_quote_subst"`'
pic_mode='`$ECHO "X$pic_mode" | $Xsed -e "$delay_single_quote_subst"`'
enable_fast_install='`$ECHO "X$enable_fast_install" | $Xsed -e "$delay_single_quote_subst"`'
host_alias='`$ECHO "X$host_alias" | $Xsed -e "$delay_single_quote_subst"`'
@@ -13425,12 +13457,12 @@ available_tags=""
macro_version=$macro_version
macro_revision=$macro_revision
-# Whether or not to build shared libraries.
-build_libtool_libs=$enable_shared
-
# Whether or not to build static libraries.
build_old_libs=$enable_static
+# Whether or not to build shared libraries.
+build_libtool_libs=$enable_shared
+
# What type of objects to build.
pic_mode=$pic_mode
diff --git a/configure.ac b/configure.ac
index d297c30..4d493bd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,14 +1,14 @@
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ([2.63])
-AC_INIT([ibacm], [1.0.9], [linux-rdma at vger.kernel.org])
+AC_INIT([ibacm], [1.1.0], [linux-rdma at vger.kernel.org])
AC_CONFIG_SRCDIR([src/acm.c])
AC_CONFIG_AUX_DIR(config)
AC_CONFIG_HEADERS([config.h])
AM_INIT_AUTOMAKE([foreign subdir-objects])
m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
-LT_INIT
+LT_INIT(disable-static)
AC_ARG_ENABLE(libcheck, [ --disable-libcheck do not test for presence of ib libraries],
[ if test "$enableval" = "no"; then
@@ -31,6 +31,27 @@ AC_CHECK_LIB(ibumad, umad_send, [],
AC_MSG_ERROR([umad_send() not found. ibacm requires libibumad.]))
fi
+dnl Define a configure directory
+IBACM_CONFIG_PATH_TMP1="`eval echo ${sysconfdir}`"
+IBACM_CONFIG_PATH_TMP2="`echo $IBACM_CONFIG_PATH_TMP1 | sed 's/^NONE/$ac_default_prefix/'`"
+IBACM_CONFIG_PATH="`eval echo $IBACM_CONFIG_PATH_TMP2`/rdma"
+AC_SUBST(IBACM_CONFIG_PATH)
+AC_DEFINE_UNQUOTED([IBACM_CONFIG_PATH], "$IBACM_CONFIG_PATH", [Define the path to configurations])
+
+dnl Define the bin directory
+IBACM_BIN_PATH_TMP1="`eval echo ${bindir}`"
+IBACM_BIN_PATH_TMP2="`echo $IBACM_BIN_PATH_TMP1 | sed 's/^NONE/$ac_default_prefix/'`"
+IBACM_BIN_PATH="`eval echo $IBACM_BIN_PATH_TMP2`"
+AC_SUBST(IBACM_BIN_PATH)
+AC_DEFINE_UNQUOTED([IBACM_BIN_PATH], "$IBACM_BIN_PATH", [Define the path to bin directory])
+
+dnl Define the lib directory
+IBACM_LIB_PATH_TMP1="`eval echo ${libdir}`"
+IBACM_LIB_PATH_TMP2="`echo $IBACM_LIB_PATH_TMP1 | sed 's/^NONE/$ac_default_prefix/'`"
+IBACM_LIB_PATH="`eval echo $IBACM_LIB_PATH_TMP2`/ibacm"
+AC_SUBST(IBACM_LIB_PATH)
+AC_DEFINE_UNQUOTED([IBACM_LIB_PATH], "$IBACM_LIB_PATH", [Define the path to the provider lib directory])
+
dnl Checks for header files.
AC_HEADER_STDC
if test "$disable_libcheck" != "yes"; then
diff --git a/ibacm.spec b/ibacm.spec
index 8d8916e..0167e26 100644
--- a/ibacm.spec
+++ b/ibacm.spec
@@ -1,5 +1,5 @@
Name: ibacm
-Version: 1.0.9
+Version: 1.1.0
Release: 1%{?dist}
Summary: InfiniBand Communication Manager Assistant
@@ -76,10 +76,12 @@ fi
%{_mandir}/man1/*
%{_mandir}/man7/*
%{_sysconfdir}/init.d/ibacm
+%{_libdir}/ibacm/lib*.*
%files devel
%defattr(-,root,root,-)
%{_includedir}/infiniband/acm.h
+%{_includedir}/infiniband/acm_prov.h
%changelog
* Tue Feb 28 2012 Doug Ledford <dledford at redhat.com> - 1.0.5-1
diff --git a/ibacm.spec.in b/ibacm.spec.in
index 8d8916e..0167e26 100644
--- a/ibacm.spec.in
+++ b/ibacm.spec.in
@@ -1,5 +1,5 @@
Name: ibacm
-Version: 1.0.9
+Version: 1.1.0
Release: 1%{?dist}
Summary: InfiniBand Communication Manager Assistant
@@ -76,10 +76,12 @@ fi
%{_mandir}/man1/*
%{_mandir}/man7/*
%{_sysconfdir}/init.d/ibacm
+%{_libdir}/ibacm/lib*.*
%files devel
%defattr(-,root,root,-)
%{_includedir}/infiniband/acm.h
+%{_includedir}/infiniband/acm_prov.h
%changelog
* Tue Feb 28 2012 Doug Ledford <dledford at redhat.com> - 1.0.5-1
diff --git a/src/acm_mad.h b/include/acm_mad.h
similarity index 100%
rename from src/acm_mad.h
rename to include/acm_mad.h
diff --git a/include/infiniband/acm.h b/include/infiniband/acm.h
index bc85137..b4d677c 100644
--- a/include/infiniband/acm.h
+++ b/include/infiniband/acm.h
@@ -38,6 +38,7 @@
#define ACM_OP_MASK 0x0F
#define ACM_OP_RESOLVE 0x01
#define ACM_OP_PERF_QUERY 0x02
+#define ACM_OP_EP_QUERY 0x03
#define ACM_OP_ACK 0x80
#define ACM_STATUS_SUCCESS 0
@@ -57,6 +58,7 @@
#define ACM_MSG_HDR_LENGTH 16
#define ACM_MAX_ADDRESS 64
#define ACM_MSG_EP_LENGTH 72
+#define ACM_MAX_PROV_NAME 64
/*
* Support up to 6 path records (primary and alternate CM paths,
* inbound and outbound primary and alternate data paths), plus CM data.
@@ -116,19 +118,38 @@ enum {
};
/*
- * Performance messages are sent/receive in network byte order.
+ * Performance messages are sent/received in network byte order.
*/
struct acm_perf_msg {
struct acm_hdr hdr;
uint64_t data[0];
};
+/*
+ * Endpoint query messages are sent/received in network byte order.
+ */
+struct acm_ep_config_data {
+ uint64_t dev_guid;
+ uint8_t port_num;
+ uint8_t rsvd[3];
+ uint16_t pkey;
+ uint16_t addr_cnt;
+ uint8_t prov_name[ACM_MAX_PROV_NAME];
+ union acm_ep_info addrs[0];
+};
+
+struct acm_ep_query_msg {
+ struct acm_hdr hdr;
+ struct acm_ep_config_data data[0];
+};
+
struct acm_msg {
struct acm_hdr hdr;
union{
uint8_t data[ACM_MSG_DATA_LENGTH];
struct acm_ep_addr_data resolve_data[0];
uint64_t perf_data[0];
+ struct acm_ep_config_data ep_data[0];
};
};
diff --git a/include/infiniband/acm_prov.h b/include/infiniband/acm_prov.h
new file mode 100644
index 0000000..890e6ba
--- /dev/null
+++ b/include/infiniband/acm_prov.h
@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2014 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under the OpenFabrics.org BSD license
+ * below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if !defined(ACM_PROV_H)
+#define ACM_PROV_H
+
+#include <infiniband/acm.h>
+#include <infiniband/umad.h>
+#include <infiniband/umad_sa.h>
+
+#define ACM_PROV_VERSION 1
+
+struct acm_device {
+ struct ibv_context *verbs;
+ uint64_t dev_guid;
+};
+
+struct acm_port {
+ struct acm_device *dev;
+ uint8_t port_num;
+};
+
+struct acm_endpoint {
+ struct acm_port *port;
+ uint16_t pkey;
+};
+
+struct acm_address {
+ struct acm_endpoint *endpoint;
+ union acm_ep_info info;
+ char *id_string;
+ uint16_t type;
+};
+
+struct acm_provider {
+ size_t size;
+ uint32_t version;
+ char *name;
+ int (*open_device)(const struct acm_device *device,
+ void **dev_context);
+ void (*close_device)(void *dev_context);
+ int (*open_port)(const struct acm_port *port,
+ void *dev_context, void **port_context);
+ void (*close_port)(void *port_context);
+ int (*open_endpoint)(const struct acm_endpoint *endpoint,
+ void *port_context, void **ep_context);
+ void (*close_endpoint)(void *ep_context);
+ int (*add_address)(const struct acm_address *addr, void *ep_context,
+ void **addr_context);
+ void (*remove_address)(void *addr_context);
+ int (*resolve)(void *addr_context, struct acm_msg *msg, uint64_t id);
+ int (*query)(void *addr_context, struct acm_msg *msg, uint64_t id);
+ int (*handle_event)(void *port_context, enum ibv_event_type type);
+ void (*query_perf)(void *ep_context, uint64_t *values, uint8_t *cnt);
+};
+
+int provider_query(struct acm_provider **info, uint32_t *version);
+
+/* Functions exported from core */
+#define acm_log(level, format, ...) \
+ acm_write(level, "%s: "format, __func__, ## __VA_ARGS__)
+extern void acm_write(int level, const char *format, ...);
+extern void acm_format_name(int level, char *name, size_t name_size,
+ uint8_t addr_type, const uint8_t *addr, size_t addr_size);
+
+extern int ib_any_gid(union ibv_gid *gid);
+extern uint8_t acm_gid_index(struct acm_port *port, union ibv_gid *gid);
+extern int acm_get_gid(struct acm_port *port, int index, union ibv_gid *gid);
+extern uint64_t acm_path_comp_mask(struct ibv_path_record *path);
+
+extern int acm_resolve_response(uint64_t id, struct acm_msg *msg);
+extern int acm_query_response(uint64_t id, struct acm_msg *msg);
+
+extern enum ibv_rate acm_get_rate(uint8_t width, uint8_t speed);
+extern enum ibv_mtu acm_convert_mtu(int mtu);
+extern enum ibv_rate acm_convert_rate(int rate);
+
+struct acm_sa_mad {
+ void *context;
+ struct ib_user_mad umad;
+ struct umad_sa_packet sa_mad; /* must follow umad and be 64-bit aligned */
+};
+
+extern struct acm_sa_mad *
+acm_alloc_sa_mad(const struct acm_endpoint *endpoint, void *context,
+ void (*handler)(struct acm_sa_mad *));
+extern void acm_free_sa_mad(struct acm_sa_mad *mad);
+extern int acm_send_sa_mad(struct acm_sa_mad *mad);
+
+extern char * acm_get_opts_file(void);
+extern void acm_increment_counter(int type);
+
+#endif /* ACM_PROV_H */
diff --git a/linux/osd.h b/linux/osd.h
index 05c7481..5ca4c6f 100644
--- a/linux/osd.h
+++ b/linux/osd.h
@@ -46,16 +46,7 @@
#include <sys/time.h>
#include <netinet/in.h>
-#ifndef SYSCONFDIR
-#define SYSCONFDIR "/etc"
-#endif
-#ifndef BINDIR
-#define BINDIR "/usr/bin"
-#endif
-#ifndef RDMADIR
-#define RDMADIR "rdma"
-#endif
-#define ACM_CONF_DIR SYSCONFDIR "/" RDMADIR
+#define ACM_CONF_DIR IBACM_CONFIG_PATH
#define ACM_ADDR_FILE "ibacm_addr.cfg"
#define ACM_OPTS_FILE "ibacm_opts.cfg"
diff --git a/man/ib_acme.1 b/man/ib_acme.1
index baa1dd6..530cd16 100644
--- a/man/ib_acme.1
+++ b/man/ib_acme.1
@@ -1,10 +1,10 @@
-.TH "ib_acme" 7 "2013-06-21" "ib_acme" "ib_acme" ib_acme
+.TH "ib_acme" 1 "2014-06-16" "ib_acme" "ib_acme" ib_acme
.SH NAME
ib_acme \- test and configuration utility for the IB ACM
.SH SYNOPSIS
.sp
.nf
-\fIib_acme\fR [-f addr_format] [-s src_addr] -d dest_addr [-v] [-c] [-P] [-S svc_addr] [-C repetitions]
+\fIib_acme\fR [-f addr_format] [-s src_addr] -d dest_addr [-v] [-c] [-e] [-P] [-S svc_addr] [-C repetitions]
.fi
.nf
\fIib_acme\fR [-A [addr_file]] [-O [opt_file]] [-D dest_dir] [-V]
@@ -42,8 +42,15 @@ resolved path information is usable given the current cluster configuration.
Instructs the ACM service to only returned information that currently resides
in its local cache.
.TP
-\-P
-Queries performance data from the destination service
+\-e [N]
+Displays one (N = 1, 2, ...) or all endpoints (N = 0 or not present).
+.TP
+\-P [opt]
+Queries performance data from the destination service. Valid options are:
+"col" for outputting combined data in column format, "N" (N = 1, 2, ...) for
+outputing data for a specific endpoint N, "all" for outputting data for all
+endpoints, and "s" for outputting data for a specific endpoint with the address
+given by the -s option.
.TP
\-S svc_addr
address of ACM service, default: local service
diff --git a/man/ibacm.1 b/man/ibacm.1
index fb15752..29ecd86 100644
--- a/man/ibacm.1
+++ b/man/ibacm.1
@@ -1,4 +1,4 @@
-.TH "ibacm" 1 "2013-07-18" "ibacm" "ibacm" ibacm
+.TH "ibacm" 1 "2014-06-16" "ibacm" "ibacm" ibacm
.SH NAME
ibacm \- address and route resolution services for InfiniBand.
.SH SYNOPSIS
@@ -25,14 +25,17 @@ however existing applications should still see significant connection
scaling benefits using the calls
available in librdmacm 1.0.11 and previous releases.
.P
-The IB ACM is focused on being scalable and efficient. The current
-implementation limits network traffic, SA interactions, and centralized
-services. ACM supports multiple resolution protocols in order to handle
+The IB ACM is focused on being scalable, efficient, and extensible. It implements
+a plugin architecture that allows a vendor to supply its proprietary provider in
+addition to the default provider. The current default provider implementation
+ibacmp limits network traffic, SA interactions, and centralized
+services. Ibacmp supports multiple resolution protocols in order to handle
different fabric topologies.
.P
-The IB ACM package is comprised of two components: the ibacm service
-and a test/configuration utility - ib_acme. Both are userspace components
-and are available for Linux and Windows. Additional details are given below.
+The IB ACM package is comprised of three components: the ibacm core service,
+the default provider ibacmp shared library, and a test/configuration utility
+- ib_acme. All three are userspace components and are available for Linux.
+Additional details are given below.
.SH "OPTIONS"
.TP
\-D
@@ -51,7 +54,7 @@ option configuration file
The IB stack should be running with IPoIB configured.
These steps assume that the user has administrative privileges.
.P
-2. Install the IB ACM package. This installs ibacm, ib_acme, and init.d scripts.
+2. Install the IB ACM package. This installs ibacm, ibacmp, ib_acme, and init.d scripts.
.P
3. Run 'ibacm' as administrator to start the ibacm daemon.
.P
@@ -64,7 +67,9 @@ The librdmacm will automatically use the ibacm service.
On failures, the librdmacm will fall back to normal resolution.
.P
6. You can use ib_acme -P to gather performance statistics from the local ibacm
-daemon to see if the service is working correctly.
+daemon to see if the service is working correctly. Similarly, the command
+ib_acme -e could be used to enumerate all endpoints created by the local ibacm
+service.
.SH "NOTES"
ib_acme:
.P
@@ -80,14 +85,15 @@ The ibacm service relies on two configuration files.
.P
The ibacm_addr.cfg file contains name and address mappings for each IB
<device, port, pkey> endpoint. Although the names in the ibacm_addr.cfg
-file can be anything, ib_acme maps the host name and IP addresses to
-the IB endpoints. If the address file cannot be found, the ibacm
-service will attempt to create one using default values.
+file can be anything, ib_acme maps the host name to the IB endpoints. IP
+addresses, on the other hand, are assigned dynamically. If the address file
+cannot be found, the ibacm service will attempt to create one using default
+values.
.P
The ibacm_opts.cfg file provides a set of configurable options for the
-ibacm service, such as timeout, number of retries, logging level, etc.
-ib_acme generates the ibacm_opts.cfg file using static information. If
-an option file cannot be found, ibacm will use default values.
+ibacm core service and default provider, such as timeout, number of retries,
+logging level, etc. ib_acme generates the ibacm_opts.cfg file using static
+information. If an option file cannot be found, ibacm will use default values.
.P
ibacm:
.P
@@ -96,34 +102,33 @@ InfiniBand path information and caching such data. It
should execute with administrative privileges.
.P
The ibacm implements a client interface over TCP sockets, which is
-abstracted by the librdmacm library. One or more back-end protocols are
-used by the ibacm service to satisfy user requests. Although the
-ibacm supports standard SA path record queries on the back-end, it
+abstracted by the librdmacm library. One or more providers can be loaded
+by the core service, depending on the configuration. In the default provider
+ibacmp, one or more back-end protocols are used to satisfy user requests.
+Although ibacmp supports standard SA path record queries on the back-end, it
also supports a resolution protocol based on multicast traffic.
The latter is not usable on all fabric topologies, specifically
ones that may not have reversible paths or fabrics using torus routing.
Users should use the ib_acme utility to verify that multicast protocol
is usable before running other applications.
.P
-Conceptually, the ibacm service implements an ARP like protocol and either
+Conceptually, the default provider ibacmp implements an ARP like protocol and either
uses IB multicast records to construct path record data or queries the
SA directly, depending on the selected route protocol. By default, the
-ibacm services uses and caches SA path record queries.
+ibacmp provider uses and caches SA path record queries.
.P
Specifically, all IB endpoints join a number of multicast groups.
Multicast groups differ based on rates, mtu, sl, etc., and are prioritized.
All participating endpoints must be able to communicate on the lowest
-priority multicast group. The ibacm assigns one or more names/addresses
+priority multicast group. The ibacmp assigns one or more names/addresses
to each IB endpoint using the ibacm_addr.cfg file. Clients provide source
and destination names or addresses as input to the service, and receive
as output path record data.
.P
The service maps a client's source name/address to a local IB endpoint.
-If a client does not provide a source address, then the ibacm service
-will select one based on the destination and local routing tables. If the
-destination name/address is not cached locally, it sends a multicast
-request out on the lowest priority multicast group on the local endpoint.
-The request carries a list of multicast groups that the sender can use.
+If the destination name/address is not cached locally in the default provider,
+it sends a multicast request out on the lowest priority multicast group on the
+local endpoint. The request carries a list of multicast groups that the sender can use.
The recipient of the request selects the highest priority multicast group
that it can use as well and returns that information directly to the sender.
The request data is cached by all endpoints that receive the multicast
@@ -131,9 +136,9 @@ request message. The source endpoint also caches the response and uses
the multicast group that was selected to construct or obtain path record
data, which is returned to the client.
.P
-The current implementation of the IB ACM has several additional restrictions:
+The current implementation of the provider ibacmp has several additional restrictions:
.P
-- The ibacm is limited in its handling of dynamic changes.
+- The ibacmp is limited in its handling of dynamic changes.
ibacm must be stopped and restarted if a cluster is reconfigured.
.P
- Cached data does not timed out and is only updated if a new resolution
@@ -146,7 +151,7 @@ limited to 4.
.P
- The number of multicast groups that an endpoint can support is limited to 2.
.P
-The ibacm contains several internal caches. These include caches for GID
+The ibacmp contains several internal caches. These include caches for GID
and LID destination addresses. These caches can be optionally
preloaded. ibacm supports the OpenSM dump_pr plugin "full" PathRecord
format which is used to preload these caches.
diff --git a/man/ibacm.7 b/man/ibacm.7
index 06acf5b..ad502cd 100644
--- a/man/ibacm.7
+++ b/man/ibacm.7
@@ -1,8 +1,8 @@
-.TH "IBACM" 7 "2012-03-01" "IBACM" "IB ACM User Guide" IBACM
+.TH "IBACM" 7 "2014-06-16" "IBACM" "IB ACM User Guide" IBACM
.SH NAME
ibacm \- InfiniBand communication management assistant
.SH SYNOPSIS
-.B "#include <infiniband/ib_acm.h>"
+.B "#include <infiniband/acm.h>"
.SH "DESCRIPTION"
Used to resolve remote endpoint information before establishing communications
over InfiniBand.
diff --git a/man/ibacm_prov.7 b/man/ibacm_prov.7
new file mode 100644
index 0000000..d04617e
--- /dev/null
+++ b/man/ibacm_prov.7
@@ -0,0 +1,80 @@
+.TH "IBACM_PROV" 7 "2014-06-16" "IBACM_PROV" "IB ACM Provider Guide" IBACM_PROV
+.SH NAME
+ibacm_prov \- InfiniBand communication management assistant provider interface
+.SH SYNOPSIS
+.B "#include <infiniband/acm_prov.h>"
+.SH "DESCRIPTION"
+The ibacm provider interface provides a plugin interface that allows a vendor
+to implement proprietary solutions to support scalable address and route
+resolution services over InfiniBand.
+.P
+To add a provider to the ibacm core service, the provider must
+.TP
+1. be implemented as a shared library;
+.TP
+2. be installed under a configured directory, eg., /usr/lib64/ibacm/;
+.TP
+3 export a function provider_query() that returns a pointer to its provider info
+and version info.
+.P
+The prototype of provider_query function is defined below:
+.P
+.nf
+int provider_query(struct acm_provider **info, uint32_t *version);
+.fi
+.P
+This function should return a pointer to its provider structure:
+.P
+.nf
+struct acm_provider {
+ size_t size;
+ uint32_t version;
+ char *name;
+ int (*open_device)(const struct acm_device *device,
+ void **dev_context);
+ void (*close_device)(void *dev_context);
+ int (*open_port)(const struct acm_port *port,
+ void *dev_context, void **port_context);
+ void (*close_port)(void *port_context);
+ int (*open_endpoint)(const struct acm_endpoint *endpoint,
+ void *port_context, void **ep_context);
+ void (*close_endpoint)(void *ep_context);
+ int (*add_address)(const struct acm_address *addr, void *ep_context,
+ void **addr_context);
+ void (*remove_address)(void *addr_context);
+ int (*resolve)(void *addr_context, struct acm_msg *msg, uint64_t id);
+ int (*query)(void *addr_context, struct acm_msg *msg, uint64_t id);
+ int (*handle_event)(void *port_context, enum ibv_event_type type);
+ void (*query_perf)(void *ep_context, uint64_t *values, uint8_t *cnt);
+};
+.fi
+.P
+The size and version fields provide a way to detect version compatibility.
+When a port is assigned to the provider, the ibacm core will call the
+open/add_address functions; Similarly, when a port is down or re-assigned to
+another provider, the close/remove_address functions will be invoked to release
+resources. The ibacm core will centralize the management of events for each device
+and events not handled by the ibacm core will be forwarded to the relevant port
+through the handle_event() function. The resolve() function will be called to
+resolve a destination name into a path record. The performance of the provider
+for each endpoint can be queried by calling perf_query().
+.P
+To share a configuration file, the path for the ibacm configuration file is
+exported through the variable opts_file. Each loaded provider can open this
+configuration file and parse the contents related to its own operation.
+Non-related sections should be ignored.
+.P
+Some helper functions are also exported by the ibacm core. For example, the
+acm_log define (or the acm_write() function) can be used to log messages into
+ibacm's log file (default /var/log/ibacm.log). For details, refer to
+the acm_prov.h file.
+.SH "NOTES"
+A provider should always set the version in its provider info structure as the
+value of the define ACM_PROV_VERSION at the time the provider is implemented. Never
+set the version to ACM_PROV_VERSION itself as the define may be changed over time
+when the provider interface is changed, unless the provider itself is placed in
+ibacm source tree. This is to avoid the version problem when the old provider
+implementation is built against a new acm_prov.h file. The ibacm will always
+check the version of the provider at loading time.
+.SH "SEE ALSO"
+ib_acme(1), ibacm(1), ibacm(7)
diff --git a/prov/acmp/src/acmp.c b/prov/acmp/src/acmp.c
new file mode 100644
index 0000000..a590058
--- /dev/null
+++ b/prov/acmp/src/acmp.c
@@ -0,0 +1,2917 @@
+/*
+ * Copyright (c) 2009-2014 Intel Corporation. All rights reserved.
+ * Copyright (c) 2013 Mellanox Technologies LTD. All rights reserved.
+ *
+ * This software is available to you under the OpenIB.org BSD license
+ * below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+#include <osd.h>
+#include <arpa/inet.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <infiniband/acm.h>
+#include <infiniband/acm_prov.h>
+#include <infiniband/umad.h>
+#include <infiniband/verbs.h>
+#include <dlist.h>
+#include <dlfcn.h>
+#include <search.h>
+#include <net/if.h>
+#include <sys/ioctl.h>
+#include <net/if_arp.h>
+#include <netinet/in.h>
+#include <linux/netlink.h>
+#include <linux/rtnetlink.h>
+#include "acm_mad.h"
+
+#define src_out data[0]
+#define src_index data[1]
+#define dst_index data[2]
+
+#define IB_LID_MCAST_START 0xc000
+
+#define MAX_EP_ADDR 4
+#define MAX_EP_MC 2
+
+enum acmp_state {
+ ACMP_INIT,
+ ACMP_QUERY_ADDR,
+ ACMP_ADDR_RESOLVED,
+ ACMP_QUERY_ROUTE,
+ ACMP_READY
+};
+
+enum acmp_addr_prot {
+ ACMP_ADDR_PROT_ACM
+};
+
+enum acmp_route_prot {
+ ACMP_ROUTE_PROT_ACM,
+ ACMP_ROUTE_PROT_SA
+};
+
+enum acmp_loopback_prot {
+ ACMP_LOOPBACK_PROT_NONE,
+ ACMP_LOOPBACK_PROT_LOCAL
+};
+
+enum acmp_route_preload {
+ ACMP_ROUTE_PRELOAD_NONE,
+ ACMP_ROUTE_PRELOAD_OSM_FULL_V1
+};
+
+enum acmp_addr_preload {
+ ACMP_ADDR_PRELOAD_NONE,
+ ACMP_ADDR_PRELOAD_HOSTS
+};
+
+/*
+ * Nested locking order: dest -> ep, dest -> port
+ */
+struct acmp_ep;
+
+struct acmp_dest {
+ uint8_t address[ACM_MAX_ADDRESS]; /* keep first */
+ char name[ACM_MAX_ADDRESS];
+ struct ibv_ah *ah;
+ struct ibv_ah_attr av;
+ struct ibv_path_record path;
+ union ibv_gid mgid;
+ uint64_t req_id;
+ DLIST_ENTRY req_queue;
+ uint32_t remote_qpn;
+ lock_t lock;
+ enum acmp_state state;
+ atomic_t refcnt;
+ uint64_t addr_timeout;
+ uint64_t route_timeout;
+ uint8_t addr_type;
+ struct acmp_ep *ep;
+};
+
+struct acmp_device;
+
+struct acmp_port {
+ struct acmp_device *dev;
+ const struct acm_port *port;
+ DLIST_ENTRY ep_list;
+ lock_t lock;
+ struct acmp_dest sa_dest;
+ enum ibv_port_state state;
+ enum ibv_mtu mtu;
+ enum ibv_rate rate;
+ int subnet_timeout;
+ uint16_t default_pkey_ix;
+ uint16_t lid;
+ uint16_t lid_mask;
+ uint8_t port_num;
+};
+
+struct acmp_device {
+ struct ibv_context *verbs;
+ const struct acm_device *device;
+ struct ibv_comp_channel *channel;
+ struct ibv_pd *pd;
+ uint64_t guid;
+ DLIST_ENTRY entry;
+ pthread_t comp_thread_id;
+ int port_cnt;
+ struct acmp_port port[0];
+};
+
+/* Maintain separate virtual send queues to avoid deadlock */
+struct acmp_send_queue {
+ int credits;
+ DLIST_ENTRY pending;
+};
+
+struct acmp_addr {
+ uint16_t type;
+ union acm_ep_info info;
+ struct acm_address *addr;
+ struct acmp_ep *ep;
+};
+
+struct acmp_ep {
+ struct acmp_port *port;
+ struct ibv_cq *cq;
+ struct ibv_qp *qp;
+ struct ibv_mr *mr;
+ uint8_t *recv_bufs;
+ DLIST_ENTRY entry;
+ char id_string[ACM_MAX_ADDRESS];
+ void *dest_map[ACM_ADDRESS_RESERVED - 1];
+ struct acmp_dest mc_dest[MAX_EP_MC];
+ int mc_cnt;
+ uint16_t pkey_index;
+ uint16_t pkey;
+ const struct acm_endpoint *endpoint;
+ lock_t lock;
+ struct acmp_send_queue resolve_queue;
+ struct acmp_send_queue resp_queue;
+ DLIST_ENTRY active_queue;
+ DLIST_ENTRY wait_queue;
+ enum acmp_state state;
+ struct acmp_addr addr_info[MAX_EP_ADDR];
+ atomic_t counters[ACM_MAX_COUNTER];
+};
+
+struct acmp_send_msg {
+ DLIST_ENTRY entry;
+ struct acmp_ep *ep;
+ struct acmp_dest *dest;
+ struct ibv_ah *ah;
+ void *context;
+ void (*resp_handler)(struct acmp_send_msg *req,
+ struct ibv_wc *wc, struct acm_mad *resp);
+ struct acmp_send_queue *req_queue;
+ struct ibv_mr *mr;
+ struct ibv_send_wr wr;
+ struct ibv_sge sge;
+ uint64_t expires;
+ int tries;
+ uint8_t data[ACM_SEND_SIZE];
+};
+
+struct acmp_request {
+ uint64_t id;
+ DLIST_ENTRY entry;
+ struct acm_msg msg;
+ struct acmp_ep *ep;
+};
+
+static int acmp_open_dev(const struct acm_device *device, void **dev_context);
+static void acmp_close_dev(void *dev_context);
+static int acmp_open_port(const struct acm_port *port, void *dev_context,
+ void **port_context);
+static void acmp_close_port(void *port_context);
+static int acmp_open_endpoint(const struct acm_endpoint *endpoint,
+ void *port_context, void **ep_context);
+static void acmp_close_endpoint(void *ep_context);
+static int acmp_add_addr(const struct acm_address *addr, void *ep_context,
+ void **addr_context);
+static void acmp_remove_addr(void *addr_context);
+static int acmp_resolve(void *addr_context, struct acm_msg *msg, uint64_t id);
+static int acmp_query(void *addr_context, struct acm_msg *msg, uint64_t id);
+static int acmp_handle_event(void *port_context, enum ibv_event_type type);
+static void acmp_query_perf(void *ep_context, uint64_t *values, uint8_t *cnt);
+
+static struct acm_provider def_prov = {
+ .size = sizeof(struct acm_provider),
+ .version = ACM_PROV_VERSION,
+ .name = "ibacmp",
+ .open_device = acmp_open_dev,
+ .close_device = acmp_close_dev,
+ .open_port = acmp_open_port,
+ .close_port = acmp_close_port,
+ .open_endpoint = acmp_open_endpoint,
+ .close_endpoint = acmp_close_endpoint,
+ .add_address = acmp_add_addr,
+ .remove_address = acmp_remove_addr,
+ .resolve = acmp_resolve,
+ .query = acmp_query,
+ .handle_event = acmp_handle_event,
+ .query_perf = acmp_query_perf,
+};
+
+static DLIST_ENTRY acmp_dev_list;
+static lock_t acmp_dev_lock;
+
+static atomic_t tid;
+static DLIST_ENTRY timeout_list;
+static event_t timeout_event;
+static atomic_t wait_cnt;
+static pthread_t retry_thread_id;
+static int retry_thread_started = 0;
+
+PER_THREAD char log_data[ACM_MAX_ADDRESS];
+
+/*
+ * Service options - may be set through ibacm_opts.cfg file.
+ */
+static char route_data_file[128] = ACM_CONF_DIR "/ibacm_route.data";
+static char addr_data_file[128] = ACM_CONF_DIR "/ibacm_hosts.data";
+static enum acmp_addr_prot addr_prot = ACMP_ADDR_PROT_ACM;
+static int addr_timeout = 1440;
+static enum acmp_route_prot route_prot = ACMP_ROUTE_PROT_SA;
+static int route_timeout = -1;
+static enum acmp_loopback_prot loopback_prot = ACMP_LOOPBACK_PROT_LOCAL;
+static int timeout = 2000;
+static int retries = 2;
+static int resolve_depth = 1;
+static int send_depth = 1;
+static int recv_depth = 1024;
+static uint8_t min_mtu = IBV_MTU_2048;
+static uint8_t min_rate = IBV_RATE_10_GBPS;
+static enum acmp_route_preload route_preload;
+static enum acmp_addr_preload addr_preload;
+
+static int acmp_initialized = 0;
+
+static int acmp_compare_dest(const void *dest1, const void *dest2)
+{
+ return memcmp(dest1, dest2, ACM_MAX_ADDRESS);
+}
+
+static void
+acmp_set_dest_addr(struct acmp_dest *dest, uint8_t addr_type,
+ const uint8_t *addr, size_t size)
+{
+ memcpy(dest->address, addr, size);
+ dest->addr_type = addr_type;
+ acm_format_name(0, dest->name, sizeof dest->name, addr_type, addr, size);
+}
+
+static void
+acmp_init_dest(struct acmp_dest *dest, uint8_t addr_type,
+ const uint8_t *addr, size_t size)
+{
+ DListInit(&dest->req_queue);
+ atomic_init(&dest->refcnt);
+ atomic_set(&dest->refcnt, 1);
+ lock_init(&dest->lock);
+ if (size)
+ acmp_set_dest_addr(dest, addr_type, addr, size);
+ dest->state = ACMP_INIT;
+}
+
+static struct acmp_dest *
+acmp_alloc_dest(uint8_t addr_type, const uint8_t *addr)
+{
+ struct acmp_dest *dest;
+
+ dest = calloc(1, sizeof *dest);
+ if (!dest) {
+ acm_log(0, "ERROR - unable to allocate dest\n");
+ return NULL;
+ }
+
+ acmp_init_dest(dest, addr_type, addr, ACM_MAX_ADDRESS);
+ acm_log(1, "%s\n", dest->name);
+ return dest;
+}
+
+/* Caller must hold ep lock. */
+static struct acmp_dest *
+acmp_get_dest(struct acmp_ep *ep, uint8_t addr_type, const uint8_t *addr)
+{
+ struct acmp_dest *dest, **tdest;
+
+ tdest = tfind(addr, &ep->dest_map[addr_type - 1], acmp_compare_dest);
+ if (tdest) {
+ dest = *tdest;
+ (void) atomic_inc(&dest->refcnt);
+ acm_log(2, "%s\n", dest->name);
+ } else {
+ dest = NULL;
+ acm_format_name(2, log_data, sizeof log_data,
+ addr_type, addr, ACM_MAX_ADDRESS);
+ acm_log(2, "%s not found\n", log_data);
+ }
+ return dest;
+}
+
+static void
+acmp_put_dest(struct acmp_dest *dest)
+{
+ acm_log(2, "%s\n", dest->name);
+ if (atomic_dec(&dest->refcnt) == 0) {
+ free(dest);
+ }
+}
+
+static struct acmp_dest *
+acmp_acquire_dest(struct acmp_ep *ep, uint8_t addr_type, const uint8_t *addr)
+{
+ struct acmp_dest *dest;
+
+ acm_format_name(2, log_data, sizeof log_data,
+ addr_type, addr, ACM_MAX_ADDRESS);
+ acm_log(2, "%s\n", log_data);
+ lock_acquire(&ep->lock);
+ dest = acmp_get_dest(ep, addr_type, addr);
+ if (!dest) {
+ dest = acmp_alloc_dest(addr_type, addr);
+ if (dest) {
+ dest->ep = ep;
+ tsearch(dest, &ep->dest_map[addr_type - 1], acmp_compare_dest);
+ (void) atomic_inc(&dest->refcnt);
+ }
+ }
+ lock_release(&ep->lock);
+ return dest;
+}
+
+/* Caller must hold ep lock. */
+//static void
+//acmp_remove_dest(struct acmp_ep *ep, struct acmp_dest *dest)
+//{
+// acm_log(2, "%s\n", dest->name);
+// tdelete(dest->address, &ep->dest_map[dest->addr_type - 1], acmp_compare_dest);
+// acmp_put_dest(dest);
+//}
+
+static struct acmp_request *acmp_alloc_req(uint64_t id, struct acm_msg *msg)
+{
+ struct acmp_request *req;
+
+ req = calloc(1, sizeof *req);
+ if (!req) {
+ acm_log(0, "ERROR - unable to alloc client request\n");
+ return NULL;
+ }
+
+ req->id = id;
+ memcpy(&req->msg, msg, sizeof(req->msg));
+ acm_log(2, "id %llu, req %p\n", id, req);
+ return req;
+}
+
+static void acmp_free_req(struct acmp_request *req)
+{
+ acm_log(2, "%p\n", req);
+ free(req);
+}
+
+static struct acmp_send_msg *
+acmp_alloc_send(struct acmp_ep *ep, struct acmp_dest *dest, size_t size)
+{
+ struct acmp_send_msg *msg;
+
+ msg = (struct acmp_send_msg *) calloc(1, sizeof *msg);
+ if (!msg) {
+ acm_log(0, "ERROR - unable to allocate send buffer\n");
+ return NULL;
+ }
+
+ msg->ep = ep;
+ msg->mr = ibv_reg_mr(ep->port->dev->pd, msg->data, size, 0);
+ if (!msg->mr) {
+ acm_log(0, "ERROR - failed to register send buffer\n");
+ goto err1;
+ }
+
+ if (!dest->ah) {
+ msg->ah = ibv_create_ah(ep->port->dev->pd, &dest->av);
+ if (!msg->ah) {
+ acm_log(0, "ERROR - unable to create ah\n");
+ goto err2;
+ }
+ msg->wr.wr.ud.ah = msg->ah;
+ } else {
+ msg->wr.wr.ud.ah = dest->ah;
+ }
+
+ acm_log(2, "get dest %s\n", dest->name);
+ (void) atomic_inc(&dest->refcnt);
+ msg->dest = dest;
+
+ msg->wr.next = NULL;
+ msg->wr.sg_list = &msg->sge;
+ msg->wr.num_sge = 1;
+ msg->wr.opcode = IBV_WR_SEND;
+ msg->wr.send_flags = IBV_SEND_SIGNALED;
+ msg->wr.wr_id = (uintptr_t) msg;
+ msg->wr.wr.ud.remote_qpn = dest->remote_qpn;
+ msg->wr.wr.ud.remote_qkey = ACM_QKEY;
+
+ msg->sge.length = size;
+ msg->sge.lkey = msg->mr->lkey;
+ msg->sge.addr = (uintptr_t) msg->data;
+ acm_log(2, "%p\n", msg);
+ return msg;
+
+err2:
+ ibv_dereg_mr(msg->mr);
+err1:
+ free(msg);
+ return NULL;
+}
+
+static void
+acmp_init_send_req(struct acmp_send_msg *msg, void *context,
+ void (*resp_handler)(struct acmp_send_msg *req,
+ struct ibv_wc *wc, struct acm_mad *resp))
+{
+ acm_log(2, "%p\n", msg);
+ msg->tries = retries + 1;
+ msg->context = context;
+ msg->resp_handler = resp_handler;
+}
+
+static void acmp_free_send(struct acmp_send_msg *msg)
+{
+ acm_log(2, "%p\n", msg);
+ if (msg->ah)
+ ibv_destroy_ah(msg->ah);
+ ibv_dereg_mr(msg->mr);
+ acmp_put_dest(msg->dest);
+ free(msg);
+}
+
+static void acmp_post_send(struct acmp_send_queue *queue, struct acmp_send_msg *msg)
+{
+ struct acmp_ep *ep = msg->ep;
+ struct ibv_send_wr *bad_wr;
+
+ msg->req_queue = queue;
+ lock_acquire(&ep->lock);
+ if (queue->credits) {
+ acm_log(2, "posting send to QP\n");
+ queue->credits--;
+ DListInsertTail(&msg->entry, &ep->active_queue);
+ ibv_post_send(ep->qp, &msg->wr, &bad_wr);
+ } else {
+ acm_log(2, "no sends available, queuing message\n");
+ DListInsertTail(&msg->entry, &queue->pending);
+ }
+ lock_release(&ep->lock);
+}
+
+static void acmp_post_recv(struct acmp_ep *ep, uint64_t address)
+{
+ struct ibv_recv_wr wr, *bad_wr;
+ struct ibv_sge sge;
+
+ wr.next = NULL;
+ wr.sg_list = &sge;
+ wr.num_sge = 1;
+ wr.wr_id = address;
+
+ sge.length = ACM_RECV_SIZE;
+ sge.lkey = ep->mr->lkey;
+ sge.addr = address;
+
+ ibv_post_recv(ep->qp, &wr, &bad_wr);
+}
+
+/* Caller must hold ep lock */
+static void acmp_send_available(struct acmp_ep *ep, struct acmp_send_queue *queue)
+{
+ struct acmp_send_msg *msg;
+ struct ibv_send_wr *bad_wr;
+ DLIST_ENTRY *entry;
+
+ if (DListEmpty(&queue->pending)) {
+ queue->credits++;
+ } else {
+ acm_log(2, "posting queued send message\n");
+ entry = queue->pending.Next;
+ DListRemove(entry);
+ msg = container_of(entry, struct acmp_send_msg, entry);
+ DListInsertTail(&msg->entry, &ep->active_queue);
+ ibv_post_send(ep->qp, &msg->wr, &bad_wr);
+ }
+}
+
+static void acmp_complete_send(struct acmp_send_msg *msg)
+{
+ struct acmp_ep *ep = msg->ep;
+
+ lock_acquire(&ep->lock);
+ DListRemove(&msg->entry);
+ if (msg->tries) {
+ acm_log(2, "waiting for response\n");
+ msg->expires = time_stamp_ms() + ep->port->subnet_timeout + timeout;
+ DListInsertTail(&msg->entry, &ep->wait_queue);
+ if (atomic_inc(&wait_cnt) == 1)
+ event_signal(&timeout_event);
+ } else {
+ acm_log(2, "freeing\n");
+ acmp_send_available(ep, msg->req_queue);
+ acmp_free_send(msg);
+ }
+ lock_release(&ep->lock);
+}
+
+static struct acmp_send_msg *acmp_get_request(struct acmp_ep *ep, uint64_t tid, int *free)
+{
+ struct acmp_send_msg *msg, *req = NULL;
+ struct acm_mad *mad;
+ DLIST_ENTRY *entry, *next;
+
+ acm_log(2, "\n");
+ lock_acquire(&ep->lock);
+ for (entry = ep->wait_queue.Next; entry != &ep->wait_queue; entry = next) {
+ next = entry->Next;
+ msg = container_of(entry, struct acmp_send_msg, entry);
+ mad = (struct acm_mad *) msg->data;
+ if (mad->tid == tid) {
+ acm_log(2, "match found in wait queue\n");
+ req = msg;
+ DListRemove(entry);
+ (void) atomic_dec(&wait_cnt);
+ acmp_send_available(ep, msg->req_queue);
+ *free = 1;
+ goto unlock;
+ }
+ }
+
+ for (entry = ep->active_queue.Next; entry != &ep->active_queue; entry = entry->Next) {
+ msg = container_of(entry, struct acmp_send_msg, entry);
+ mad = (struct acm_mad *) msg->data;
+ if (mad->tid == tid && msg->tries) {
+ acm_log(2, "match found in active queue\n");
+ req = msg;
+ req->tries = 0;
+ *free = 0;
+ break;
+ }
+ }
+unlock:
+ lock_release(&ep->lock);
+ return req;
+}
+
+static int acmp_mc_index(struct acmp_ep *ep, union ibv_gid *gid)
+{
+ int i;
+
+ for (i = 0; i < ep->mc_cnt; i++) {
+ if (!memcmp(&ep->mc_dest[i].address, gid, sizeof(*gid)))
+ return i;
+ }
+ return -1;
+}
+
+/* Multicast groups are ordered lowest to highest preference. */
+static int acmp_best_mc_index(struct acmp_ep *ep, struct acm_resolve_rec *rec)
+{
+ int i, index;
+
+ for (i = min(rec->gid_cnt, ACM_MAX_GID_COUNT) - 1; i >= 0; i--) {
+ index = acmp_mc_index(ep, &rec->gid[i]);
+ if (index >= 0) {
+ return index;
+ }
+ }
+ return -1;
+}
+
+static void
+acmp_record_mc_av(struct acmp_port *port, struct ib_mc_member_rec *mc_rec,
+ struct acmp_dest *dest)
+{
+ uint32_t sl_flow_hop;
+
+ sl_flow_hop = ntohl(mc_rec->sl_flow_hop);
+
+ dest->av.dlid = ntohs(mc_rec->mlid);
+ dest->av.sl = (uint8_t) (sl_flow_hop >> 28);
+ dest->av.src_path_bits = port->sa_dest.av.src_path_bits;
+ dest->av.static_rate = mc_rec->rate & 0x3F;
+ dest->av.port_num = port->port_num;
+
+ dest->av.is_global = 1;
+ dest->av.grh.dgid = mc_rec->mgid;
+ dest->av.grh.flow_label = (sl_flow_hop >> 8) & 0xFFFFF;
+ dest->av.grh.sgid_index = acm_gid_index((struct acm_port *) port->port,
+ &mc_rec->port_gid);
+ dest->av.grh.hop_limit = (uint8_t) sl_flow_hop;
+ dest->av.grh.traffic_class = mc_rec->tclass;
+
+ dest->path.dgid = mc_rec->mgid;
+ dest->path.sgid = mc_rec->port_gid;
+ dest->path.dlid = mc_rec->mlid;
+ dest->path.slid = htons(port->lid) | port->sa_dest.av.src_path_bits;
+ dest->path.flowlabel_hoplimit = htonl(sl_flow_hop & 0xFFFFFFF);
+ dest->path.tclass = mc_rec->tclass;
+ dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE | 1;
+ dest->path.pkey = mc_rec->pkey;
+ dest->path.qosclass_sl = htons((uint16_t) (sl_flow_hop >> 28));
+ dest->path.mtu = mc_rec->mtu;
+ dest->path.rate = mc_rec->rate;
+ dest->path.packetlifetime = mc_rec->packet_lifetime;
+}
+
+/* Always send the GRH to transfer GID data to remote side */
+static void
+acmp_init_path_av(struct acmp_port *port, struct acmp_dest *dest)
+{
+ uint32_t flow_hop;
+
+ dest->av.dlid = ntohs(dest->path.dlid);
+ dest->av.sl = ntohs(dest->path.qosclass_sl) & 0xF;
+ dest->av.src_path_bits = dest->path.slid & 0x7F;
+ dest->av.static_rate = dest->path.rate & 0x3F;
+ dest->av.port_num = port->port_num;
+
+ flow_hop = ntohl(dest->path.flowlabel_hoplimit);
+ dest->av.is_global = 1;
+ dest->av.grh.flow_label = (flow_hop >> 8) & 0xFFFFF;
+ lock_acquire(&port->lock);
+ if (port->port)
+ dest->av.grh.sgid_index = acm_gid_index(
+ (struct acm_port *) port->port, &dest->path.sgid);
+ else
+ dest->av.grh.sgid_index = 0;
+ lock_release(&port->lock);
+ dest->av.grh.hop_limit = (uint8_t) flow_hop;
+ dest->av.grh.traffic_class = dest->path.tclass;
+}
+
+static void acmp_process_join_resp(struct acm_sa_mad *sa_mad)
+{
+ struct acmp_dest *dest;
+ struct ib_mc_member_rec *mc_rec;
+ struct ib_sa_mad *mad;
+ int index, ret;
+ struct acmp_ep *ep = sa_mad->context;
+
+ mad = (struct ib_sa_mad *) &sa_mad->sa_mad;
+ acm_log(1, "response status: 0x%x, mad status: 0x%x\n",
+ sa_mad->umad.status, mad->status);
+ lock_acquire(&ep->lock);
+ if (sa_mad->umad.status) {
+ acm_log(0, "ERROR - send join failed 0x%x\n", sa_mad->umad.status);
+ goto out;
+ }
+ if (mad->status) {
+ acm_log(0, "ERROR - join response status 0x%x\n", mad->status);
+ goto out;
+ }
+
+ mc_rec = (struct ib_mc_member_rec *) mad->data;
+ index = acmp_mc_index(ep, &mc_rec->mgid);
+ if (index < 0) {
+ acm_log(0, "ERROR - MGID in join response not found\n");
+ goto out;
+ }
+
+ dest = &ep->mc_dest[index];
+ dest->remote_qpn = IB_MC_QPN;
+ dest->mgid = mc_rec->mgid;
+ acmp_record_mc_av(ep->port, mc_rec, dest);
+
+ if (index == 0) {
+ dest->ah = ibv_create_ah(ep->port->dev->pd, &dest->av);
+ if (!dest->ah) {
+ acm_log(0, "ERROR - unable to create ah\n");
+ goto out;
+ }
+ ret = ibv_attach_mcast(ep->qp, &mc_rec->mgid, mc_rec->mlid);
+ if (ret) {
+ acm_log(0, "ERROR - unable to attach QP to multicast group\n");
+ ibv_destroy_ah(dest->ah);
+ dest->ah = NULL;
+ goto out;
+ }
+ ep->state = ACMP_READY;
+ }
+
+ atomic_set(&dest->refcnt, 1);
+ dest->state = ACMP_READY;
+ acm_log(1, "join successful\n");
+out:
+ acm_free_sa_mad(sa_mad);
+ lock_release(&ep->lock);
+}
+
+static uint8_t
+acmp_record_acm_route(struct acmp_ep *ep, struct acmp_dest *dest)
+{
+ int i;
+
+ acm_log(2, "\n");
+ for (i = 0; i < MAX_EP_MC; i++) {
+ if (!memcmp(&dest->mgid, &ep->mc_dest[i].mgid, sizeof dest->mgid))
+ break;
+ }
+ if (i == MAX_EP_MC) {
+ acm_log(0, "ERROR - cannot match mgid\n");
+ return ACM_STATUS_EINVAL;
+ }
+
+ dest->path = ep->mc_dest[i].path;
+ dest->path.dgid = dest->av.grh.dgid;
+ dest->path.dlid = htons(dest->av.dlid);
+ dest->addr_timeout = time_stamp_min() + (unsigned) addr_timeout;
+ dest->route_timeout = time_stamp_min() + (unsigned) route_timeout;
+ dest->state = ACMP_READY;
+ return ACM_STATUS_SUCCESS;
+}
+
+static void acmp_init_path_query(struct ib_sa_mad *mad)
+{
+ acm_log(2, "\n");
+ mad->base_version = 1;
+ mad->mgmt_class = IB_MGMT_CLASS_SA;
+ mad->class_version = 2;
+ mad->method = IB_METHOD_GET;
+ mad->tid = htonll((uint64_t) atomic_inc(&tid));
+ mad->attr_id = IB_SA_ATTR_PATH_REC;
+}
+
+/* Caller must hold dest lock */
+static uint8_t acmp_resolve_path_sa(struct acmp_ep *ep, struct acmp_dest *dest,
+ void (*handler)(struct acm_sa_mad *))
+{
+ struct ib_sa_mad *mad;
+ uint8_t ret;
+ struct acm_sa_mad *sa_mad;
+
+ acm_log(2, "%s\n", dest->name);
+
+ sa_mad = acm_alloc_sa_mad(ep->endpoint, dest, handler);
+ if (!sa_mad) {
+ acm_log(0, "Error - failed to allocate sa_mad\n");
+ ret = ACM_STATUS_ENOMEM;
+ goto err;
+ }
+
+ mad = (struct ib_sa_mad *) &sa_mad->sa_mad;
+ acmp_init_path_query(mad);
+
+ memcpy(mad->data, &dest->path, sizeof(dest->path));
+ mad->comp_mask = acm_path_comp_mask(&dest->path);
+
+ acm_increment_counter(ACM_CNTR_ROUTE_QUERY);
+ atomic_inc(&ep->counters[ACM_CNTR_ROUTE_QUERY]);
+ dest->state = ACMP_QUERY_ROUTE;
+ if (acm_send_sa_mad(sa_mad)) {
+ acm_log(0, "Error - Failed to send sa mad\n");
+ ret = ACM_STATUS_ENODATA;
+ goto free_mad;
+ }
+ return ACM_STATUS_SUCCESS;
+free_mad:
+ acm_free_sa_mad(sa_mad);
+err:
+ dest->state = ACMP_INIT;
+ return ret;
+}
+
+static uint8_t
+acmp_record_acm_addr(struct acmp_ep *ep, struct acmp_dest *dest, struct ibv_wc *wc,
+ struct acm_resolve_rec *rec)
+{
+ int index;
+
+ acm_log(2, "%s\n", dest->name);
+ index = acmp_best_mc_index(ep, rec);
+ if (index < 0) {
+ acm_log(0, "ERROR - no shared multicast groups\n");
+ dest->state = ACMP_INIT;
+ return ACM_STATUS_ENODATA;
+ }
+
+ acm_log(2, "selecting MC group at index %d\n", index);
+ dest->av = ep->mc_dest[index].av;
+ dest->av.dlid = wc->slid;
+ dest->av.src_path_bits = wc->dlid_path_bits;
+ dest->av.grh.dgid = ((struct ibv_grh *) (uintptr_t) wc->wr_id)->sgid;
+
+ dest->mgid = ep->mc_dest[index].mgid;
+ dest->path.sgid = ep->mc_dest[index].path.sgid;
+ dest->path.dgid = dest->av.grh.dgid;
+ dest->path.tclass = ep->mc_dest[index].path.tclass;
+ dest->path.pkey = ep->mc_dest[index].path.pkey;
+ dest->remote_qpn = wc->src_qp;
+
+ dest->state = ACMP_ADDR_RESOLVED;
+ return ACM_STATUS_SUCCESS;
+}
+
+static void
+acmp_record_path_addr(struct acmp_ep *ep, struct acmp_dest *dest,
+ struct ibv_path_record *path)
+{
+ acm_log(2, "%s\n", dest->name);
+ dest->path.pkey = htons(ep->pkey);
+ dest->path.dgid = path->dgid;
+ if (path->slid) {
+ dest->path.slid = path->slid;
+ } else {
+ dest->path.slid = htons(ep->port->lid);
+ }
+ if (!ib_any_gid(&path->sgid)) {
+ dest->path.sgid = path->sgid;
+ } else {
+ dest->path.sgid = ep->mc_dest[0].path.sgid;
+ }
+ dest->path.dlid = path->dlid;
+ dest->state = ACMP_ADDR_RESOLVED;
+}
+
+static uint8_t acmp_validate_addr_req(struct acm_mad *mad)
+{
+ struct acm_resolve_rec *rec;
+
+ if (mad->method != IB_METHOD_GET) {
+ acm_log(0, "ERROR - invalid method 0x%x\n", mad->method);
+ return ACM_STATUS_EINVAL;
+ }
+
+ rec = (struct acm_resolve_rec *) mad->data;
+ if (!rec->src_type || rec->src_type >= ACM_ADDRESS_RESERVED) {
+ acm_log(0, "ERROR - unknown src type 0x%x\n", rec->src_type);
+ return ACM_STATUS_EINVAL;
+ }
+
+ return ACM_STATUS_SUCCESS;
+}
+
+static void
+acmp_send_addr_resp(struct acmp_ep *ep, struct acmp_dest *dest)
+{
+ struct acm_resolve_rec *rec;
+ struct acmp_send_msg *msg;
+ struct acm_mad *mad;
+
+ acm_log(2, "%s\n", dest->name);
+ msg = acmp_alloc_send(ep, dest, sizeof (*mad));
+ if (!msg) {
+ acm_log(0, "ERROR - failed to allocate message\n");
+ return;
+ }
+
+ mad = (struct acm_mad *) msg->data;
+ rec = (struct acm_resolve_rec *) mad->data;
+
+ mad->base_version = 1;
+ mad->mgmt_class = ACM_MGMT_CLASS;
+ mad->class_version = 1;
+ mad->method = IB_METHOD_GET | IB_METHOD_RESP;
+ mad->status = ACM_STATUS_SUCCESS;
+ mad->control = ACM_CTRL_RESOLVE;
+ mad->tid = dest->req_id;
+ rec->gid_cnt = 1;
+ memcpy(rec->gid, dest->mgid.raw, sizeof(union ibv_gid));
+
+ acmp_post_send(&ep->resp_queue, msg);
+}
+
+static int
+acmp_resolve_response(uint64_t id, struct acm_msg *req_msg,
+ struct acmp_dest *dest, uint8_t status)
+{
+ struct acm_msg msg;
+
+ acm_log(2, "client %lld, status 0x%x\n", id, status);
+ memset(&msg, 0, sizeof msg);
+
+ if (dest) {
+ if (status == ACM_STATUS_ENODATA)
+ atomic_inc(&dest->ep->counters[ACM_CNTR_NODATA]);
+ else if (status)
+ atomic_inc(&dest->ep->counters[ACM_CNTR_ERROR]);
+ }
+ msg.hdr = req_msg->hdr;
+ msg.hdr.status = status;
+ msg.hdr.length = ACM_MSG_HDR_LENGTH;
+ memset(msg.hdr.data, 0, sizeof(msg.hdr.data));
+
+ if (status == ACM_STATUS_SUCCESS) {
+ msg.hdr.length += ACM_MSG_EP_LENGTH;
+ msg.resolve_data[0].flags = IBV_PATH_FLAG_GMP |
+ IBV_PATH_FLAG_PRIMARY | IBV_PATH_FLAG_BIDIRECTIONAL;
+ msg.resolve_data[0].type = ACM_EP_INFO_PATH;
+ msg.resolve_data[0].info.path = dest->path;
+
+ if (req_msg->hdr.src_out) {
+ msg.hdr.length += ACM_MSG_EP_LENGTH;
+ memcpy(&msg.resolve_data[1],
+ &req_msg->resolve_data[req_msg->hdr.src_index],
+ ACM_MSG_EP_LENGTH);
+ }
+ }
+
+ return acm_resolve_response(id, &msg);
+}
+
+static void
+acmp_complete_queued_req(struct acmp_dest *dest, uint8_t status)
+{
+ struct acmp_request *req;
+ DLIST_ENTRY *entry;
+
+ acm_log(2, "status %d\n", status);
+ lock_acquire(&dest->lock);
+ while (!DListEmpty(&dest->req_queue)) {
+ entry = dest->req_queue.Next;
+ DListRemove(entry);
+ req = container_of(entry, struct acmp_request, entry);
+ lock_release(&dest->lock);
+
+ acm_log(2, "completing request, client %d\n", req->id);
+ acmp_resolve_response(req->id, &req->msg, dest, status);
+ acmp_free_req(req);
+
+ lock_acquire(&dest->lock);
+ }
+ lock_release(&dest->lock);
+}
+
+static void
+acmp_dest_sa_resp(struct acm_sa_mad *mad)
+{
+ struct acmp_dest *dest = (struct acmp_dest *) mad->context;
+ struct ib_sa_mad *sa_mad = (struct ib_sa_mad *) &mad->sa_mad;
+ uint8_t status;
+
+ if (!mad->umad.status) {
+ status = (uint8_t) (ntohs(sa_mad->status) >> 8);
+ } else {
+ status = ACM_STATUS_ETIMEDOUT;
+ }
+ acm_log(2, "%s status=0x%x\n", dest->name, status);
+
+ lock_acquire(&dest->lock);
+ if (dest->state != ACMP_QUERY_ROUTE) {
+ acm_log(1, "notice - discarding SA response\n");
+ lock_release(&dest->lock);
+ goto out;
+ }
+
+ if (!status) {
+ memcpy(&dest->path, sa_mad->data, sizeof(dest->path));
+ acmp_init_path_av(dest->ep->port, dest);
+ dest->addr_timeout = time_stamp_min() + (unsigned) addr_timeout;
+ dest->route_timeout = time_stamp_min() + (unsigned) route_timeout;
+ acm_log(2, "timeout addr %llu route %llu\n", dest->addr_timeout, dest->route_timeout);
+ dest->state = ACMP_READY;
+ } else {
+ dest->state = ACMP_INIT;
+ }
+ lock_release(&dest->lock);
+
+ acmp_complete_queued_req(dest, status);
+out:
+ acm_free_sa_mad(mad);
+}
+
+static void
+acmp_resolve_sa_resp(struct acm_sa_mad *mad)
+{
+ struct acmp_dest *dest = (struct acmp_dest *) mad->context;
+ int send_resp;
+
+ acm_log(2, "\n");
+ acmp_dest_sa_resp(mad);
+
+ lock_acquire(&dest->lock);
+ send_resp = (dest->state == ACMP_READY);
+ lock_release(&dest->lock);
+
+ if (send_resp)
+ acmp_send_addr_resp(dest->ep, dest);
+}
+
+static struct acmp_addr *
+acmp_addr_lookup(struct acmp_ep *ep, uint8_t *addr, uint16_t type)
+{
+ int i;
+
+ for (i = 0; i < MAX_EP_ADDR; i++) {
+ if (ep->addr_info[i].type != type)
+ continue;
+
+ if ((type == ACM_ADDRESS_NAME &&
+ !strnicmp((char *) ep->addr_info[i].info.name,
+ (char *) addr, ACM_MAX_ADDRESS)) ||
+ !memcmp(ep->addr_info[i].info.addr, addr,
+ ACM_MAX_ADDRESS)) {
+ return &ep->addr_info[i];
+ }
+ }
+
+ return NULL;
+}
+
+static void
+acmp_process_addr_req(struct acmp_ep *ep, struct ibv_wc *wc, struct acm_mad *mad)
+{
+ struct acm_resolve_rec *rec;
+ struct acmp_dest *dest;
+ uint8_t status;
+ struct acmp_addr *addr;
+
+ acm_log(2, "\n");
+ if ((status = acmp_validate_addr_req(mad))) {
+ acm_log(0, "ERROR - invalid request\n");
+ return;
+ }
+
+ rec = (struct acm_resolve_rec *) mad->data;
+ dest = acmp_acquire_dest(ep, rec->src_type, rec->src);
+ if (!dest) {
+ acm_log(0, "ERROR - unable to add source\n");
+ return;
+ }
+
+ addr = acmp_addr_lookup(ep, rec->dest, rec->dest_type);
+ if (addr)
+ dest->req_id = mad->tid;
+
+ lock_acquire(&dest->lock);
+ acm_log(2, "dest state %d\n", dest->state);
+ switch (dest->state) {
+ case ACMP_READY:
+ if (dest->remote_qpn == wc->src_qp)
+ break;
+
+ acm_log(2, "src service has new qp, resetting\n");
+ /* fall through */
+ case ACMP_INIT:
+ case ACMP_QUERY_ADDR:
+ status = acmp_record_acm_addr(ep, dest, wc, rec);
+ if (status)
+ break;
+ /* fall through */
+ case ACMP_ADDR_RESOLVED:
+ if (route_prot == ACMP_ROUTE_PROT_ACM) {
+ status = acmp_record_acm_route(ep, dest);
+ break;
+ }
+ if (addr || !DListEmpty(&dest->req_queue)) {
+ status = acmp_resolve_path_sa(ep, dest, acmp_resolve_sa_resp);
+ if (status)
+ break;
+ }
+ /* fall through */
+ default:
+ lock_release(&dest->lock);
+ acmp_put_dest(dest);
+ return;
+ }
+ lock_release(&dest->lock);
+ acmp_complete_queued_req(dest, status);
+
+ if (addr && !status) {
+ acmp_send_addr_resp(ep, dest);
+ }
+ acmp_put_dest(dest);
+}
+
+static void
+acmp_process_addr_resp(struct acmp_send_msg *msg, struct ibv_wc *wc, struct acm_mad *mad)
+{
+ struct acm_resolve_rec *resp_rec;
+ struct acmp_dest *dest = (struct acmp_dest *) msg->context;
+ uint8_t status;
+
+ if (mad) {
+ status = acm_class_status(mad->status);
+ resp_rec = (struct acm_resolve_rec *) mad->data;
+ } else {
+ status = ACM_STATUS_ETIMEDOUT;
+ resp_rec = NULL;
+ }
+ acm_log(2, "resp status 0x%x\n", status);
+
+ lock_acquire(&dest->lock);
+ if (dest->state != ACMP_QUERY_ADDR) {
+ lock_release(&dest->lock);
+ goto put;
+ }
+
+ if (!status) {
+ status = acmp_record_acm_addr(msg->ep, dest, wc, resp_rec);
+ if (!status) {
+ if (route_prot == ACMP_ROUTE_PROT_ACM) {
+ status = acmp_record_acm_route(msg->ep, dest);
+ } else {
+ status = acmp_resolve_path_sa(msg->ep, dest, acmp_dest_sa_resp);
+ if (!status) {
+ lock_release(&dest->lock);
+ goto put;
+ }
+ }
+ }
+ } else {
+ dest->state = ACMP_INIT;
+ }
+ lock_release(&dest->lock);
+
+ acmp_complete_queued_req(dest, status);
+put:
+ acmp_put_dest(dest);
+}
+
+static void acmp_process_acm_recv(struct acmp_ep *ep, struct ibv_wc *wc, struct acm_mad *mad)
+{
+ struct acmp_send_msg *req;
+ struct acm_resolve_rec *rec;
+ int free;
+
+ acm_log(2, "\n");
+ if (mad->base_version != 1 || mad->class_version != 1) {
+ acm_log(0, "ERROR - invalid version %d %d\n",
+ mad->base_version, mad->class_version);
+ return;
+ }
+
+ if (mad->control != ACM_CTRL_RESOLVE) {
+ acm_log(0, "ERROR - invalid control 0x%x\n", mad->control);
+ return;
+ }
+
+ rec = (struct acm_resolve_rec *) mad->data;
+ acm_format_name(2, log_data, sizeof log_data,
+ rec->src_type, rec->src, sizeof rec->src);
+ acm_log(2, "src %s\n", log_data);
+ acm_format_name(2, log_data, sizeof log_data,
+ rec->dest_type, rec->dest, sizeof rec->dest);
+ acm_log(2, "dest %s\n", log_data);
+ if (mad->method & IB_METHOD_RESP) {
+ acm_log(2, "received response\n");
+ req = acmp_get_request(ep, mad->tid, &free);
+ if (!req) {
+ acm_log(1, "notice - response did not match active request\n");
+ return;
+ }
+ acm_log(2, "found matching request\n");
+ req->resp_handler(req, wc, mad);
+ if (free)
+ acmp_free_send(req);
+ } else {
+ acm_log(2, "unsolicited request\n");
+ acmp_process_addr_req(ep, wc, mad);
+ }
+}
+
+static void
+acmp_sa_resp(struct acm_sa_mad *mad)
+{
+ struct acmp_request *req = (struct acmp_request *) mad->context;
+ struct ib_sa_mad *sa_mad = (struct ib_sa_mad *) &mad->sa_mad;
+
+ req->msg.hdr.opcode |= ACM_OP_ACK;
+ if (!mad->umad.status) {
+ req->msg.hdr.status = (uint8_t) (ntohs(sa_mad->status) >> 8);
+ memcpy(&req->msg.resolve_data[0].info.path, sa_mad->data,
+ sizeof(struct ibv_path_record));
+ } else {
+ req->msg.hdr.status = ACM_STATUS_ETIMEDOUT;
+ }
+ acm_log(2, "status 0x%x\n", req->msg.hdr.status);
+
+ if (req->msg.hdr.status)
+ atomic_inc(&req->ep->counters[ACM_CNTR_ERROR]);
+ acm_query_response(req->id, &req->msg);
+ acm_free_sa_mad(mad);
+ acmp_free_req(req);
+}
+
+static void acmp_process_sa_recv(struct acmp_ep *ep, struct ibv_wc *wc, struct acm_mad *mad)
+{
+ struct ib_sa_mad *sa_mad = (struct ib_sa_mad *) mad;
+ struct acmp_send_msg *req;
+ int free;
+
+ acm_log(2, "\n");
+ if (mad->base_version != 1 || mad->class_version != 2 ||
+ !(mad->method & IB_METHOD_RESP) || sa_mad->attr_id != IB_SA_ATTR_PATH_REC) {
+ acm_log(0, "ERROR - unexpected SA MAD %d %d\n",
+ mad->base_version, mad->class_version);
+ return;
+ }
+
+ req = acmp_get_request(ep, mad->tid, &free);
+ if (!req) {
+ acm_log(1, "notice - response did not match active request\n");
+ return;
+ }
+ acm_log(2, "found matching request\n");
+ req->resp_handler(req, wc, mad);
+ if (free)
+ acmp_free_send(req);
+}
+
+static void acmp_process_recv(struct acmp_ep *ep, struct ibv_wc *wc)
+{
+ struct acm_mad *mad;
+
+ acm_log(2, "base endpoint name %s\n", ep->id_string);
+ mad = (struct acm_mad *) (uintptr_t) (wc->wr_id + sizeof(struct ibv_grh));
+ switch (mad->mgmt_class) {
+ case IB_MGMT_CLASS_SA:
+ acmp_process_sa_recv(ep, wc, mad);
+ break;
+ case ACM_MGMT_CLASS:
+ acmp_process_acm_recv(ep, wc, mad);
+ break;
+ default:
+ acm_log(0, "ERROR - invalid mgmt class 0x%x\n", mad->mgmt_class);
+ break;
+ }
+
+ acmp_post_recv(ep, wc->wr_id);
+}
+
+static void acmp_process_comp(struct acmp_ep *ep, struct ibv_wc *wc)
+{
+ if (wc->status) {
+ acm_log(0, "ERROR - work completion error\n"
+ "\topcode %d, completion status %d\n",
+ wc->opcode, wc->status);
+ return;
+ }
+
+ if (wc->opcode & IBV_WC_RECV)
+ acmp_process_recv(ep, wc);
+ else
+ acmp_complete_send((struct acmp_send_msg *) (uintptr_t) wc->wr_id);
+}
+
+static void *acmp_comp_handler(void *context)
+{
+ struct acmp_device *dev = (struct acmp_device *) context;
+ struct acmp_ep *ep;
+ struct ibv_cq *cq;
+ struct ibv_wc wc;
+ int cnt;
+
+ acm_log(1, "started\n");
+
+ if (pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL)) {
+ acm_log(0, "Error: failed to set cancel type for dev %s\n",
+ dev->verbs->device->name);
+ pthread_exit(NULL);
+ }
+
+ if (pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL)) {
+ acm_log(0, "Error: failed to set cancel state for dev %s\n",
+ dev->verbs->device->name);
+ pthread_exit(NULL);
+ }
+ while (1) {
+ pthread_testcancel();
+ ibv_get_cq_event(dev->channel, &cq, (void *) &ep);
+
+ cnt = 0;
+ while (ibv_poll_cq(cq, 1, &wc) > 0) {
+ cnt++;
+ acmp_process_comp(ep, &wc);
+ }
+
+ ibv_req_notify_cq(cq, 0);
+ while (ibv_poll_cq(cq, 1, &wc) > 0) {
+ cnt++;
+ acmp_process_comp(ep, &wc);
+ }
+
+ ibv_ack_cq_events(cq, cnt);
+ }
+
+ return NULL;
+}
+
+static void acmp_format_mgid(union ibv_gid *mgid, uint16_t pkey, uint8_t tos,
+ uint8_t rate, uint8_t mtu)
+{
+ mgid->raw[0] = 0xFF;
+ mgid->raw[1] = 0x10 | 0x05;
+ mgid->raw[2] = 0x40;
+ mgid->raw[3] = 0x01;
+ mgid->raw[4] = (uint8_t) (pkey >> 8);
+ mgid->raw[5] = (uint8_t) pkey;
+ mgid->raw[6] = tos;
+ mgid->raw[7] = rate;
+ mgid->raw[8] = mtu;
+ mgid->raw[9] = 0;
+ mgid->raw[10] = 0;
+ mgid->raw[11] = 0;
+ mgid->raw[12] = 0;
+ mgid->raw[13] = 0;
+ mgid->raw[14] = 0;
+ mgid->raw[15] = 0;
+}
+
+static void acmp_init_join(struct ib_sa_mad *mad, union ibv_gid *port_gid,
+ uint16_t pkey, uint8_t tos, uint8_t tclass, uint8_t sl, uint8_t rate, uint8_t mtu)
+{
+ struct ib_mc_member_rec *mc_rec;
+
+ acm_log(2, "\n");
+ mad->base_version = 1;
+ mad->mgmt_class = IB_MGMT_CLASS_SA;
+ mad->class_version = 2;
+ mad->method = IB_METHOD_SET;
+ mad->tid = htonll((uint64_t) atomic_inc(&tid));
+ mad->attr_id = IB_SA_ATTR_MC_MEMBER_REC;
+ mad->comp_mask =
+ IB_COMP_MASK_MC_MGID | IB_COMP_MASK_MC_PORT_GID |
+ IB_COMP_MASK_MC_QKEY | IB_COMP_MASK_MC_MTU_SEL| IB_COMP_MASK_MC_MTU |
+ IB_COMP_MASK_MC_TCLASS | IB_COMP_MASK_MC_PKEY | IB_COMP_MASK_MC_RATE_SEL |
+ IB_COMP_MASK_MC_RATE | IB_COMP_MASK_MC_SL | IB_COMP_MASK_MC_FLOW |
+ IB_COMP_MASK_MC_SCOPE | IB_COMP_MASK_MC_JOIN_STATE;
+
+ mc_rec = (struct ib_mc_member_rec *) mad->data;
+ acmp_format_mgid(&mc_rec->mgid, pkey | 0x8000, tos, rate, mtu);
+ mc_rec->port_gid = *port_gid;
+ mc_rec->qkey = htonl(ACM_QKEY);
+ mc_rec->mtu = 0x80 | mtu;
+ mc_rec->tclass = tclass;
+ mc_rec->pkey = htons(pkey);
+ mc_rec->rate = 0x80 | rate;
+ mc_rec->sl_flow_hop = htonl(((uint32_t) sl) << 28);
+ mc_rec->scope_state = 0x51;
+}
+
+static void acmp_join_group(struct acmp_ep *ep, union ibv_gid *port_gid,
+ uint8_t tos, uint8_t tclass, uint8_t sl, uint8_t rate, uint8_t mtu)
+{
+ struct ib_sa_mad *mad;
+ struct ib_mc_member_rec *mc_rec;
+ struct acm_sa_mad *sa_mad;
+
+ acm_log(2, "\n");
+ sa_mad = acm_alloc_sa_mad(ep->endpoint, ep, acmp_process_join_resp);
+ if (!sa_mad) {
+ acm_log(0, "Error - failed to allocate sa_mad\n");
+ return;
+ }
+
+ acm_log(0, "%s %d pkey 0x%x, sl 0x%x, rate 0x%x, mtu 0x%x\n",
+ ep->port->dev->verbs->device->name,
+ ep->port->port_num, ep->pkey, sl, rate, mtu);
+ mad = (struct ib_sa_mad *) &sa_mad->sa_mad;
+ acmp_init_join(mad, port_gid, ep->pkey, tos, tclass, sl, rate, mtu);
+ mc_rec = (struct ib_mc_member_rec *) mad->data;
+ acmp_set_dest_addr(&ep->mc_dest[ep->mc_cnt++], ACM_ADDRESS_GID,
+ mc_rec->mgid.raw, sizeof(mc_rec->mgid));
+ ep->mc_dest[ep->mc_cnt - 1].state = ACMP_INIT;
+
+ if (acm_send_sa_mad(sa_mad)) {
+ acm_log(0, "Error - Failed to send sa mad\n");
+ acm_free_sa_mad(sa_mad);
+ }
+}
+
+static void acmp_ep_join(struct acmp_ep *ep)
+{
+ struct acmp_port *port;
+ union ibv_gid gid;
+
+ port = ep->port;
+ acm_log(1, "%s\n", ep->id_string);
+
+ if (ep->mc_dest[0].state == ACMP_READY && ep->mc_dest[0].ah) {
+ ibv_detach_mcast(ep->qp, &ep->mc_dest[0].mgid,
+ ntohs(ep->mc_dest[0].av.dlid));
+ ibv_destroy_ah(ep->mc_dest[0].ah);
+ ep->mc_dest[0].ah = NULL;
+ }
+ ep->mc_cnt = 0;
+ ep->state = ACMP_INIT;
+ acm_get_gid((struct acm_port *)ep->port->port, 0, &gid);
+ acmp_join_group(ep, &gid, 0, 0, 0, min_rate, min_mtu);
+
+ if ((route_prot == ACMP_ROUTE_PROT_ACM) &&
+ (port->rate != min_rate || port->mtu != min_mtu))
+ acmp_join_group(ep, &gid, 0, 0, 0, port->rate, port->mtu);
+
+ acm_log(1, "join for %s complete\n", ep->id_string);
+}
+
+static int acmp_port_join(void *port_context)
+{
+ struct acmp_ep *ep;
+ DLIST_ENTRY *ep_entry;
+ struct acmp_port *port = port_context;
+
+ acm_log(1, "device %s port %d\n", port->dev->verbs->device->name,
+ port->port_num);
+
+ for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list;
+ ep_entry = ep_entry->Next) {
+ ep = container_of(ep_entry, struct acmp_ep, entry);
+ if (!ep->endpoint) {
+ /* Stale endpoint */
+ continue;
+ }
+ acmp_ep_join(ep);
+ }
+ acm_log(1, "joins for device %s port %d complete\n",
+ port->dev->verbs->device->name, port->port_num);
+
+ return 0;
+}
+
+static int acmp_handle_event(void *port_context, enum ibv_event_type type)
+{
+ int ret = 0;
+
+ acm_log(2, "event %s\n", ibv_event_type_str(type));
+
+ switch (type) {
+ case IBV_EVENT_CLIENT_REREGISTER:
+ ret = acmp_port_join(port_context);
+ break;
+ default:
+ break;
+ }
+ return ret;
+}
+
+static void acmp_process_timeouts(void)
+{
+ DLIST_ENTRY *entry;
+ struct acmp_send_msg *msg;
+ struct acm_resolve_rec *rec;
+ struct acm_mad *mad;
+
+ while (!DListEmpty(&timeout_list)) {
+ entry = timeout_list.Next;
+ DListRemove(entry);
+
+ msg = container_of(entry, struct acmp_send_msg, entry);
+ mad = (struct acm_mad *) &msg->data[0];
+ rec = (struct acm_resolve_rec *) mad->data;
+
+ acm_format_name(0, log_data, sizeof log_data,
+ rec->dest_type, rec->dest, sizeof rec->dest);
+ acm_log(0, "notice - dest %s\n", log_data);
+
+ msg->resp_handler(msg, NULL, NULL);
+ acmp_free_send(msg);
+ }
+}
+
+static void acmp_process_wait_queue(struct acmp_ep *ep, uint64_t *next_expire)
+{
+ struct acmp_send_msg *msg;
+ DLIST_ENTRY *entry, *next;
+ struct ibv_send_wr *bad_wr;
+
+ for (entry = ep->wait_queue.Next; entry != &ep->wait_queue; entry = next) {
+ next = entry->Next;
+ msg = container_of(entry, struct acmp_send_msg, entry);
+ if (msg->expires < time_stamp_ms()) {
+ DListRemove(entry);
+ (void) atomic_dec(&wait_cnt);
+ if (--msg->tries) {
+ acm_log(1, "notice - retrying request\n");
+ DListInsertTail(&msg->entry, &ep->active_queue);
+ ibv_post_send(ep->qp, &msg->wr, &bad_wr);
+ } else {
+ acm_log(0, "notice - failing request\n");
+ acmp_send_available(ep, msg->req_queue);
+ DListInsertTail(&msg->entry, &timeout_list);
+ }
+ } else {
+ *next_expire = min(*next_expire, msg->expires);
+ break;
+ }
+ }
+}
+
+/* While the device/port/ep will not be freed, we need to be careful of
+ * their addition while walking the link lists. Therefore, we need to acquire
+ * the appropriate locks.
+ */
+static void *acmp_retry_handler(void *context)
+{
+ struct acmp_device *dev;
+ struct acmp_port *port;
+ struct acmp_ep *ep;
+ DLIST_ENTRY *dev_entry, *ep_entry;
+ uint64_t next_expire;
+ int i, wait;
+
+ acm_log(0, "started\n");
+ if (pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL)) {
+ acm_log(0, "Error: failed to set cancel type \n");
+ pthread_exit(NULL);
+ }
+ if (pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL)) {
+ acm_log(0, "Error: failed to set cancel state\n");
+ pthread_exit(NULL);
+ }
+ retry_thread_started = 1;
+
+ while (1) {
+ while (!atomic_get(&wait_cnt)) {
+ pthread_testcancel();
+ event_wait(&timeout_event, -1);
+ }
+
+ next_expire = -1;
+ lock_acquire(&acmp_dev_lock);
+ for (dev_entry = acmp_dev_list.Next; dev_entry != &acmp_dev_list;
+ dev_entry = dev_entry->Next) {
+
+ dev = container_of(dev_entry, struct acmp_device, entry);
+ lock_release(&acmp_dev_lock);
+
+ for (i = 0; i < dev->port_cnt; i++) {
+ port = &dev->port[i];
+
+ lock_acquire(&port->lock);
+ for (ep_entry = port->ep_list.Next;
+ ep_entry != &port->ep_list;
+ ep_entry = ep_entry->Next) {
+
+ ep = container_of(ep_entry, struct acmp_ep, entry);
+ lock_release(&port->lock);
+ lock_acquire(&ep->lock);
+ if (!DListEmpty(&ep->wait_queue))
+ acmp_process_wait_queue(ep, &next_expire);
+ lock_release(&ep->lock);
+ lock_acquire(&port->lock);
+ }
+ lock_release(&port->lock);
+ }
+ lock_acquire(&acmp_dev_lock);
+ }
+ lock_release(&acmp_dev_lock);
+
+ acmp_process_timeouts();
+ wait = (int) (next_expire - time_stamp_ms());
+ if (wait > 0 && atomic_get(&wait_cnt)) {
+ pthread_testcancel();
+ event_wait(&timeout_event, wait);
+ }
+ }
+
+ retry_thread_started = 0;
+ return NULL;
+}
+
+static int
+acmp_query(void *addr_context, struct acm_msg *msg, uint64_t id)
+{
+ struct acmp_request *req;
+ struct ib_sa_mad *mad;
+ struct acmp_addr *address = addr_context;
+ struct acmp_ep *ep = address->ep;
+ uint8_t status;
+ struct acm_sa_mad *sa_mad;
+
+ if (ep->state != ACMP_READY) {
+ status = ACM_STATUS_ENODATA;
+ goto resp;
+ }
+
+ req = acmp_alloc_req(id, msg);
+ if (!req) {
+ status = ACM_STATUS_ENOMEM;
+ goto resp;
+ }
+ req->ep = ep;
+
+ sa_mad = acm_alloc_sa_mad(ep->endpoint, req, acmp_sa_resp);
+ if (!sa_mad) {
+ acm_log(0, "Error - failed to allocate sa_mad\n");
+ status = ACM_STATUS_ENOMEM;
+ goto free_req;
+ }
+
+ mad = (struct ib_sa_mad *) &sa_mad->sa_mad;
+ acmp_init_path_query(mad);
+
+ memcpy(mad->data, &msg->resolve_data[0].info.path,
+ sizeof(struct ibv_path_record));
+ mad->comp_mask = acm_path_comp_mask(&msg->resolve_data[0].info.path);
+
+ acm_increment_counter(ACM_CNTR_ROUTE_QUERY);
+ atomic_inc(&ep->counters[ACM_CNTR_ROUTE_QUERY]);
+ if (acm_send_sa_mad(sa_mad)) {
+ acm_log(0, "Error - Failed to send sa mad\n");
+ status = ACM_STATUS_ENODATA;
+ goto free_mad;
+ }
+ return ACM_STATUS_SUCCESS;
+
+free_mad:
+ acm_free_sa_mad(sa_mad);
+free_req:
+ acmp_free_req(req);
+resp:
+ msg->hdr.opcode |= ACM_OP_ACK;
+ msg->hdr.status = status;
+ if (status == ACM_STATUS_ENODATA)
+ atomic_inc(&ep->counters[ACM_CNTR_NODATA]);
+ else
+ atomic_inc(&ep->counters[ACM_CNTR_ERROR]);
+ return acm_query_response(id, msg);
+}
+
+static uint8_t
+acmp_send_resolve(struct acmp_ep *ep, struct acmp_dest *dest,
+ struct acm_ep_addr_data *saddr)
+{
+ struct acmp_send_msg *msg;
+ struct acm_mad *mad;
+ struct acm_resolve_rec *rec;
+ int i;
+
+ acm_log(2, "\n");
+ msg = acmp_alloc_send(ep, &ep->mc_dest[0], sizeof(*mad));
+ if (!msg) {
+ acm_log(0, "ERROR - cannot allocate send msg\n");
+ return ACM_STATUS_ENOMEM;
+ }
+
+ acmp_init_send_req(msg, (void *) dest, acmp_process_addr_resp);
+ (void) atomic_inc(&dest->refcnt);
+
+ mad = (struct acm_mad *) msg->data;
+ mad->base_version = 1;
+ mad->mgmt_class = ACM_MGMT_CLASS;
+ mad->class_version = 1;
+ mad->method = IB_METHOD_GET;
+ mad->control = ACM_CTRL_RESOLVE;
+ mad->tid = htonll((uint64_t) atomic_inc(&tid));
+
+ rec = (struct acm_resolve_rec *) mad->data;
+ rec->src_type = (uint8_t) saddr->type;
+ rec->src_length = ACM_MAX_ADDRESS;
+ memcpy(rec->src, saddr->info.addr, ACM_MAX_ADDRESS);
+ rec->dest_type = dest->addr_type;
+ rec->dest_length = ACM_MAX_ADDRESS;
+ memcpy(rec->dest, dest->address, ACM_MAX_ADDRESS);
+
+ rec->gid_cnt = (uint8_t) ep->mc_cnt;
+ for (i = 0; i < ep->mc_cnt; i++)
+ memcpy(&rec->gid[i], ep->mc_dest[i].address, 16);
+
+ acm_increment_counter(ACM_CNTR_ADDR_QUERY);
+ atomic_inc(&ep->counters[ACM_CNTR_ADDR_QUERY]);
+ acmp_post_send(&ep->resolve_queue, msg);
+ return 0;
+}
+
+/* Caller must hold dest lock */
+static uint8_t acmp_queue_req(struct acmp_dest *dest, uint64_t id, struct acm_msg *msg)
+{
+ struct acmp_request *req;
+
+ acm_log(2, "id %llu\n", id);
+ req = acmp_alloc_req(id, msg);
+ if (!req) {
+ return ACM_STATUS_ENOMEM;
+ }
+ req->ep = dest->ep;
+
+ DListInsertTail(&req->entry, &dest->req_queue);
+ return ACM_STATUS_SUCCESS;
+}
+
+static int acmp_dest_timeout(struct acmp_dest *dest)
+{
+ uint64_t timestamp = time_stamp_min();
+
+ if (timestamp > dest->addr_timeout) {
+ acm_log(2, "%s address timed out\n", dest->name);
+ dest->state = ACMP_INIT;
+ return 1;
+ } else if (timestamp > dest->route_timeout) {
+ acm_log(2, "%s route timed out\n", dest->name);
+ dest->state = ACMP_ADDR_RESOLVED;
+ return 1;
+ }
+ return 0;
+}
+
+static int
+acmp_resolve_dest(struct acmp_ep *ep, struct acm_msg *msg, uint64_t id)
+{
+ struct acmp_dest *dest;
+ struct acm_ep_addr_data *saddr, *daddr;
+ uint8_t status;
+ int ret;
+
+ saddr = &msg->resolve_data[msg->hdr.src_index];
+ daddr = &msg->resolve_data[msg->hdr.dst_index];
+ acm_format_name(2, log_data, sizeof log_data,
+ daddr->type, daddr->info.addr, sizeof daddr->info.addr);
+ acm_log(2, "dest %s\n", log_data);
+
+ dest = acmp_acquire_dest(ep, daddr->type, daddr->info.addr);
+ if (!dest) {
+ acm_log(0, "ERROR - unable to allocate destination in request\n");
+ atomic_inc(&ep->counters[ACM_CNTR_ERROR]);
+ return acmp_resolve_response(id, msg, NULL, ACM_STATUS_ENOMEM);
+ }
+
+ lock_acquire(&dest->lock);
+test:
+ switch (dest->state) {
+ case ACMP_READY:
+ if (acmp_dest_timeout(dest))
+ goto test;
+ acm_log(2, "request satisfied from local cache\n");
+ acm_increment_counter(ACM_CNTR_ROUTE_CACHE);
+ atomic_inc(&ep->counters[ACM_CNTR_ROUTE_CACHE]);
+ status = ACM_STATUS_SUCCESS;
+ break;
+ case ACMP_ADDR_RESOLVED:
+ acm_log(2, "have address, resolving route\n");
+ acm_increment_counter(ACM_CNTR_ADDR_CACHE);
+ atomic_inc(&ep->counters[ACM_CNTR_ADDR_CACHE]);
+ status = acmp_resolve_path_sa(ep, dest, acmp_dest_sa_resp);
+ if (status) {
+ break;
+ }
+ goto queue;
+ case ACMP_INIT:
+ acm_log(2, "sending resolve msg to dest\n");
+ status = acmp_send_resolve(ep, dest, saddr);
+ if (status) {
+ break;
+ }
+ dest->state = ACMP_QUERY_ADDR;
+ /* fall through */
+ default:
+queue:
+ if (daddr->flags & ACM_FLAGS_NODELAY) {
+ acm_log(2, "lookup initiated, but client wants no delay\n");
+ status = ACM_STATUS_ENODATA;
+ break;
+ }
+ status = acmp_queue_req(dest, id, msg);
+ if (status) {
+ break;
+ }
+ ret = 0;
+ lock_release(&dest->lock);
+ goto put;
+ }
+ lock_release(&dest->lock);
+ ret = acmp_resolve_response(id, msg, dest, status);
+put:
+ acmp_put_dest(dest);
+ return ret;
+}
+
+static int
+acmp_resolve_path(struct acmp_ep *ep, struct acm_msg *msg, uint64_t id)
+{
+ struct acmp_dest *dest;
+ struct ibv_path_record *path;
+ uint8_t *addr;
+ uint8_t status;
+ int ret;
+
+ path = &msg->resolve_data[0].info.path;
+ addr = msg->resolve_data[1].info.addr;
+ memset(addr, 0, ACM_MAX_ADDRESS);
+ if (path->dlid) {
+ * ((uint16_t *) addr) = path->dlid;
+ dest = acmp_acquire_dest(ep, ACM_ADDRESS_LID, addr);
+ } else {
+ memcpy(addr, &path->dgid, sizeof path->dgid);
+ dest = acmp_acquire_dest(ep, ACM_ADDRESS_GID, addr);
+ }
+ if (!dest) {
+ acm_log(0, "ERROR - unable to allocate destination in request\n");
+ atomic_inc(&ep->counters[ACM_CNTR_ERROR]);
+ return acmp_resolve_response(id, msg, NULL, ACM_STATUS_ENOMEM);
+ }
+
+ lock_acquire(&dest->lock);
+test:
+ switch (dest->state) {
+ case ACMP_READY:
+ if (acmp_dest_timeout(dest))
+ goto test;
+ acm_log(2, "request satisfied from local cache\n");
+ acm_increment_counter(ACM_CNTR_ROUTE_CACHE);
+ atomic_inc(&ep->counters[ACM_CNTR_ROUTE_CACHE]);
+ status = ACM_STATUS_SUCCESS;
+ break;
+ case ACMP_INIT:
+ acm_log(2, "have path, bypassing address resolution\n");
+ acmp_record_path_addr(ep, dest, path);
+ /* fall through */
+ case ACMP_ADDR_RESOLVED:
+ acm_log(2, "have address, resolving route\n");
+ status = acmp_resolve_path_sa(ep, dest, acmp_dest_sa_resp);
+ if (status) {
+ break;
+ }
+ /* fall through */
+ default:
+ if (msg->resolve_data[0].flags & ACM_FLAGS_NODELAY) {
+ acm_log(2, "lookup initiated, but client wants no delay\n");
+ status = ACM_STATUS_ENODATA;
+ break;
+ }
+ status = acmp_queue_req(dest, id, msg);
+ if (status) {
+ break;
+ }
+ ret = 0;
+ lock_release(&dest->lock);
+ goto put;
+ }
+ lock_release(&dest->lock);
+ ret = acmp_resolve_response(id, msg, dest, status);
+put:
+ acmp_put_dest(dest);
+ return ret;
+}
+
+static int
+acmp_resolve(void *addr_context, struct acm_msg *msg, uint64_t id)
+{
+ struct acmp_addr *address = addr_context;
+ struct acmp_ep *ep = address->ep;
+
+ if (ep->state != ACMP_READY) {
+ atomic_inc(&ep->counters[ACM_CNTR_NODATA]);
+ return acmp_resolve_response(id, msg, NULL, ACM_STATUS_ENODATA);
+ }
+
+ atomic_inc(&ep->counters[ACM_CNTR_RESOLVE]);
+ if (msg->resolve_data[0].type == ACM_EP_INFO_PATH)
+ return acmp_resolve_path(ep, msg, id);
+ else
+ return acmp_resolve_dest(ep, msg, id);
+}
+
+static void acmp_query_perf(void *ep_context, uint64_t *values, uint8_t *cnt)
+{
+ struct acmp_ep *ep = ep_context;
+ int i;
+
+ for (i = 0; i < ACM_MAX_COUNTER; i++)
+ values[i] = htonll((uint64_t) atomic_get(&ep->counters[i]));
+ *cnt = ACM_MAX_COUNTER;
+}
+
+static enum acmp_addr_prot acmp_convert_addr_prot(char *param)
+{
+ if (!stricmp("acm", param))
+ return ACMP_ADDR_PROT_ACM;
+
+ return addr_prot;
+}
+
+static enum acmp_route_prot acmp_convert_route_prot(char *param)
+{
+ if (!stricmp("acm", param))
+ return ACMP_ROUTE_PROT_ACM;
+ else if (!stricmp("sa", param))
+ return ACMP_ROUTE_PROT_SA;
+
+ return route_prot;
+}
+
+static enum acmp_loopback_prot acmp_convert_loopback_prot(char *param)
+{
+ if (!stricmp("none", param))
+ return ACMP_LOOPBACK_PROT_NONE;
+ else if (!stricmp("local", param))
+ return ACMP_LOOPBACK_PROT_LOCAL;
+
+ return loopback_prot;
+}
+
+static enum acmp_route_preload acmp_convert_route_preload(char *param)
+{
+ if (!stricmp("none", param) || !stricmp("no", param))
+ return ACMP_ROUTE_PRELOAD_NONE;
+ else if (!stricmp("opensm_full_v1", param))
+ return ACMP_ROUTE_PRELOAD_OSM_FULL_V1;
+
+ return route_preload;
+}
+
+static enum acmp_addr_preload acmp_convert_addr_preload(char *param)
+{
+ if (!stricmp("none", param) || !stricmp("no", param))
+ return ACMP_ADDR_PRELOAD_NONE;
+ else if (!stricmp("acm_hosts", param))
+ return ACMP_ADDR_PRELOAD_HOSTS;
+
+ return addr_preload;
+}
+
+static int acmp_post_recvs(struct acmp_ep *ep)
+{
+ int i, size;
+
+ size = recv_depth * ACM_RECV_SIZE;
+ ep->recv_bufs = malloc(size);
+ if (!ep->recv_bufs) {
+ acm_log(0, "ERROR - unable to allocate receive buffer\n");
+ return ACM_STATUS_ENOMEM;
+ }
+
+ ep->mr = ibv_reg_mr(ep->port->dev->pd, ep->recv_bufs, size,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (!ep->mr) {
+ acm_log(0, "ERROR - unable to register receive buffer\n");
+ goto err;
+ }
+
+ for (i = 0; i < recv_depth; i++) {
+ acmp_post_recv(ep, (uintptr_t) (ep->recv_bufs + ACM_RECV_SIZE * i));
+ }
+ return 0;
+
+err:
+ free(ep->recv_bufs);
+ return -1;
+}
+
+/* Parse "opensm full v1" file to build LID to GUID table */
+static void acmp_parse_osm_fullv1_lid2guid(FILE *f, uint64_t *lid2guid)
+{
+ char s[128];
+ char *p, *ptr, *p_guid, *p_lid;
+ uint64_t guid;
+ uint16_t lid;
+
+ while (fgets(s, sizeof s, f)) {
+ if (s[0] == '#')
+ continue;
+ if (!(p = strtok_r(s, " \n", &ptr)))
+ continue; /* ignore blank lines */
+
+ if (strncmp(p, "Switch", sizeof("Switch") - 1) &&
+ strncmp(p, "Channel", sizeof("Channel") - 1) &&
+ strncmp(p, "Router", sizeof("Router") - 1))
+ continue;
+
+ if (!strncmp(p, "Channel", sizeof("Channel") - 1)) {
+ p = strtok_r(NULL, " ", &ptr); /* skip 'Adapter' */
+ if (!p)
+ continue;
+ }
+
+ p_guid = strtok_r(NULL, ",", &ptr);
+ if (!p_guid)
+ continue;
+
+ guid = (uint64_t) strtoull(p_guid, NULL, 16);
+
+ ptr = strstr(ptr, "base LID");
+ if (!ptr)
+ continue;
+ ptr += sizeof("base LID");
+ p_lid = strtok_r(NULL, ",", &ptr);
+ if (!p_lid)
+ continue;
+
+ lid = (uint16_t) strtoul(p_lid, NULL, 0);
+ if (lid >= IB_LID_MCAST_START)
+ continue;
+ if (lid2guid[lid])
+ acm_log(0, "ERROR - duplicate lid %u\n", lid);
+ else
+ lid2guid[lid] = htonll(guid);
+ }
+}
+
+/* Parse 'opensm full v1' file to populate PR cache */
+static int acmp_parse_osm_fullv1_paths(FILE *f, uint64_t *lid2guid, struct acmp_ep *ep)
+{
+ union ibv_gid sgid, dgid;
+ struct ibv_port_attr attr = { 0 };
+ struct acmp_dest *dest;
+ char s[128];
+ char *p, *ptr, *p_guid, *p_lid;
+ uint64_t guid;
+ uint16_t lid, dlid, net_dlid;
+ int sl, mtu, rate;
+ int ret = 1, i;
+ uint8_t addr[ACM_MAX_ADDRESS];
+ uint8_t addr_type;
+
+ acm_get_gid((struct acm_port *)ep->port->port, 0, &sgid);
+
+ /* Search for endpoint's SLID */
+ while (fgets(s, sizeof s, f)) {
+ if (s[0] == '#')
+ continue;
+ if (!(p = strtok_r(s, " \n", &ptr)))
+ continue; /* ignore blank lines */
+
+ if (strncmp(p, "Switch", sizeof("Switch") - 1) &&
+ strncmp(p, "Channel", sizeof("Channel") - 1) &&
+ strncmp(p, "Router", sizeof("Router") - 1))
+ continue;
+
+ if (!strncmp(p, "Channel", sizeof("Channel") - 1)) {
+ p = strtok_r(NULL, " ", &ptr); /* skip 'Adapter' */
+ if (!p)
+ continue;
+ }
+
+ p_guid = strtok_r(NULL, ",", &ptr);
+ if (!p_guid)
+ continue;
+
+ guid = (uint64_t) strtoull(p_guid, NULL, 16);
+ if (guid != ntohll(sgid.global.interface_id))
+ continue;
+
+ ptr = strstr(ptr, "base LID");
+ if (!ptr)
+ continue;
+ ptr += sizeof("base LID");
+ p_lid = strtok_r(NULL, ",", &ptr);
+ if (!p_lid)
+ continue;
+
+ lid = (uint16_t) strtoul(p_lid, NULL, 0);
+ if (lid != ep->port->lid)
+ continue;
+
+ ibv_query_port(ep->port->dev->verbs, ep->port->port_num, &attr);
+ ret = 0;
+ break;
+ }
+
+ while (fgets(s, sizeof s, f)) {
+ if (s[0] == '#')
+ continue;
+ if (!(p = strtok_r(s, " \n", &ptr)))
+ continue; /* ignore blank lines */
+
+ if (!strncmp(p, "Switch", sizeof("Switch") - 1) ||
+ !strncmp(p, "Channel", sizeof("Channel") - 1) ||
+ !strncmp(p, "Router", sizeof("Router") - 1))
+ break;
+
+ dlid = strtoul(p, NULL, 0);
+ net_dlid = htons(dlid);
+
+ p = strtok_r(NULL, ":", &ptr);
+ if (!p)
+ continue;
+ if (strcmp(p, "UNREACHABLE") == 0)
+ continue;
+ sl = atoi(p);
+
+ p = strtok_r(NULL, ":", &ptr);
+ if (!p)
+ continue;
+ mtu = atoi(p);
+
+ p = strtok_r(NULL, ":", &ptr);
+ if (!p)
+ continue;
+ rate = atoi(p);
+
+ if (!lid2guid[dlid]) {
+ acm_log(0, "ERROR - dlid %u not found in lid2guid table\n", dlid);
+ continue;
+ }
+
+ dgid.global.subnet_prefix = sgid.global.subnet_prefix;
+ dgid.global.interface_id = lid2guid[dlid];
+
+ for (i = 0; i < 2; i++) {
+ memset(addr, 0, ACM_MAX_ADDRESS);
+ if (i == 0) {
+ addr_type = ACM_ADDRESS_LID;
+ memcpy(addr, &net_dlid, sizeof net_dlid);
+ } else {
+ addr_type = ACM_ADDRESS_GID;
+ memcpy(addr, &dgid, sizeof(dgid));
+ }
+ dest = acmp_acquire_dest(ep, addr_type, addr);
+ if (!dest) {
+ acm_log(0, "ERROR - unable to create dest\n");
+ break;
+ }
+
+ dest->path.sgid = sgid;
+ dest->path.slid = htons(ep->port->lid);
+ dest->path.dgid = dgid;
+ dest->path.dlid = net_dlid;
+ dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE;
+ dest->path.pkey = htons(ep->pkey);
+ dest->path.mtu = (uint8_t) mtu;
+ dest->path.rate = (uint8_t) rate;
+ dest->path.qosclass_sl = htons((uint16_t) sl & 0xF);
+ if (dlid == ep->port->lid) {
+ dest->path.packetlifetime = 0;
+ dest->addr_timeout = (uint64_t)~0ULL;
+ dest->route_timeout = (uint64_t)~0ULL;
+ } else {
+ dest->path.packetlifetime = attr.subnet_timeout;
+ dest->addr_timeout = time_stamp_min() + (unsigned) addr_timeout;
+ dest->route_timeout = time_stamp_min() + (unsigned) route_timeout;
+ }
+ dest->remote_qpn = 1;
+ dest->state = ACMP_READY;
+ acmp_put_dest(dest);
+ acm_log(1, "added cached dest %s\n", dest->name);
+ }
+ }
+ return ret;
+}
+
+static int acmp_parse_osm_fullv1(struct acmp_ep *ep)
+{
+ FILE *f;
+ uint64_t *lid2guid;
+ int ret = 1;
+
+ if (!(f = fopen(route_data_file, "r"))) {
+ acm_log(0, "ERROR - couldn't open %s\n", route_data_file);
+ return ret;
+ }
+
+ lid2guid = calloc(IB_LID_MCAST_START, sizeof(*lid2guid));
+ if (!lid2guid) {
+ acm_log(0, "ERROR - no memory for path record parsing\n");
+ goto err;
+ }
+
+ acmp_parse_osm_fullv1_lid2guid(f, lid2guid);
+ rewind(f);
+ ret = acmp_parse_osm_fullv1_paths(f, lid2guid, ep);
+ free(lid2guid);
+err:
+ fclose(f);
+ return ret;
+}
+
+static void acmp_parse_hosts_file(struct acmp_ep *ep)
+{
+ FILE *f;
+ char s[120];
+ char addr[INET6_ADDRSTRLEN], gid[INET6_ADDRSTRLEN];
+ uint8_t name[ACM_MAX_ADDRESS];
+ struct in6_addr ip_addr, ib_addr;
+ struct acmp_dest *dest, *gid_dest;
+ uint8_t addr_type;
+
+ if (!(f = fopen(addr_data_file, "r"))) {
+ acm_log(0, "ERROR - couldn't open %s\n", addr_data_file);
+ return;
+ }
+
+ while (fgets(s, sizeof s, f)) {
+ if (s[0] == '#')
+ continue;
+
+ if (sscanf(s, "%46s%46s", addr, gid) != 2)
+ continue;
+
+ acm_log(2, "%s", s);
+ if (inet_pton(AF_INET6, gid, &ib_addr) <= 0) {
+ acm_log(0, "ERROR - %s is not IB GID\n", gid);
+ continue;
+ }
+ memset(name, 0, ACM_MAX_ADDRESS);
+ if (inet_pton(AF_INET, addr, &ip_addr) > 0) {
+ addr_type = ACM_ADDRESS_IP;
+ memcpy(name, &ip_addr, 4);
+ } else if (inet_pton(AF_INET6, addr, &ip_addr) > 0) {
+ addr_type = ACM_ADDRESS_IP6;
+ memcpy(name, &ip_addr, sizeof(ip_addr));
+ } else {
+ addr_type = ACM_ADDRESS_NAME;
+ strncpy((char *)name, addr, ACM_MAX_ADDRESS);
+ }
+
+ dest = acmp_acquire_dest(ep, addr_type, name);
+ if (!dest) {
+ acm_log(0, "ERROR - unable to create dest %s\n", addr);
+ continue;
+ }
+
+ memset(name, 0, ACM_MAX_ADDRESS);
+ memcpy(name, &ib_addr, sizeof(ib_addr));
+ gid_dest = acmp_get_dest(ep, ACM_ADDRESS_GID, name);
+ if (gid_dest) {
+ dest->path = gid_dest->path;
+ dest->state = ACMP_READY;
+ acmp_put_dest(gid_dest);
+ } else {
+ memcpy(&dest->path.dgid, &ib_addr, 16);
+ //ibv_query_gid(ep->port->dev->verbs, ep->port->port_num,
+ // 0, &dest->path.sgid);
+ dest->path.slid = htons(ep->port->lid);
+ dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE;
+ dest->path.pkey = htons(ep->pkey);
+ dest->state = ACMP_ADDR_RESOLVED;
+ }
+
+ dest->remote_qpn = 1;
+ dest->addr_timeout = time_stamp_min() + (unsigned) addr_timeout;
+ dest->route_timeout = time_stamp_min() + (unsigned) route_timeout;
+ acmp_put_dest(dest);
+ acm_log(1, "added host %s address type %d IB GID %s\n",
+ addr, addr_type, gid);
+ }
+
+ fclose(f);
+}
+
+/*
+ * We currently require that the routing data be preloaded in order to
+ * load the address data. This is backwards from normal operation, which
+ * usually resolves the address before the route.
+ */
+static void acmp_ep_preload(struct acmp_ep *ep)
+{
+ switch (route_preload) {
+ case ACMP_ROUTE_PRELOAD_OSM_FULL_V1:
+ if (acmp_parse_osm_fullv1(ep))
+ acm_log(0, "ERROR - failed to preload EP\n");
+ break;
+ default:
+ break;
+ }
+
+ switch (addr_preload) {
+ case ACMP_ADDR_PRELOAD_HOSTS:
+ acmp_parse_hosts_file(ep);
+ break;
+ default:
+ break;
+ }
+}
+
+static int acmp_add_addr(const struct acm_address *addr, void *ep_context,
+ void **addr_context)
+{
+ struct acmp_ep *ep = ep_context;
+ struct acmp_dest *dest;
+ int i;
+
+ acm_log(2, "\n");
+
+ for (i = 0; (i < MAX_EP_ADDR) &&
+ (ep->addr_info[i].type != ACM_ADDRESS_INVALID); i++)
+ ;
+
+ if (i == MAX_EP_ADDR) {
+ acm_log(0, "ERROR - no more space for local address\n");
+ return -1;
+ }
+ ep->addr_info[i].type = addr->type;
+ memcpy(&ep->addr_info[i].info, &addr->info, sizeof(addr->info));
+ ep->addr_info[i].addr = (struct acm_address *) addr;
+ ep->addr_info[i].ep = ep;
+
+ if (loopback_prot != ACMP_LOOPBACK_PROT_LOCAL) {
+ *addr_context = &ep->addr_info[i];
+ return 0;
+ }
+
+ dest = acmp_acquire_dest(ep, addr->type, (uint8_t *) addr->info.addr);
+ if (!dest) {
+ acm_log(0, "ERROR - unable to create loopback dest %s\n",
+ addr->id_string);
+ memset(&ep->addr_info[i], 0, sizeof(ep->addr_info[i]));
+ return -1;
+ }
+
+ acm_get_gid((struct acm_port *) ep->port->port, 0, &dest->path.sgid);
+ dest->path.dgid = dest->path.sgid;
+ dest->path.dlid = dest->path.slid = htons(ep->port->lid);
+ dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE;
+ dest->path.pkey = htons(ep->pkey);
+ dest->path.mtu = (uint8_t) ep->port->mtu;
+ dest->path.rate = (uint8_t) ep->port->rate;
+
+ dest->remote_qpn = ep->qp->qp_num;
+ dest->addr_timeout = (uint64_t) ~0ULL;
+ dest->route_timeout = (uint64_t) ~0ULL;
+ dest->state = ACMP_READY;
+ acmp_put_dest(dest);
+ *addr_context = &ep->addr_info[i];
+ acm_log(1, "added loopback dest %s\n", dest->name);
+
+ return 0;
+}
+
+static void acmp_remove_addr(void *addr_context)
+{
+ struct acmp_addr *address = addr_context;
+
+ acm_log(2, "\n");
+ memset(address, 0, sizeof(*address));
+}
+
+static struct acmp_port *acmp_get_port(struct acm_endpoint *endpoint)
+{
+ struct acmp_device *dev;
+ DLIST_ENTRY *dev_entry;
+
+ acm_log(1, "dev 0x%llx port %d pkey 0x%x\n",
+ endpoint->port->dev->dev_guid, endpoint->port->port_num,
+ endpoint->pkey);
+ for (dev_entry = acmp_dev_list.Next; dev_entry != &acmp_dev_list;
+ dev_entry = dev_entry->Next) {
+
+ dev = container_of(dev_entry, struct acmp_device, entry);
+ if (dev->guid == endpoint->port->dev->dev_guid)
+ return &dev->port[endpoint->port->port_num - 1];
+ }
+
+ return NULL;
+}
+
+static struct acmp_ep *
+acmp_get_ep(struct acmp_port *port, struct acm_endpoint *endpoint)
+{
+ struct acmp_ep *ep;
+ DLIST_ENTRY *entry;
+
+ acm_log(1, "dev 0x%llx port %d pkey 0x%x\n",
+ endpoint->port->dev->dev_guid, endpoint->port->port_num, endpoint->pkey);
+ for (entry = port->ep_list.Next; entry != &port->ep_list;
+ entry = entry->Next) {
+ ep = container_of(entry, struct acmp_ep, entry);
+ if (ep->pkey == endpoint->pkey)
+ return ep;
+ }
+
+ return NULL;
+}
+
+static uint16_t acmp_get_pkey_index(struct acm_endpoint *endpoint)
+{
+ struct acmp_port *port;
+ int ret;
+ uint16_t pkey, i;
+
+ port = acmp_get_port(endpoint);
+ if (!port)
+ return 0;
+
+ for (i = 0, ret = 0; !ret; i++) {
+ ret = ibv_query_pkey(port->dev->verbs, port->port_num, i, &pkey);
+ if (!ret && endpoint->pkey == pkey)
+ return i;
+ }
+ return 0;
+}
+
+static void acmp_close_endpoint(void *ep_context)
+{
+
+ struct acmp_ep *ep = ep_context;
+
+ acm_log(1, "%s %d pkey 0x%04x\n",
+ ep->port->dev->verbs->device->name,
+ ep->port->port_num, ep->pkey);
+
+ ep->endpoint = NULL;
+}
+
+static struct acmp_ep *
+acmp_alloc_ep(struct acmp_port *port, struct acm_endpoint *endpoint)
+{
+ struct acmp_ep *ep;
+ int i;
+
+ acm_log(1, "\n");
+ ep = calloc(1, sizeof *ep);
+ if (!ep)
+ return NULL;
+
+ ep->port = port;
+ ep->endpoint = endpoint;
+ ep->pkey = endpoint->pkey;
+ ep->resolve_queue.credits = resolve_depth;
+ ep->resp_queue.credits = send_depth;
+ DListInit(&ep->resolve_queue.pending);
+ DListInit(&ep->resp_queue.pending);
+ DListInit(&ep->active_queue);
+ DListInit(&ep->wait_queue);
+ lock_init(&ep->lock);
+ sprintf(ep->id_string, "%s-%d-0x%x", port->dev->verbs->device->name,
+ port->port_num, endpoint->pkey);
+ for (i = 0; i < ACM_MAX_COUNTER; i++)
+ atomic_init(&ep->counters[i]);
+
+ return ep;
+}
+
+static int acmp_open_endpoint(const struct acm_endpoint *endpoint,
+ void *port_context, void **ep_context)
+{
+ struct acmp_port *port = port_context;
+ struct acmp_ep *ep;
+ struct ibv_qp_init_attr init_attr;
+ struct ibv_qp_attr attr;
+ int ret, sq_size;
+
+ ep = acmp_get_ep(port, (struct acm_endpoint *) endpoint);
+ if (ep) {
+ acm_log(2, "endpoint for pkey 0x%x already exists\n", endpoint->pkey);
+ lock_acquire(&ep->lock);
+ ep->endpoint = (struct acm_endpoint *) endpoint;
+ lock_release(&ep->lock);
+ *ep_context = (void *) ep;
+ return 0;
+ }
+
+ acm_log(2, "creating endpoint for pkey 0x%x\n", endpoint->pkey);
+ ep = acmp_alloc_ep(port, (struct acm_endpoint *) endpoint);
+ if (!ep)
+ return -1;
+
+ sprintf(ep->id_string, "%s-%d-0x%x",
+ port->dev->verbs->device->name,
+ port->port_num, endpoint->pkey);
+
+ sq_size = resolve_depth + send_depth;
+ ep->cq = ibv_create_cq(port->dev->verbs, sq_size + recv_depth,
+ ep, port->dev->channel, 0);
+ if (!ep->cq) {
+ acm_log(0, "ERROR - failed to create CQ\n");
+ goto err0;
+ }
+
+ ret = ibv_req_notify_cq(ep->cq, 0);
+ if (ret) {
+ acm_log(0, "ERROR - failed to arm CQ\n");
+ goto err1;
+ }
+
+ memset(&init_attr, 0, sizeof init_attr);
+ init_attr.cap.max_send_wr = sq_size;
+ init_attr.cap.max_recv_wr = recv_depth;
+ init_attr.cap.max_send_sge = 1;
+ init_attr.cap.max_recv_sge = 1;
+ init_attr.qp_context = ep;
+ init_attr.sq_sig_all = 1;
+ init_attr.qp_type = IBV_QPT_UD;
+ init_attr.send_cq = ep->cq;
+ init_attr.recv_cq = ep->cq;
+ ep->qp = ibv_create_qp(ep->port->dev->pd, &init_attr);
+ if (!ep->qp) {
+ acm_log(0, "ERROR - failed to create QP\n");
+ goto err1;
+ }
+
+ attr.qp_state = IBV_QPS_INIT;
+ attr.port_num = port->port_num;
+ attr.pkey_index = acmp_get_pkey_index((struct acm_endpoint *) endpoint);
+ attr.qkey = ACM_QKEY;
+ ret = ibv_modify_qp(ep->qp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX |
+ IBV_QP_PORT | IBV_QP_QKEY);
+ if (ret) {
+ acm_log(0, "ERROR - failed to modify QP to init\n");
+ goto err2;
+ }
+
+ attr.qp_state = IBV_QPS_RTR;
+ ret = ibv_modify_qp(ep->qp, &attr, IBV_QP_STATE);
+ if (ret) {
+ acm_log(0, "ERROR - failed to modify QP to rtr\n");
+ goto err2;
+ }
+
+ attr.qp_state = IBV_QPS_RTS;
+ attr.sq_psn = 0;
+ ret = ibv_modify_qp(ep->qp, &attr, IBV_QP_STATE | IBV_QP_SQ_PSN);
+ if (ret) {
+ acm_log(0, "ERROR - failed to modify QP to rts\n");
+ goto err2;
+ }
+
+ ret = acmp_post_recvs(ep);
+ if (ret)
+ goto err2;
+
+ lock_acquire(&port->lock);
+ DListInsertHead(&ep->entry, &port->ep_list);
+ lock_release(&port->lock);
+ acmp_ep_preload(ep);
+ acmp_ep_join(ep);
+ *ep_context = (void *) ep;
+ return 0;
+
+err2:
+ ibv_destroy_qp(ep->qp);
+err1:
+ ibv_destroy_cq(ep->cq);
+err0:
+ free(ep);
+ return -1;
+}
+
+static void acmp_port_up(struct acmp_port *port)
+{
+ struct ibv_port_attr attr;
+ uint16_t pkey, sm_lid;
+ int i, ret;
+
+ acm_log(1, "%s %d\n", port->dev->verbs->device->name, port->port_num);
+ ret = ibv_query_port(port->dev->verbs, port->port_num, &attr);
+ if (ret) {
+ acm_log(0, "ERROR - unable to get port attribute\n");
+ return;
+ }
+
+ port->mtu = attr.active_mtu;
+ port->rate = acm_get_rate(attr.active_width, attr.active_speed);
+ if (attr.subnet_timeout >= 8)
+ port->subnet_timeout = 1 << (attr.subnet_timeout - 8);
+
+ port->lid = attr.lid;
+ port->lid_mask = 0xffff - ((1 << attr.lmc) - 1);
+
+ port->sa_dest.av.src_path_bits = 0;
+ port->sa_dest.av.dlid = attr.sm_lid;
+ port->sa_dest.av.sl = attr.sm_sl;
+ port->sa_dest.av.port_num = port->port_num;
+ port->sa_dest.remote_qpn = 1;
+ sm_lid = htons(attr.sm_lid);
+ acmp_set_dest_addr(&port->sa_dest, ACM_ADDRESS_LID,
+ (uint8_t *) &sm_lid, sizeof(sm_lid));
+
+ atomic_set(&port->sa_dest.refcnt, 1);
+ port->sa_dest.state = ACMP_READY;
+ for (i = 0; i < attr.pkey_tbl_len; i++) {
+ ret = ibv_query_pkey(port->dev->verbs, port->port_num, i, &pkey);
+ if (ret)
+ continue;
+ pkey = ntohs(pkey);
+ if (!(pkey & 0x7fff))
+ continue;
+
+ /* Determine pkey index for default partition with preference
+ * for full membership
+ */
+ if ((pkey & 0x7fff) == 0x7fff) {
+ port->default_pkey_ix = i;
+ break;
+ }
+ }
+
+ port->state = IBV_PORT_ACTIVE;
+ acm_log(1, "%s %d is up\n", port->dev->verbs->device->name, port->port_num);
+}
+
+static void acmp_port_down(struct acmp_port *port)
+{
+ acm_log(1, "%s %d\n", port->dev->verbs->device->name, port->port_num);
+ lock_acquire(&port->lock);
+ port->state = IBV_PORT_DOWN;
+ lock_release(&port->lock);
+
+ /*
+ * We wait for the SA destination to be released. We could use an
+ * event instead of a sleep loop, but it's not worth it given how
+ * infrequently we should be processing a port down event in practice.
+ */
+ atomic_dec(&port->sa_dest.refcnt);
+ while (atomic_get(&port->sa_dest.refcnt))
+ sleep(0);
+ lock_acquire(&port->sa_dest.lock);
+ port->sa_dest.state = ACMP_INIT;
+ lock_release(&port->sa_dest.lock);
+ acm_log(1, "%s %d is down\n", port->dev->verbs->device->name, port->port_num);
+}
+
+static int acmp_open_port(const struct acm_port *cport, void *dev_context,
+ void **port_context)
+{
+ struct acmp_device *dev = dev_context;
+ struct acmp_port *port;
+
+ if (cport->port_num < 1 || cport->port_num > dev->port_cnt) {
+ acm_log(0, "Error: port_num %d is out of range (max %d)\n",
+ cport->port_num, dev->port_cnt);
+ return -1;
+ }
+
+ port = &dev->port[cport->port_num - 1];
+ lock_acquire(&port->lock);
+ port->port = cport;
+ port->state = IBV_PORT_DOWN;
+ lock_release(&port->lock);
+ acmp_port_up(port);
+ *port_context = port;
+ return 0;
+}
+
+static void acmp_close_port(void *port_context)
+{
+ struct acmp_port *port = port_context;
+
+ acmp_port_down(port);
+ lock_acquire(&port->lock);
+ port->port = NULL;
+ lock_release(&port->lock);
+}
+
+static void acmp_init_port(struct acmp_port *port, struct acmp_device *dev,
+ uint8_t port_num)
+{
+ acm_log(1, "%s %d\n", dev->verbs->device->name, port_num);
+ port->dev = dev;
+ port->port_num = port_num;
+ lock_init(&port->lock);
+ DListInit(&port->ep_list);
+ acmp_init_dest(&port->sa_dest, ACM_ADDRESS_LID, NULL, 0);
+ port->state = IBV_PORT_DOWN;
+}
+
+static int acmp_open_dev(const struct acm_device *device, void **dev_context)
+{
+ struct acmp_device *dev;
+ size_t size;
+ struct ibv_device_attr attr;
+ int i, ret;
+ DLIST_ENTRY *dev_entry;
+ struct ibv_context *verbs;
+
+ acm_log(1, "dev_guid 0x%llx %s\n", device->dev_guid,
+ device->verbs->device->name);
+
+ for (dev_entry = acmp_dev_list.Next; dev_entry != &acmp_dev_list;
+ dev_entry = dev_entry->Next) {
+ dev = container_of(dev_entry, struct acmp_device, entry);
+
+ if (dev->guid == device->dev_guid) {
+ acm_log(2, "dev_guid 0x%llx already exits\n",
+ device->dev_guid);
+ *dev_context = dev;
+ dev->device = device;
+ return 0;
+ }
+ }
+
+ /* We need to release the core device structure when device close is
+ * called. But this provider does not support dynamic add/removal of
+ * devices/ports/endpoints. To avoid use-after-free issues, we open
+ * our own verbs context, rather than using the one in the core
+ * device structure.
+ */
+ verbs = ibv_open_device(device->verbs->device);
+ if (!verbs) {
+ acm_log(0, "ERROR - opening device %s\n",
+ device->verbs->device->name);
+ goto err;
+ }
+
+ ret = ibv_query_device(verbs, &attr);
+ if (ret) {
+ acm_log(0, "ERROR - ibv_query_device (%s) %d\n",
+ verbs->device->name, ret);
+ goto err;
+ }
+
+ size = sizeof(*dev) + sizeof(struct acmp_port) * attr.phys_port_cnt;
+ dev = (struct acmp_device *) calloc(1, size);
+ if (!dev)
+ goto err;
+
+ dev->verbs = verbs;
+ dev->device = device;
+ dev->port_cnt = attr.phys_port_cnt;
+
+ dev->pd = ibv_alloc_pd(dev->verbs);
+ if (!dev->pd) {
+ acm_log(0, "ERROR - unable to allocate PD\n");
+ goto err1;
+ }
+
+ dev->channel = ibv_create_comp_channel(dev->verbs);
+ if (!dev->channel) {
+ acm_log(0, "ERROR - unable to create comp channel\n");
+ goto err2;
+ }
+
+ for (i = 0; i < dev->port_cnt; i++) {
+ acmp_init_port(&dev->port[i], dev, i + 1);
+ }
+
+ if (pthread_create(&dev->comp_thread_id, NULL, acmp_comp_handler, dev)) {
+ acm_log(0, "Error -- failed to create the comp thread for dev %s",
+ dev->verbs->device->name);
+ goto err3;
+ }
+
+ lock_acquire(&acmp_dev_lock);
+ DListInsertHead(&dev->entry, &acmp_dev_list);
+ lock_release(&acmp_dev_lock);
+ dev->guid = device->dev_guid;
+ *dev_context = dev;
+
+ acm_log(1, "%s opened\n", dev->verbs->device->name);
+ return 0;
+
+err3:
+ ibv_destroy_comp_channel(dev->channel);
+err2:
+ ibv_dealloc_pd(dev->pd);
+err1:
+ free(dev);
+err:
+ return -1;
+}
+
+static void acmp_close_dev(void *dev_context)
+{
+ struct acmp_device *dev = dev_context;
+
+ acm_log(1, "dev_guid 0x%llx\n", dev->device->dev_guid);
+ dev->device = NULL;
+}
+
+static void acmp_set_options(void)
+{
+ FILE *f;
+ char s[120];
+ char opt[32], value[256];
+ char *opts_file = acm_get_opts_file();
+
+ if (!(f = fopen(opts_file, "r")))
+ return;
+
+ while (fgets(s, sizeof s, f)) {
+ if (s[0] == '#')
+ continue;
+
+ if (sscanf(s, "%32s%256s", opt, value) != 2)
+ continue;
+
+ if (!stricmp("addr_prot", opt))
+ addr_prot = acmp_convert_addr_prot(value);
+ else if (!stricmp("addr_timeout", opt))
+ addr_timeout = atoi(value);
+ else if (!stricmp("route_prot", opt))
+ route_prot = acmp_convert_route_prot(value);
+ else if (!strcmp("route_timeout", opt))
+ route_timeout = atoi(value);
+ else if (!stricmp("loopback_prot", opt))
+ loopback_prot = acmp_convert_loopback_prot(value);
+ else if (!stricmp("timeout", opt))
+ timeout = atoi(value);
+ else if (!stricmp("retries", opt))
+ retries = atoi(value);
+ else if (!stricmp("resolve_depth", opt))
+ resolve_depth = atoi(value);
+ else if (!stricmp("send_depth", opt))
+ send_depth = atoi(value);
+ else if (!stricmp("recv_depth", opt))
+ recv_depth = atoi(value);
+ else if (!stricmp("min_mtu", opt))
+ min_mtu = acm_convert_mtu(atoi(value));
+ else if (!stricmp("min_rate", opt))
+ min_rate = acm_convert_rate(atoi(value));
+ else if (!stricmp("route_preload", opt))
+ route_preload = acmp_convert_route_preload(value);
+ else if (!stricmp("route_data_file", opt))
+ strcpy(route_data_file, value);
+ else if (!stricmp("addr_preload", opt))
+ addr_preload = acmp_convert_addr_preload(value);
+ else if (!stricmp("addr_data_file", opt))
+ strcpy(addr_data_file, value);
+ }
+
+ fclose(f);
+}
+
+static void acmp_log_options(void)
+{
+ acm_log(0, "address resolution %d\n", addr_prot);
+ acm_log(0, "address timeout %d\n", addr_timeout);
+ acm_log(0, "route resolution %d\n", route_prot);
+ acm_log(0, "route timeout %d\n", route_timeout);
+ acm_log(0, "loopback resolution %d\n", loopback_prot);
+ acm_log(0, "timeout %d ms\n", timeout);
+ acm_log(0, "retries %d\n", retries);
+ acm_log(0, "resolve depth %d\n", resolve_depth);
+ acm_log(0, "send depth %d\n", send_depth);
+ acm_log(0, "receive depth %d\n", recv_depth);
+ acm_log(0, "minimum mtu %d\n", min_mtu);
+ acm_log(0, "minimum rate %d\n", min_rate);
+ acm_log(0, "route preload %d\n", route_preload);
+ acm_log(0, "route data file %s\n", route_data_file);
+ acm_log(0, "address preload %d\n", addr_preload);
+ acm_log(0, "address data file %s\n", addr_data_file);
+}
+
+static void __attribute__((constructor)) acmp_init(void)
+{
+ if (osd_init())
+ return;
+
+ acmp_set_options();
+
+ acmp_log_options();
+
+ atomic_init(&tid);
+ atomic_init(&wait_cnt);
+ DListInit(&acmp_dev_list);
+ lock_init(&acmp_dev_lock);
+ DListInit(&timeout_list);
+ event_init(&timeout_event);
+
+ umad_init();
+
+ acm_log(1, "starting timeout/retry thread\n");
+ if (pthread_create(&retry_thread_id, NULL, acmp_retry_handler, NULL)) {
+ acm_log(0, "Error: failed to create the retry thread");
+ retry_thread_started = 0;
+ return;
+ }
+
+ acmp_initialized = 1;
+}
+
+static void __attribute__((destructor)) acmp_exit(void)
+{
+ acm_log(1, "Unloading...\n");
+ if (retry_thread_started) {
+ if (pthread_cancel(retry_thread_id)) {
+ acm_log(0, "Error: failed to cancel the retry thread\n");
+ return;
+ }
+ if (pthread_join(retry_thread_id, NULL)) {
+ acm_log(0, "Error: failed to join the retry thread\n");
+ return;
+ }
+ retry_thread_started = 0;
+ }
+
+ umad_done();
+ acmp_initialized = 0;
+}
+
+int provider_query(struct acm_provider **provider, uint32_t *version)
+{
+ acm_log(1, "\n");
+
+ if (!acmp_initialized)
+ return -1;
+
+ if (provider)
+ *provider = &def_prov;
+ if (version)
+ *version = ACM_PROV_VERSION;
+
+ return 0;
+}
+
diff --git a/prov/acmp/src/libibacmp.map b/prov/acmp/src/libibacmp.map
new file mode 100644
index 0000000..cccd166
--- /dev/null
+++ b/prov/acmp/src/libibacmp.map
@@ -0,0 +1,5 @@
+ACMP_1.0 {
+ global:
+ provider_query;
+ local: *;
+};
diff --git a/src/acm.c b/src/acm.c
index a6b4cd2..7649725 100644
--- a/src/acm.c
+++ b/src/acm.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2009-2012 Intel Corporation. All rights reserved.
+ * Copyright (c) 2009-2014 Intel Corporation. All rights reserved.
* Copyright (c) 2013 Mellanox Technologies LTD. All rights reserved.
*
* This software is available to you under the OpenIB.org BSD license
@@ -41,10 +41,13 @@
#include <sys/stat.h>
#include <sys/time.h>
#include <fcntl.h>
+#include <dirent.h>
#include <infiniband/acm.h>
+#include <infiniband/acm_prov.h>
#include <infiniband/umad.h>
#include <infiniband/verbs.h>
#include <dlist.h>
+#include <dlfcn.h>
#include <search.h>
#include <net/if.h>
#include <sys/ioctl.h>
@@ -52,207 +55,146 @@
#include <netinet/in.h>
#include <linux/netlink.h>
#include <linux/rtnetlink.h>
+#include <poll.h>
#include "acm_mad.h"
#include "acm_util.h"
#define src_out data[0]
-
-#define IB_LID_MCAST_START 0xc000
+#define src_index data[1]
+#define dst_index data[2]
#define MAX_EP_ADDR 4
-#define MAX_EP_MC 2
-
-enum acm_state {
- ACM_INIT,
- ACM_QUERY_ADDR,
- ACM_ADDR_RESOLVED,
- ACM_QUERY_ROUTE,
- ACM_READY
-};
-
-enum acm_addr_prot {
- ACM_ADDR_PROT_ACM
-};
-
-enum acm_route_prot {
- ACM_ROUTE_PROT_ACM,
- ACM_ROUTE_PROT_SA
-};
-
-enum acm_loopback_prot {
- ACM_LOOPBACK_PROT_NONE,
- ACM_LOOPBACK_PROT_LOCAL
-};
+#define NL_MSG_BUF_SIZE 4096
+#define ACM_PROV_NAME_SIZE 64
-enum acm_route_preload {
- ACM_ROUTE_PRELOAD_NONE,
- ACM_ROUTE_PRELOAD_OSM_FULL_V1
+struct acmc_subnet {
+ DLIST_ENTRY entry;
+ uint64_t subnet_prefix;
};
-enum acm_addr_preload {
- ACM_ADDR_PRELOAD_NONE,
- ACM_ADDR_PRELOAD_HOSTS
+struct acmc_prov {
+ struct acm_provider *prov;
+ void *handle;
+ DLIST_ENTRY entry;
+ DLIST_ENTRY subnet_list;
};
-/*
- * Nested locking order: dest -> ep, dest -> port
- */
-struct acm_dest {
- uint8_t address[ACM_MAX_ADDRESS]; /* keep first */
- char name[ACM_MAX_ADDRESS];
- struct ibv_ah *ah;
- struct ibv_ah_attr av;
- struct ibv_path_record path;
- union ibv_gid mgid;
- uint64_t req_id;
- DLIST_ENTRY req_queue;
- uint32_t remote_qpn;
- lock_t lock;
- enum acm_state state;
- atomic_t refcnt;
- uint64_t addr_timeout;
- uint64_t route_timeout;
- uint8_t addr_type;
+struct acmc_prov_context {
+ DLIST_ENTRY entry;
+ atomic_t refcnt;
+ struct acm_provider *prov;
+ void *context;
};
-struct acm_port {
- struct acm_device *dev;
- DLIST_ENTRY ep_list;
+struct acmc_device;
+
+struct acmc_port {
+ struct acmc_device *dev;
+ struct acm_port port;
+ struct acm_provider *prov; /* limit to 1 provider per port for now */
+ void *prov_port_context;
+ int mad_portid;
+ int mad_agentid;
+ struct ib_mad_addr sa_addr;
+ DLIST_ENTRY sa_pending;
+ DLIST_ENTRY sa_wait;
+ int sa_credits;
lock_t lock;
- int mad_portid;
- int mad_agentid;
- struct acm_dest sa_dest;
+ DLIST_ENTRY ep_list;
enum ibv_port_state state;
- enum ibv_mtu mtu;
- enum ibv_rate rate;
- int subnet_timeout;
int gid_cnt;
- uint16_t default_pkey_ix;
+ union ibv_gid *gid_tbl;
uint16_t lid;
uint16_t lid_mask;
- uint8_t port_num;
+ int default_pkey_index;
};
-struct acm_device {
- struct ibv_context *verbs;
- struct ibv_comp_channel *channel;
- struct ibv_pd *pd;
- uint64_t guid;
+struct acmc_device {
+ struct acm_device device;
DLIST_ENTRY entry;
+ DLIST_ENTRY prov_dev_context_list;
int port_cnt;
- struct acm_port port[0];
-};
-
-/* Maintain separate virtual send queues to avoid deadlock */
-struct acm_send_queue {
- int credits;
- DLIST_ENTRY pending;
+ struct acmc_port port[0];
};
-struct acm_ep {
- struct acm_port *port;
- struct ibv_cq *cq;
- struct ibv_qp *qp;
- struct ibv_mr *mr;
- uint8_t *recv_bufs;
- DLIST_ENTRY entry;
- union acm_ep_info addr[MAX_EP_ADDR];
- char name[MAX_EP_ADDR][ACM_MAX_ADDRESS];
- uint8_t addr_type[MAX_EP_ADDR];
- void *dest_map[ACM_ADDRESS_RESERVED - 1];
- struct acm_dest mc_dest[MAX_EP_MC];
- int mc_cnt;
- uint16_t pkey_index;
- uint16_t pkey;
- lock_t lock;
- struct acm_send_queue resolve_queue;
- struct acm_send_queue sa_queue;
- struct acm_send_queue resp_queue;
- DLIST_ENTRY active_queue;
- DLIST_ENTRY wait_queue;
- enum acm_state state;
+struct acmc_addr {
+ struct acm_address addr;
+ void *prov_addr_context;
+ char string_buf[ACM_MAX_ADDRESS];
};
-struct acm_send_msg {
- DLIST_ENTRY entry;
- struct acm_ep *ep;
- struct acm_dest *dest;
- struct ibv_ah *ah;
- void *context;
- void (*resp_handler)(struct acm_send_msg *req,
- struct ibv_wc *wc, struct acm_mad *resp);
- struct acm_send_queue *req_queue;
- struct ibv_mr *mr;
- struct ibv_send_wr wr;
- struct ibv_sge sge;
- uint64_t expires;
- int tries;
- uint8_t data[ACM_SEND_SIZE];
+struct acmc_ep {
+ struct acmc_port *port;
+ struct acm_endpoint endpoint;
+ void *prov_ep_context;
+ struct acmc_addr addr_info[MAX_EP_ADDR];
+ DLIST_ENTRY entry;
};
-struct acm_client {
+struct acmc_client {
lock_t lock; /* acquire ep lock first */
SOCKET sock;
int index;
atomic_t refcnt;
};
-struct acm_request {
- struct acm_client *client;
- DLIST_ENTRY entry;
- struct acm_msg msg;
-};
-
union socket_addr {
struct sockaddr sa;
struct sockaddr_in sin;
struct sockaddr_in6 sin6;
};
-static DLIST_ENTRY dev_list;
+struct acmc_sa_req {
+ DLIST_ENTRY entry;
+ struct acmc_ep *ep;
+ void (*resp_handler)(struct acm_sa_mad *);
+ struct acm_sa_mad mad;
+};
-static atomic_t tid;
-static DLIST_ENTRY timeout_list;
-static event_t timeout_event;
-static atomic_t wait_cnt;
+static char def_prov_name[ACM_PROV_NAME_SIZE] = "ibacmp";
+static DLIST_ENTRY provider_list;
+static struct acmc_prov *def_provider = NULL;
+
+static DLIST_ENTRY dev_list;
static SOCKET listen_socket;
static SOCKET ip_mon_socket;
-static struct acm_client client_array[FD_SETSIZE - 1];
+static struct acmc_client client_array[FD_SETSIZE - 1];
static FILE *flog;
static lock_t log_lock;
PER_THREAD char log_data[ACM_MAX_ADDRESS];
static atomic_t counter[ACM_MAX_COUNTER];
+static struct acmc_device *
+acm_get_device_from_gid(union ibv_gid *sgid, uint8_t *port);
+static struct acmc_ep *acm_find_ep(struct acmc_port *port, uint16_t pkey);
+static int acm_ep_insert_addr(struct acmc_ep *ep, const char *name, uint8_t *addr,
+ size_t addr_len, uint8_t addr_type);
+static void acm_event_handler(struct acmc_device *dev);
+
+static struct sa_data {
+ int timeout;
+ int retries;
+ int depth;
+ pthread_t thread_id;
+ struct pollfd *fds;
+ struct acmc_port **ports;
+ int nfds;
+} sa = { 2000, 2, 1};
+
/*
* Service options - may be set through ibacm_opts.cfg file.
*/
-static char *acme = BINDIR "/ib_acme -A";
+static char *acme = IBACM_BIN_PATH "/ib_acme -A";
static char *opts_file = ACM_CONF_DIR "/" ACM_OPTS_FILE;
static char *addr_file = ACM_CONF_DIR "/" ACM_ADDR_FILE;
-static char route_data_file[128] = ACM_CONF_DIR "/ibacm_route.data";
-static char addr_data_file[128] = ACM_CONF_DIR "/ibacm_hosts.data";
static char log_file[128] = "/var/log/ibacm.log";
static int log_level = 0;
static char lock_file[128] = "/var/run/ibacm.pid";
-static enum acm_addr_prot addr_prot = ACM_ADDR_PROT_ACM;
-static int addr_timeout = 1440;
-static enum acm_route_prot route_prot = ACM_ROUTE_PROT_SA;
-static int route_timeout = -1;
-static enum acm_loopback_prot loopback_prot = ACM_LOOPBACK_PROT_LOCAL;
static short server_port = 6125;
-static int timeout = 2000;
-static int retries = 2;
-static int resolve_depth = 1;
-static int sa_depth = 1;
-static int send_depth = 1;
-static int recv_depth = 1024;
-static uint8_t min_mtu = IBV_MTU_2048;
-static uint8_t min_rate = IBV_RATE_10_GBPS;
-static enum acm_route_preload route_preload;
-static enum acm_addr_preload addr_preload;
static int support_ips_in_addr_cfg = 0;
+static char prov_lib_path[256] = IBACM_LIB_PATH;
void acm_write(int level, const char *format, ...)
{
@@ -272,9 +214,8 @@ void acm_write(int level, const char *format, ...)
va_end(args);
}
-static void
-acm_format_name(int level, char *name, size_t name_size,
- uint8_t addr_type, uint8_t *addr, size_t addr_size)
+void acm_format_name(int level, char *name, size_t name_size,
+ uint8_t addr_type, const uint8_t *addr, size_t addr_size)
{
struct ibv_path_record *path;
@@ -311,566 +252,149 @@ acm_format_name(int level, char *name, size_t name_size,
}
}
-static int ib_any_gid(union ibv_gid *gid)
+int ib_any_gid(union ibv_gid *gid)
{
return ((gid->global.subnet_prefix | gid->global.interface_id) == 0);
}
-static int acm_compare_dest(const void *dest1, const void *dest2)
-{
- return memcmp(dest1, dest2, ACM_MAX_ADDRESS);
-}
-
-static void
-acm_set_dest_addr(struct acm_dest *dest, uint8_t addr_type, uint8_t *addr, size_t size)
+char * acm_get_opts_file(void)
{
- memcpy(dest->address, addr, size);
- dest->addr_type = addr_type;
- acm_format_name(0, dest->name, sizeof dest->name, addr_type, addr, size);
+ return opts_file;
}
-static void
-acm_init_dest(struct acm_dest *dest, uint8_t addr_type, uint8_t *addr, size_t size)
+void acm_increment_counter(int type)
{
- DListInit(&dest->req_queue);
- atomic_init(&dest->refcnt);
- atomic_set(&dest->refcnt, 1);
- lock_init(&dest->lock);
- if (size)
- acm_set_dest_addr(dest, addr_type, addr, size);
+ if (type >= 0 && type < ACM_MAX_COUNTER)
+ atomic_inc(&counter[type]);
}
-static struct acm_dest *
-acm_alloc_dest(uint8_t addr_type, uint8_t *addr)
+static struct acmc_prov_context *
+acm_alloc_prov_context(struct acm_provider *prov)
{
- struct acm_dest *dest;
+ struct acmc_prov_context *ctx;
- dest = calloc(1, sizeof *dest);
- if (!dest) {
- acm_log(0, "ERROR - unable to allocate dest\n");
+ ctx = calloc(1, sizeof(*ctx));
+ if (!ctx) {
+ acm_log(0, "Error: failed to allocate prov context\n");
return NULL;
}
-
- acm_init_dest(dest, addr_type, addr, ACM_MAX_ADDRESS);
- acm_log(1, "%s\n", dest->name);
- return dest;
-}
-
-/* Caller must hold ep lock. */
-static struct acm_dest *
-acm_get_dest(struct acm_ep *ep, uint8_t addr_type, uint8_t *addr)
-{
- struct acm_dest *dest, **tdest;
-
- tdest = tfind(addr, &ep->dest_map[addr_type - 1], acm_compare_dest);
- if (tdest) {
- dest = *tdest;
- (void) atomic_inc(&dest->refcnt);
- acm_log(2, "%s\n", dest->name);
- } else {
- dest = NULL;
- acm_format_name(2, log_data, sizeof log_data,
- addr_type, addr, ACM_MAX_ADDRESS);
- acm_log(2, "%s not found\n", log_data);
- }
- return dest;
-}
-
-static void
-acm_put_dest(struct acm_dest *dest)
-{
- acm_log(2, "%s\n", dest->name);
- if (atomic_dec(&dest->refcnt) == 0) {
- free(dest);
- }
+ atomic_set(&ctx->refcnt, 1);
+ ctx->prov = prov;
+ return ctx;
}
-static struct acm_dest *
-acm_acquire_dest(struct acm_ep *ep, uint8_t addr_type, uint8_t *addr)
+static struct acmc_prov_context *
+acm_get_prov_context(DLIST_ENTRY *list, struct acm_provider *prov)
{
- struct acm_dest *dest;
+ DLIST_ENTRY *entry;
+ struct acmc_prov_context *ctx;
- acm_format_name(2, log_data, sizeof log_data,
- addr_type, addr, ACM_MAX_ADDRESS);
- acm_log(2, "%s\n", log_data);
- lock_acquire(&ep->lock);
- dest = acm_get_dest(ep, addr_type, addr);
- if (!dest) {
- dest = acm_alloc_dest(addr_type, addr);
- if (dest) {
- tsearch(dest, &ep->dest_map[addr_type - 1], acm_compare_dest);
- (void) atomic_inc(&dest->refcnt);
+ for (entry = list->Next; entry != list; entry = entry->Next) {
+ ctx = container_of(entry, struct acmc_prov_context, entry);
+ if (ctx->prov == prov) {
+ return ctx;
}
}
- lock_release(&ep->lock);
- return dest;
-}
-
-static struct acm_dest *
-acm_acquire_sa_dest(struct acm_port *port)
-{
- struct acm_dest *dest;
-
- lock_acquire(&port->lock);
- if (port->state == IBV_PORT_ACTIVE) {
- dest = &port->sa_dest;
- atomic_inc(&port->sa_dest.refcnt);
- } else {
- dest = NULL;
- }
- lock_release(&port->lock);
- return dest;
-}
-
-static void acm_release_sa_dest(struct acm_dest *dest)
-{
- atomic_dec(&dest->refcnt);
-}
-
-/* Caller must hold ep lock. */
-//static void
-//acm_remove_dest(struct acm_ep *ep, struct acm_dest *dest)
-//{
-// acm_log(2, "%s\n", dest->name);
-// tdelete(dest->address, &ep->dest_map[dest->addr_type - 1], acm_compare_dest);
-// acm_put_dest(dest);
-//}
-
-static struct acm_request *
-acm_alloc_req(struct acm_client *client, struct acm_msg *msg)
-{
- struct acm_request *req;
-
- req = calloc(1, sizeof *req);
- if (!req) {
- acm_log(0, "ERROR - unable to alloc client request\n");
- return NULL;
- }
- (void) atomic_inc(&client->refcnt);
- req->client = client;
- memcpy(&req->msg, msg, sizeof(req->msg));
- acm_log(2, "client %d, req %p\n", client->index, req);
- return req;
-}
-
-static void
-acm_free_req(struct acm_request *req)
-{
- acm_log(2, "%p\n", req);
- (void) atomic_dec(&req->client->refcnt);
- free(req);
-}
-
-static struct acm_send_msg *
-acm_alloc_send(struct acm_ep *ep, struct acm_dest *dest, size_t size)
-{
- struct acm_send_msg *msg;
-
- msg = (struct acm_send_msg *) calloc(1, sizeof *msg);
- if (!msg) {
- acm_log(0, "ERROR - unable to allocate send buffer\n");
- return NULL;
- }
-
- msg->ep = ep;
- msg->mr = ibv_reg_mr(ep->port->dev->pd, msg->data, size, 0);
- if (!msg->mr) {
- acm_log(0, "ERROR - failed to register send buffer\n");
- goto err1;
- }
-
- if (!dest->ah) {
- msg->ah = ibv_create_ah(ep->port->dev->pd, &dest->av);
- if (!msg->ah) {
- acm_log(0, "ERROR - unable to create ah\n");
- goto err2;
- }
- msg->wr.wr.ud.ah = msg->ah;
- } else {
- msg->wr.wr.ud.ah = dest->ah;
- }
-
- acm_log(2, "get dest %s\n", dest->name);
- (void) atomic_inc(&dest->refcnt);
- msg->dest = dest;
-
- msg->wr.next = NULL;
- msg->wr.sg_list = &msg->sge;
- msg->wr.num_sge = 1;
- msg->wr.opcode = IBV_WR_SEND;
- msg->wr.send_flags = IBV_SEND_SIGNALED;
- msg->wr.wr_id = (uintptr_t) msg;
- msg->wr.wr.ud.remote_qpn = dest->remote_qpn;
- msg->wr.wr.ud.remote_qkey = ACM_QKEY;
-
- msg->sge.length = size;
- msg->sge.lkey = msg->mr->lkey;
- msg->sge.addr = (uintptr_t) msg->data;
- acm_log(2, "%p\n", msg);
- return msg;
-
-err2:
- ibv_dereg_mr(msg->mr);
-err1:
- free(msg);
return NULL;
}
-static void
-acm_init_send_req(struct acm_send_msg *msg, void *context,
- void (*resp_handler)(struct acm_send_msg *req,
- struct ibv_wc *wc, struct acm_mad *resp))
-{
- acm_log(2, "%p\n", msg);
- msg->tries = retries + 1;
- msg->context = context;
- msg->resp_handler = resp_handler;
-}
-
-static void acm_free_send(struct acm_send_msg *msg)
-{
- acm_log(2, "%p\n", msg);
- if (msg->ah)
- ibv_destroy_ah(msg->ah);
- ibv_dereg_mr(msg->mr);
- acm_put_dest(msg->dest);
- free(msg);
-}
-
-static void acm_post_send(struct acm_send_queue *queue, struct acm_send_msg *msg)
-{
- struct acm_ep *ep = msg->ep;
- struct ibv_send_wr *bad_wr;
-
- msg->req_queue = queue;
- lock_acquire(&ep->lock);
- if (queue->credits) {
- acm_log(2, "posting send to QP\n");
- queue->credits--;
- DListInsertTail(&msg->entry, &ep->active_queue);
- ibv_post_send(ep->qp, &msg->wr, &bad_wr);
- } else {
- acm_log(2, "no sends available, queuing message\n");
- DListInsertTail(&msg->entry, &queue->pending);
- }
- lock_release(&ep->lock);
-}
-
-static void acm_post_recv(struct acm_ep *ep, uint64_t address)
-{
- struct ibv_recv_wr wr, *bad_wr;
- struct ibv_sge sge;
-
- wr.next = NULL;
- wr.sg_list = &sge;
- wr.num_sge = 1;
- wr.wr_id = address;
-
- sge.length = ACM_RECV_SIZE;
- sge.lkey = ep->mr->lkey;
- sge.addr = address;
-
- ibv_post_recv(ep->qp, &wr, &bad_wr);
-}
-
-/* Caller must hold ep lock */
-static void acm_send_available(struct acm_ep *ep, struct acm_send_queue *queue)
+static struct acmc_prov_context *
+acm_acquire_prov_context(DLIST_ENTRY *list, struct acm_provider *prov)
{
- struct acm_send_msg *msg;
- struct ibv_send_wr *bad_wr;
- DLIST_ENTRY *entry;
+ struct acmc_prov_context *ctx;
- if (DListEmpty(&queue->pending)) {
- queue->credits++;
+ ctx = acm_get_prov_context(list, prov);
+ if (!ctx) {
+ ctx = acm_alloc_prov_context(prov);
+ if (!ctx) {
+ acm_log(0, "Error -- failed to allocate provider context\n");
+ return NULL;
+ }
+ DListInsertTail(&ctx->entry, list);
} else {
- acm_log(2, "posting queued send message\n");
- entry = queue->pending.Next;
- DListRemove(entry);
- msg = container_of(entry, struct acm_send_msg, entry);
- DListInsertTail(&msg->entry, &ep->active_queue);
- ibv_post_send(ep->qp, &msg->wr, &bad_wr);
+ atomic_inc(&ctx->refcnt);
}
-}
-
-static void acm_complete_send(struct acm_send_msg *msg)
-{
- struct acm_ep *ep = msg->ep;
- lock_acquire(&ep->lock);
- DListRemove(&msg->entry);
- if (msg->tries) {
- acm_log(2, "waiting for response\n");
- msg->expires = time_stamp_ms() + ep->port->subnet_timeout + timeout;
- DListInsertTail(&msg->entry, &ep->wait_queue);
- if (atomic_inc(&wait_cnt) == 1)
- event_signal(&timeout_event);
- } else {
- acm_log(2, "freeing\n");
- acm_send_available(ep, msg->req_queue);
- acm_free_send(msg);
- }
- lock_release(&ep->lock);
+ return ctx;
}
-static struct acm_send_msg *acm_get_request(struct acm_ep *ep, uint64_t tid, int *free)
+static void
+acm_release_prov_context(struct acmc_prov_context *ctx)
{
- struct acm_send_msg *msg, *req = NULL;
- struct acm_mad *mad;
- DLIST_ENTRY *entry, *next;
-
- acm_log(2, "\n");
- lock_acquire(&ep->lock);
- for (entry = ep->wait_queue.Next; entry != &ep->wait_queue; entry = next) {
- next = entry->Next;
- msg = container_of(entry, struct acm_send_msg, entry);
- mad = (struct acm_mad *) msg->data;
- if (mad->tid == tid) {
- acm_log(2, "match found in wait queue\n");
- req = msg;
- DListRemove(entry);
- (void) atomic_dec(&wait_cnt);
- acm_send_available(ep, msg->req_queue);
- *free = 1;
- goto unlock;
- }
- }
-
- for (entry = ep->active_queue.Next; entry != &ep->active_queue; entry = entry->Next) {
- msg = container_of(entry, struct acm_send_msg, entry);
- mad = (struct acm_mad *) msg->data;
- if (mad->tid == tid && msg->tries) {
- acm_log(2, "match found in active queue\n");
- req = msg;
- req->tries = 0;
- *free = 0;
- break;
- }
+ if (atomic_dec(&ctx->refcnt) <= 0) {
+ DListRemove(&ctx->entry);
+ free(ctx);
}
-unlock:
- lock_release(&ep->lock);
- return req;
}
-static uint8_t acm_gid_index(struct acm_port *port, union ibv_gid *gid)
+uint8_t acm_gid_index(struct acm_port *port, union ibv_gid *gid)
{
- union ibv_gid cmp_gid;
uint8_t i;
+ struct acmc_port *cport;
- for (i = 0; i < port->gid_cnt; i++) {
- ibv_query_gid(port->dev->verbs, port->port_num, i, &cmp_gid);
- if (!memcmp(&cmp_gid, gid, sizeof cmp_gid))
+ cport = container_of(port, struct acmc_port, port);
+ for (i = 0; i < cport->gid_cnt; i++) {
+ if (!memcmp(&cport->gid_tbl[i], gid, sizeof (*gid)))
break;
}
return i;
}
-static int acm_mc_index(struct acm_ep *ep, union ibv_gid *gid)
-{
- int i;
-
- for (i = 0; i < ep->mc_cnt; i++) {
- if (!memcmp(&ep->mc_dest[i].address, gid, sizeof(*gid)))
- return i;
- }
- return -1;
-}
-
-/* Multicast groups are ordered lowest to highest preference. */
-static int acm_best_mc_index(struct acm_ep *ep, struct acm_resolve_rec *rec)
-{
- int i, index;
-
- for (i = min(rec->gid_cnt, ACM_MAX_GID_COUNT) - 1; i >= 0; i--) {
- index = acm_mc_index(ep, &rec->gid[i]);
- if (index >= 0) {
- return index;
- }
- }
- return -1;
-}
-
-static void
-acm_record_mc_av(struct acm_port *port, struct ib_mc_member_rec *mc_rec,
- struct acm_dest *dest)
-{
- uint32_t sl_flow_hop;
-
- sl_flow_hop = ntohl(mc_rec->sl_flow_hop);
-
- dest->av.dlid = ntohs(mc_rec->mlid);
- dest->av.sl = (uint8_t) (sl_flow_hop >> 28);
- dest->av.src_path_bits = port->sa_dest.av.src_path_bits;
- dest->av.static_rate = mc_rec->rate & 0x3F;
- dest->av.port_num = port->port_num;
-
- dest->av.is_global = 1;
- dest->av.grh.dgid = mc_rec->mgid;
- dest->av.grh.flow_label = (sl_flow_hop >> 8) & 0xFFFFF;
- dest->av.grh.sgid_index = acm_gid_index(port, &mc_rec->port_gid);
- dest->av.grh.hop_limit = (uint8_t) sl_flow_hop;
- dest->av.grh.traffic_class = mc_rec->tclass;
-
- dest->path.dgid = mc_rec->mgid;
- dest->path.sgid = mc_rec->port_gid;
- dest->path.dlid = mc_rec->mlid;
- dest->path.slid = htons(port->lid) | port->sa_dest.av.src_path_bits;
- dest->path.flowlabel_hoplimit = htonl(sl_flow_hop & 0xFFFFFFF);
- dest->path.tclass = mc_rec->tclass;
- dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE | 1;
- dest->path.pkey = mc_rec->pkey;
- dest->path.qosclass_sl = htons((uint16_t) (sl_flow_hop >> 28));
- dest->path.mtu = mc_rec->mtu;
- dest->path.rate = mc_rec->rate;
- dest->path.packetlifetime = mc_rec->packet_lifetime;
-}
-
-/* Always send the GRH to transfer GID data to remote side */
-static void
-acm_init_path_av(struct acm_port *port, struct acm_dest *dest)
-{
- uint32_t flow_hop;
-
- dest->av.dlid = ntohs(dest->path.dlid);
- dest->av.sl = ntohs(dest->path.qosclass_sl) & 0xF;
- dest->av.src_path_bits = dest->path.slid & 0x7F;
- dest->av.static_rate = dest->path.rate & 0x3F;
- dest->av.port_num = port->port_num;
-
- flow_hop = ntohl(dest->path.flowlabel_hoplimit);
- dest->av.is_global = 1;
- dest->av.grh.flow_label = (flow_hop >> 8) & 0xFFFFF;
- dest->av.grh.sgid_index = acm_gid_index(port, &dest->path.sgid);
- dest->av.grh.hop_limit = (uint8_t) flow_hop;
- dest->av.grh.traffic_class = dest->path.tclass;
-}
-
-static void acm_process_join_resp(struct acm_ep *ep, struct ib_user_mad *umad)
+int acm_get_gid(struct acm_port *port, int index, union ibv_gid *gid)
{
- struct acm_dest *dest;
- struct ib_mc_member_rec *mc_rec;
- struct ib_sa_mad *mad;
- int index, ret;
-
- mad = (struct ib_sa_mad *) umad->data;
- acm_log(1, "response status: 0x%x, mad status: 0x%x\n",
- umad->status, mad->status);
- if (umad->status) {
- acm_log(0, "ERROR - send join failed 0x%x\n", umad->status);
- return;
- }
- if (mad->status) {
- acm_log(0, "ERROR - join response status 0x%x\n", mad->status);
- return;
- }
-
- mc_rec = (struct ib_mc_member_rec *) mad->data;
- lock_acquire(&ep->lock);
- index = acm_mc_index(ep, &mc_rec->mgid);
- if (index < 0) {
- acm_log(0, "ERROR - MGID in join response not found\n");
- goto out;
- }
+ struct acmc_port *cport;
- dest = &ep->mc_dest[index];
- dest->remote_qpn = IB_MC_QPN;
- dest->mgid = mc_rec->mgid;
- acm_record_mc_av(ep->port, mc_rec, dest);
-
- if (index == 0) {
- dest->ah = ibv_create_ah(ep->port->dev->pd, &dest->av);
- if (!dest->ah) {
- acm_log(0, "ERROR - unable to create ah\n");
- goto out;
- }
- ret = ibv_attach_mcast(ep->qp, &mc_rec->mgid, mc_rec->mlid);
- if (ret) {
- acm_log(0, "ERROR - unable to attach QP to multicast group\n");
- goto out;
- }
+ cport = container_of(port, struct acmc_port, port);
+ if (index >= 0 && index < cport->gid_cnt) {
+ *gid = cport->gid_tbl[index];
+ return 0;
+ } else {
+ return -1;
}
-
- atomic_set(&dest->refcnt, 1);
- dest->state = ACM_READY;
- acm_log(1, "join successful\n");
-out:
- lock_release(&ep->lock);
}
-static void acm_mark_addr_invalid(struct acm_ep *ep,
+static void acm_mark_addr_invalid(struct acmc_ep *ep,
struct acm_ep_addr_data *data)
{
int i;
- lock_acquire(&ep->lock);
for (i = 0; i < MAX_EP_ADDR; i++) {
- if (ep->addr_type[i] != data->type)
+ if (ep->addr_info[i].addr.type != data->type)
continue;
if ((data->type == ACM_ADDRESS_NAME &&
- !strnicmp((char *) ep->addr[i].name,
+ !strnicmp((char *) ep->addr_info[i].addr.info.name,
(char *) data->info.addr, ACM_MAX_ADDRESS)) ||
- !memcmp(ep->addr[i].addr, data->info.addr, ACM_MAX_ADDRESS)) {
- ep->addr_type[i] = ACM_ADDRESS_INVALID;
+ !memcmp(ep->addr_info[i].addr.info.addr, data->info.addr,
+ ACM_MAX_ADDRESS)) {
+ ep->addr_info[i].addr.type = ACM_ADDRESS_INVALID;
break;
}
}
- lock_release(&ep->lock);
}
-static int acm_addr_index(struct acm_ep *ep, uint8_t *addr, uint8_t addr_type)
+static struct acm_address *
+acm_addr_lookup(const struct acm_endpoint *endpoint, uint8_t *addr, uint8_t addr_type)
{
+ struct acmc_ep *ep;
int i;
+ ep = container_of(endpoint, struct acmc_ep, endpoint);
for (i = 0; i < MAX_EP_ADDR; i++) {
- if (ep->addr_type[i] != addr_type)
+ if (ep->addr_info[i].addr.type != addr_type)
continue;
if ((addr_type == ACM_ADDRESS_NAME &&
- !strnicmp((char *) ep->addr[i].name,
+ !strnicmp((char *) ep->addr_info[i].addr.info.name,
(char *) addr, ACM_MAX_ADDRESS)) ||
- !memcmp(ep->addr[i].addr, addr, ACM_MAX_ADDRESS))
- return i;
- }
- return -1;
-}
-
-static uint8_t
-acm_record_acm_route(struct acm_ep *ep, struct acm_dest *dest)
-{
- int i;
-
- acm_log(2, "\n");
- for (i = 0; i < MAX_EP_MC; i++) {
- if (!memcmp(&dest->mgid, &ep->mc_dest[i].mgid, sizeof dest->mgid))
- break;
- }
- if (i == MAX_EP_MC) {
- acm_log(0, "ERROR - cannot match mgid\n");
- return ACM_STATUS_EINVAL;
+ !memcmp(ep->addr_info[i].addr.info.addr, addr, ACM_MAX_ADDRESS))
+ return &ep->addr_info[i].addr;
}
-
- dest->path = ep->mc_dest[i].path;
- dest->path.dgid = dest->av.grh.dgid;
- dest->path.dlid = htons(dest->av.dlid);
- dest->addr_timeout = time_stamp_min() + (unsigned) addr_timeout;
- dest->route_timeout = time_stamp_min() + (unsigned) route_timeout;
- dest->state = ACM_READY;
- return ACM_STATUS_SUCCESS;
-}
-
-static void acm_init_path_query(struct ib_sa_mad *mad)
-{
- acm_log(2, "\n");
- mad->base_version = 1;
- mad->mgmt_class = IB_MGMT_CLASS_SA;
- mad->class_version = 2;
- mad->method = IB_METHOD_GET;
- mad->tid = htonll((uint64_t) atomic_inc(&tid));
- mad->attr_id = IB_SA_ATTR_PATH_REC;
+ return NULL;
}
-static uint64_t acm_path_comp_mask(struct ibv_path_record *path)
+uint64_t acm_path_comp_mask(struct ibv_path_record *path)
{
uint32_t fl_hop;
uint16_t qos_sl;
@@ -923,158 +447,55 @@ static uint64_t acm_path_comp_mask(struct ibv_path_record *path)
return comp_mask;
}
-/* Caller must hold dest lock */
-static uint8_t acm_resolve_path(struct acm_ep *ep, struct acm_dest *dest,
- void (*resp_handler)(struct acm_send_msg *req,
- struct ibv_wc *wc, struct acm_mad *resp))
+int acm_resolve_response(uint64_t id, struct acm_msg *msg)
{
- struct acm_send_msg *msg;
- struct ib_sa_mad *mad;
- uint8_t ret;
-
- acm_log(2, "%s\n", dest->name);
- if (!acm_acquire_sa_dest(ep->port)) {
- acm_log(1, "cannot acquire SA destination\n");
- ret = ACM_STATUS_EINVAL;
- goto err;
- }
-
- msg = acm_alloc_send(ep, &ep->port->sa_dest, sizeof(*mad));
- acm_release_sa_dest(&ep->port->sa_dest);
- if (!msg) {
- acm_log(0, "ERROR - cannot allocate send msg\n");
- ret = ACM_STATUS_ENOMEM;
- goto err;
- }
-
- (void) atomic_inc(&dest->refcnt);
- acm_init_send_req(msg, (void *) dest, resp_handler);
- mad = (struct ib_sa_mad *) msg->data;
- acm_init_path_query(mad);
-
- memcpy(mad->data, &dest->path, sizeof(dest->path));
- mad->comp_mask = acm_path_comp_mask(&dest->path);
-
- atomic_inc(&counter[ACM_CNTR_ROUTE_QUERY]);
- dest->state = ACM_QUERY_ROUTE;
- acm_post_send(&ep->sa_queue, msg);
- return ACM_STATUS_SUCCESS;
-err:
- dest->state = ACM_INIT;
- return ret;
-}
-
-static uint8_t
-acm_record_acm_addr(struct acm_ep *ep, struct acm_dest *dest, struct ibv_wc *wc,
- struct acm_resolve_rec *rec)
-{
- int index;
-
- acm_log(2, "%s\n", dest->name);
- index = acm_best_mc_index(ep, rec);
- if (index < 0) {
- acm_log(0, "ERROR - no shared multicast groups\n");
- dest->state = ACM_INIT;
- return ACM_STATUS_ENODATA;
- }
-
- acm_log(2, "selecting MC group at index %d\n", index);
- dest->av = ep->mc_dest[index].av;
- dest->av.dlid = wc->slid;
- dest->av.src_path_bits = wc->dlid_path_bits;
- dest->av.grh.dgid = ((struct ibv_grh *) (uintptr_t) wc->wr_id)->sgid;
-
- dest->mgid = ep->mc_dest[index].mgid;
- dest->path.sgid = ep->mc_dest[index].path.sgid;
- dest->path.dgid = dest->av.grh.dgid;
- dest->path.tclass = ep->mc_dest[index].path.tclass;
- dest->path.pkey = ep->mc_dest[index].path.pkey;
- dest->remote_qpn = wc->src_qp;
-
- dest->state = ACM_ADDR_RESOLVED;
- return ACM_STATUS_SUCCESS;
-}
+ struct acmc_client *client = &client_array[id];
+ int ret;
-static void
-acm_record_path_addr(struct acm_ep *ep, struct acm_dest *dest,
- struct ibv_path_record *path)
-{
- acm_log(2, "%s\n", dest->name);
- dest->path.pkey = htons(ep->pkey);
- dest->path.dgid = path->dgid;
- if (path->slid || !ib_any_gid(&path->sgid)) {
- dest->path.sgid = path->sgid;
- dest->path.slid = path->slid;
- } else {
- dest->path.slid = htons(ep->port->lid);
- }
- dest->path.dlid = path->dlid;
- dest->state = ACM_ADDR_RESOLVED;
-}
+ acm_log(2, "client %d, status 0x%x\n", client->index, msg->hdr.status);
-static uint8_t acm_validate_addr_req(struct acm_mad *mad)
-{
- struct acm_resolve_rec *rec;
+ if (msg->hdr.status == ACM_STATUS_ENODATA)
+ atomic_inc(&counter[ACM_CNTR_NODATA]);
+ else if (msg->hdr.status)
+ atomic_inc(&counter[ACM_CNTR_ERROR]);
- if (mad->method != IB_METHOD_GET) {
- acm_log(0, "ERROR - invalid method 0x%x\n", mad->method);
- return ACM_STATUS_EINVAL;
+ lock_acquire(&client->lock);
+ if (client->sock == INVALID_SOCKET) {
+ acm_log(0, "ERROR - connection lost\n");
+ ret = ACM_STATUS_ENOTCONN;
+ goto release;
}
- rec = (struct acm_resolve_rec *) mad->data;
- if (!rec->src_type || rec->src_type >= ACM_ADDRESS_RESERVED) {
- acm_log(0, "ERROR - unknown src type 0x%x\n", rec->src_type);
- return ACM_STATUS_EINVAL;
- }
+ ret = send(client->sock, (char *) msg, msg->hdr.length, 0);
+ if (ret != msg->hdr.length)
+ acm_log(0, "ERROR - failed to send response\n");
+ else
+ ret = 0;
- return ACM_STATUS_SUCCESS;
+release:
+ lock_release(&client->lock);
+ (void) atomic_dec(&client->refcnt);
+ return ret;
}
-static void
-acm_send_addr_resp(struct acm_ep *ep, struct acm_dest *dest)
+static int
+acmc_resolve_response(uint64_t id, struct acm_msg *req_msg, uint8_t status)
{
- struct acm_resolve_rec *rec;
- struct acm_send_msg *msg;
- struct acm_mad *mad;
-
- acm_log(2, "%s\n", dest->name);
- msg = acm_alloc_send(ep, dest, sizeof (*mad));
- if (!msg) {
- acm_log(0, "ERROR - failed to allocate message\n");
- return;
- }
-
- mad = (struct acm_mad *) msg->data;
- rec = (struct acm_resolve_rec *) mad->data;
-
- mad->base_version = 1;
- mad->mgmt_class = ACM_MGMT_CLASS;
- mad->class_version = 1;
- mad->method = IB_METHOD_GET | IB_METHOD_RESP;
- mad->status = ACM_STATUS_SUCCESS;
- mad->control = ACM_CTRL_RESOLVE;
- mad->tid = dest->req_id;
- rec->gid_cnt = 1;
- memcpy(rec->gid, dest->mgid.raw, sizeof(union ibv_gid));
+ req_msg->hdr.opcode |= ACM_OP_ACK;
+ req_msg->hdr.status = status;
+ if (status != ACM_STATUS_SUCCESS)
+ req_msg->hdr.length = ACM_MSG_HDR_LENGTH;
+ memset(req_msg->hdr.data, 0, sizeof(req_msg->hdr.data));
- acm_post_send(&ep->resp_queue, msg);
+ return acm_resolve_response(id, req_msg);
}
-static int
-acm_client_resolve_resp(struct acm_client *client, struct acm_msg *req_msg,
- struct acm_dest *dest, uint8_t status)
+int acm_query_response(uint64_t id, struct acm_msg *msg)
{
- struct acm_msg msg;
+ struct acmc_client *client = &client_array[id];
int ret;
- acm_log(2, "client %d, status 0x%x\n", client->index, status);
- memset(&msg, 0, sizeof msg);
-
- if (status == ACM_STATUS_ENODATA)
- atomic_inc(&counter[ACM_CNTR_NODATA]);
- else if (status)
- atomic_inc(&counter[ACM_CNTR_ERROR]);
-
+ acm_log(2, "status 0x%x\n", msg->hdr.status);
lock_acquire(&client->lock);
if (client->sock == INVALID_SOCKET) {
acm_log(0, "ERROR - connection lost\n");
@@ -1082,641 +503,30 @@ acm_client_resolve_resp(struct acm_client *client, struct acm_msg *req_msg,
goto release;
}
- msg.hdr = req_msg->hdr;
- msg.hdr.opcode |= ACM_OP_ACK;
- msg.hdr.status = status;
- msg.hdr.length = ACM_MSG_HDR_LENGTH;
- memset(msg.hdr.data, 0, sizeof(msg.hdr.data));
-
- if (status == ACM_STATUS_SUCCESS) {
- msg.hdr.length += ACM_MSG_EP_LENGTH;
- msg.resolve_data[0].flags = IBV_PATH_FLAG_GMP |
- IBV_PATH_FLAG_PRIMARY | IBV_PATH_FLAG_BIDIRECTIONAL;
- msg.resolve_data[0].type = ACM_EP_INFO_PATH;
- msg.resolve_data[0].info.path = dest->path;
-
- if (req_msg->hdr.src_out) {
- msg.hdr.length += ACM_MSG_EP_LENGTH;
- memcpy(&msg.resolve_data[1],
- &req_msg->resolve_data[req_msg->hdr.src_out],
- ACM_MSG_EP_LENGTH);
- }
- }
-
- ret = send(client->sock, (char *) &msg, msg.hdr.length, 0);
- if (ret != msg.hdr.length)
+ ret = send(client->sock, (char *) msg, msg->hdr.length, 0);
+ if (ret != msg->hdr.length)
acm_log(0, "ERROR - failed to send response\n");
else
ret = 0;
release:
lock_release(&client->lock);
+ (void) atomic_dec(&client->refcnt);
return ret;
}
-static void
-acm_complete_queued_req(struct acm_dest *dest, uint8_t status)
+static int acmc_query_response(uint64_t id, struct acm_msg *msg, uint8_t status)
{
- struct acm_request *req;
- DLIST_ENTRY *entry;
-
- acm_log(2, "status %d\n", status);
- lock_acquire(&dest->lock);
- while (!DListEmpty(&dest->req_queue)) {
- entry = dest->req_queue.Next;
- DListRemove(entry);
- req = container_of(entry, struct acm_request, entry);
- lock_release(&dest->lock);
-
- acm_log(2, "completing request, client %d\n", req->client->index);
- acm_client_resolve_resp(req->client, &req->msg, dest, status);
- acm_free_req(req);
-
- lock_acquire(&dest->lock);
- }
- lock_release(&dest->lock);
+ acm_log(2, "status 0x%x\n", status);
+ msg->hdr.opcode |= ACM_OP_ACK;
+ msg->hdr.status = status;
+ return acm_query_response(id, msg);
}
-static void
-acm_dest_sa_resp(struct acm_send_msg *msg, struct ibv_wc *wc, struct acm_mad *mad)
+static void acm_init_server(void)
{
- struct acm_dest *dest = (struct acm_dest *) msg->context;
- struct ib_sa_mad *sa_mad = (struct ib_sa_mad *) mad;
- uint8_t status;
-
- if (mad) {
- status = (uint8_t) (ntohs(mad->status) >> 8);
- } else {
- status = ACM_STATUS_ETIMEDOUT;
- }
- acm_log(2, "%s status=0x%x\n", dest->name, status);
-
- lock_acquire(&dest->lock);
- if (dest->state != ACM_QUERY_ROUTE) {
- acm_log(1, "notice - discarding SA response\n");
- lock_release(&dest->lock);
- return;
- }
-
- if (!status) {
- memcpy(&dest->path, sa_mad->data, sizeof(dest->path));
- acm_init_path_av(msg->ep->port, dest);
- dest->addr_timeout = time_stamp_min() + (unsigned) addr_timeout;
- dest->route_timeout = time_stamp_min() + (unsigned) route_timeout;
- acm_log(2, "timeout addr %llu route %llu\n", dest->addr_timeout, dest->route_timeout);
- dest->state = ACM_READY;
- } else {
- dest->state = ACM_INIT;
- }
- lock_release(&dest->lock);
-
- acm_complete_queued_req(dest, status);
-}
-
-static void
-acm_resolve_sa_resp(struct acm_send_msg *msg, struct ibv_wc *wc, struct acm_mad *mad)
-{
- struct acm_dest *dest = (struct acm_dest *) msg->context;
- int send_resp;
-
- acm_log(2, "\n");
- acm_dest_sa_resp(msg, wc, mad);
-
- lock_acquire(&dest->lock);
- send_resp = (dest->state == ACM_READY);
- lock_release(&dest->lock);
-
- if (send_resp)
- acm_send_addr_resp(msg->ep, dest);
-}
-
-static void
-acm_process_addr_req(struct acm_ep *ep, struct ibv_wc *wc, struct acm_mad *mad)
-{
- struct acm_resolve_rec *rec;
- struct acm_dest *dest;
- uint8_t status;
- int addr_index;
-
- acm_log(2, "\n");
- if ((status = acm_validate_addr_req(mad))) {
- acm_log(0, "ERROR - invalid request\n");
- return;
- }
-
- rec = (struct acm_resolve_rec *) mad->data;
- dest = acm_acquire_dest(ep, rec->src_type, rec->src);
- if (!dest) {
- acm_log(0, "ERROR - unable to add source\n");
- return;
- }
-
- addr_index = acm_addr_index(ep, rec->dest, rec->dest_type);
- if (addr_index >= 0)
- dest->req_id = mad->tid;
-
- lock_acquire(&dest->lock);
- acm_log(2, "dest state %d\n", dest->state);
- switch (dest->state) {
- case ACM_READY:
- if (dest->remote_qpn == wc->src_qp)
- break;
-
- acm_log(2, "src service has new qp, resetting\n");
- /* fall through */
- case ACM_INIT:
- case ACM_QUERY_ADDR:
- status = acm_record_acm_addr(ep, dest, wc, rec);
- if (status)
- break;
- /* fall through */
- case ACM_ADDR_RESOLVED:
- if (route_prot == ACM_ROUTE_PROT_ACM) {
- status = acm_record_acm_route(ep, dest);
- break;
- }
- if (addr_index >= 0 || !DListEmpty(&dest->req_queue)) {
- status = acm_resolve_path(ep, dest, acm_resolve_sa_resp);
- if (status)
- break;
- }
- /* fall through */
- default:
- lock_release(&dest->lock);
- acm_put_dest(dest);
- return;
- }
- lock_release(&dest->lock);
- acm_complete_queued_req(dest, status);
-
- if (addr_index >= 0 && !status) {
- acm_send_addr_resp(ep, dest);
- }
- acm_put_dest(dest);
-}
-
-static void
-acm_process_addr_resp(struct acm_send_msg *msg, struct ibv_wc *wc, struct acm_mad *mad)
-{
- struct acm_resolve_rec *resp_rec;
- struct acm_dest *dest = (struct acm_dest *) msg->context;
- uint8_t status;
-
- if (mad) {
- status = acm_class_status(mad->status);
- resp_rec = (struct acm_resolve_rec *) mad->data;
- } else {
- status = ACM_STATUS_ETIMEDOUT;
- resp_rec = NULL;
- }
- acm_log(2, "resp status 0x%x\n", status);
-
- lock_acquire(&dest->lock);
- if (dest->state != ACM_QUERY_ADDR) {
- lock_release(&dest->lock);
- goto put;
- }
-
- if (!status) {
- status = acm_record_acm_addr(msg->ep, dest, wc, resp_rec);
- if (!status) {
- if (route_prot == ACM_ROUTE_PROT_ACM) {
- status = acm_record_acm_route(msg->ep, dest);
- } else {
- status = acm_resolve_path(msg->ep, dest, acm_dest_sa_resp);
- if (!status) {
- lock_release(&dest->lock);
- goto put;
- }
- }
- }
- } else {
- dest->state = ACM_INIT;
- }
- lock_release(&dest->lock);
-
- acm_complete_queued_req(dest, status);
-put:
- acm_put_dest(dest);
-}
-
-static void acm_process_acm_recv(struct acm_ep *ep, struct ibv_wc *wc, struct acm_mad *mad)
-{
- struct acm_send_msg *req;
- struct acm_resolve_rec *rec;
- int free;
-
- acm_log(2, "\n");
- if (mad->base_version != 1 || mad->class_version != 1) {
- acm_log(0, "ERROR - invalid version %d %d\n",
- mad->base_version, mad->class_version);
- return;
- }
-
- if (mad->control != ACM_CTRL_RESOLVE) {
- acm_log(0, "ERROR - invalid control 0x%x\n", mad->control);
- return;
- }
-
- rec = (struct acm_resolve_rec *) mad->data;
- acm_format_name(2, log_data, sizeof log_data,
- rec->src_type, rec->src, sizeof rec->src);
- acm_log(2, "src %s\n", log_data);
- acm_format_name(2, log_data, sizeof log_data,
- rec->dest_type, rec->dest, sizeof rec->dest);
- acm_log(2, "dest %s\n", log_data);
- if (mad->method & IB_METHOD_RESP) {
- acm_log(2, "received response\n");
- req = acm_get_request(ep, mad->tid, &free);
- if (!req) {
- acm_log(1, "notice - response did not match active request\n");
- return;
- }
- acm_log(2, "found matching request\n");
- req->resp_handler(req, wc, mad);
- if (free)
- acm_free_send(req);
- } else {
- acm_log(2, "unsolicited request\n");
- acm_process_addr_req(ep, wc, mad);
- }
-}
-
-static int
-acm_client_query_resp(struct acm_client *client,
- struct acm_msg *msg, uint8_t status)
-{
- int ret;
-
- acm_log(2, "status 0x%x\n", status);
- lock_acquire(&client->lock);
- if (client->sock == INVALID_SOCKET) {
- acm_log(0, "ERROR - connection lost\n");
- ret = ACM_STATUS_ENOTCONN;
- goto release;
- }
-
- msg->hdr.opcode |= ACM_OP_ACK;
- msg->hdr.status = status;
-
- ret = send(client->sock, (char *) msg, msg->hdr.length, 0);
- if (ret != msg->hdr.length)
- acm_log(0, "ERROR - failed to send response\n");
- else
- ret = 0;
-
-release:
- lock_release(&client->lock);
- return ret;
-}
-
-static void
-acm_client_sa_resp(struct acm_send_msg *msg, struct ibv_wc *wc, struct acm_mad *mad)
-{
- struct acm_request *req = (struct acm_request *) msg->context;
- struct ib_sa_mad *sa_mad = (struct ib_sa_mad *) mad;
- uint8_t status;
-
- if (mad) {
- status = (uint8_t) (ntohs(sa_mad->status) >> 8);
- memcpy(&req->msg.resolve_data[0].info.path, sa_mad->data,
- sizeof(struct ibv_path_record));
- } else {
- status = ACM_STATUS_ETIMEDOUT;
- }
- acm_log(2, "status 0x%x\n", status);
-
- acm_client_query_resp(req->client, &req->msg, status);
- acm_free_req(req);
-}
-
-static void acm_process_sa_recv(struct acm_ep *ep, struct ibv_wc *wc, struct acm_mad *mad)
-{
- struct ib_sa_mad *sa_mad = (struct ib_sa_mad *) mad;
- struct acm_send_msg *req;
- int free;
-
- acm_log(2, "\n");
- if (mad->base_version != 1 || mad->class_version != 2 ||
- !(mad->method & IB_METHOD_RESP) || sa_mad->attr_id != IB_SA_ATTR_PATH_REC) {
- acm_log(0, "ERROR - unexpected SA MAD %d %d\n",
- mad->base_version, mad->class_version);
- return;
- }
-
- req = acm_get_request(ep, mad->tid, &free);
- if (!req) {
- acm_log(1, "notice - response did not match active request\n");
- return;
- }
- acm_log(2, "found matching request\n");
- req->resp_handler(req, wc, mad);
- if (free)
- acm_free_send(req);
-}
-
-static void acm_process_recv(struct acm_ep *ep, struct ibv_wc *wc)
-{
- struct acm_mad *mad;
-
- acm_log(2, "base endpoint name %s\n", ep->name[0]);
- mad = (struct acm_mad *) (uintptr_t) (wc->wr_id + sizeof(struct ibv_grh));
- switch (mad->mgmt_class) {
- case IB_MGMT_CLASS_SA:
- acm_process_sa_recv(ep, wc, mad);
- break;
- case ACM_MGMT_CLASS:
- acm_process_acm_recv(ep, wc, mad);
- break;
- default:
- acm_log(0, "ERROR - invalid mgmt class 0x%x\n", mad->mgmt_class);
- break;
- }
-
- acm_post_recv(ep, wc->wr_id);
-}
-
-static void acm_process_comp(struct acm_ep *ep, struct ibv_wc *wc)
-{
- if (wc->status) {
- acm_log(0, "ERROR - work completion error\n"
- "\topcode %d, completion status %d\n",
- wc->opcode, wc->status);
- return;
- }
-
- if (wc->opcode & IBV_WC_RECV)
- acm_process_recv(ep, wc);
- else
- acm_complete_send((struct acm_send_msg *) (uintptr_t) wc->wr_id);
-}
-
-static void CDECL_FUNC acm_comp_handler(void *context)
-{
- struct acm_device *dev = (struct acm_device *) context;
- struct acm_ep *ep;
- struct ibv_cq *cq;
- struct ibv_wc wc;
- int cnt;
-
- acm_log(1, "started\n");
- while (1) {
- ibv_get_cq_event(dev->channel, &cq, (void *) &ep);
-
- cnt = 0;
- while (ibv_poll_cq(cq, 1, &wc) > 0) {
- cnt++;
- acm_process_comp(ep, &wc);
- }
-
- ibv_req_notify_cq(cq, 0);
- while (ibv_poll_cq(cq, 1, &wc) > 0) {
- cnt++;
- acm_process_comp(ep, &wc);
- }
-
- ibv_ack_cq_events(cq, cnt);
- }
-}
-
-static void acm_format_mgid(union ibv_gid *mgid, uint16_t pkey, uint8_t tos,
- uint8_t rate, uint8_t mtu)
-{
- mgid->raw[0] = 0xFF;
- mgid->raw[1] = 0x10 | 0x05;
- mgid->raw[2] = 0x40;
- mgid->raw[3] = 0x01;
- mgid->raw[4] = (uint8_t) (pkey >> 8);
- mgid->raw[5] = (uint8_t) pkey;
- mgid->raw[6] = tos;
- mgid->raw[7] = rate;
- mgid->raw[8] = mtu;
- mgid->raw[9] = 0;
- mgid->raw[10] = 0;
- mgid->raw[11] = 0;
- mgid->raw[12] = 0;
- mgid->raw[13] = 0;
- mgid->raw[14] = 0;
- mgid->raw[15] = 0;
-}
-
-static void acm_init_join(struct ib_sa_mad *mad, union ibv_gid *port_gid,
- uint16_t pkey, uint8_t tos, uint8_t tclass, uint8_t sl, uint8_t rate, uint8_t mtu)
-{
- struct ib_mc_member_rec *mc_rec;
-
- acm_log(2, "\n");
- mad->base_version = 1;
- mad->mgmt_class = IB_MGMT_CLASS_SA;
- mad->class_version = 2;
- mad->method = IB_METHOD_SET;
- mad->tid = htonll((uint64_t) atomic_inc(&tid));
- mad->attr_id = IB_SA_ATTR_MC_MEMBER_REC;
- mad->comp_mask =
- IB_COMP_MASK_MC_MGID | IB_COMP_MASK_MC_PORT_GID |
- IB_COMP_MASK_MC_QKEY | IB_COMP_MASK_MC_MTU_SEL| IB_COMP_MASK_MC_MTU |
- IB_COMP_MASK_MC_TCLASS | IB_COMP_MASK_MC_PKEY | IB_COMP_MASK_MC_RATE_SEL |
- IB_COMP_MASK_MC_RATE | IB_COMP_MASK_MC_SL | IB_COMP_MASK_MC_FLOW |
- IB_COMP_MASK_MC_SCOPE | IB_COMP_MASK_MC_JOIN_STATE;
-
- mc_rec = (struct ib_mc_member_rec *) mad->data;
- acm_format_mgid(&mc_rec->mgid, pkey | 0x8000, tos, rate, mtu);
- mc_rec->port_gid = *port_gid;
- mc_rec->qkey = htonl(ACM_QKEY);
- mc_rec->mtu = 0x80 | mtu;
- mc_rec->tclass = tclass;
- mc_rec->pkey = htons(pkey);
- mc_rec->rate = 0x80 | rate;
- mc_rec->sl_flow_hop = htonl(((uint32_t) sl) << 28);
- mc_rec->scope_state = 0x51;
-}
-
-static void acm_join_group(struct acm_ep *ep, union ibv_gid *port_gid,
- uint8_t tos, uint8_t tclass, uint8_t sl, uint8_t rate, uint8_t mtu)
-{
- struct acm_port *port;
- struct ib_sa_mad *mad;
- struct ib_user_mad *umad;
- struct ib_mc_member_rec *mc_rec;
- int ret, len;
-
- acm_log(2, "\n");
- len = sizeof(*umad) + sizeof(*mad);
- umad = (struct ib_user_mad *) calloc(1, len);
- if (!umad) {
- acm_log(0, "ERROR - unable to allocate MAD for join\n");
- return;
- }
-
- port = ep->port;
- umad->addr.qpn = htonl(port->sa_dest.remote_qpn);
- umad->addr.pkey_index = port->default_pkey_ix;
- umad->addr.lid = htons(port->sa_dest.av.dlid);
- umad->addr.sl = port->sa_dest.av.sl;
- umad->addr.path_bits = port->sa_dest.av.src_path_bits;
-
- acm_log(0, "%s %d pkey 0x%x, sl 0x%x, rate 0x%x, mtu 0x%x\n",
- ep->port->dev->verbs->device->name, ep->port->port_num,
- ep->pkey, sl, rate, mtu);
- mad = (struct ib_sa_mad *) umad->data;
- acm_init_join(mad, port_gid, ep->pkey, tos, tclass, sl, rate, mtu);
- mc_rec = (struct ib_mc_member_rec *) mad->data;
- acm_set_dest_addr(&ep->mc_dest[ep->mc_cnt++], ACM_ADDRESS_GID,
- mc_rec->mgid.raw, sizeof(mc_rec->mgid));
-
- ret = umad_send(port->mad_portid, port->mad_agentid, (void *) umad,
- sizeof(*mad), timeout, retries);
- if (ret) {
- acm_log(0, "ERROR - failed to send multicast join request %d\n", ret);
- goto out;
- }
-
- acm_log(1, "waiting for response from SA to join request\n");
- ret = umad_recv(port->mad_portid, (void *) umad, &len, -1);
- if (ret < 0) {
- acm_log(0, "ERROR - recv error for multicast join response %d\n", ret);
- goto out;
- }
-
- acm_process_join_resp(ep, umad);
-out:
- free(umad);
-}
-
-static void acm_port_join(struct acm_port *port)
-{
- struct acm_device *dev;
- struct acm_ep *ep;
- union ibv_gid port_gid;
- DLIST_ENTRY *ep_entry;
- int ret;
-
- dev = port->dev;
- acm_log(1, "device %s port %d\n", dev->verbs->device->name,
- port->port_num);
-
- ret = ibv_query_gid(dev->verbs, port->port_num, 0, &port_gid);
- if (ret) {
- acm_log(0, "ERROR - ibv_query_gid %d device %s port %d\n",
- ret, dev->verbs->device->name, port->port_num);
- return;
- }
-
- for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list;
- ep_entry = ep_entry->Next) {
-
- ep = container_of(ep_entry, struct acm_ep, entry);
- ep->mc_cnt = 0;
- acm_join_group(ep, &port_gid, 0, 0, 0, min_rate, min_mtu);
-
- if ((ep->state = ep->mc_dest[0].state) != ACM_READY)
- continue;
-
- if ((route_prot == ACM_ROUTE_PROT_ACM) &&
- (port->rate != min_rate || port->mtu != min_mtu))
- acm_join_group(ep, &port_gid, 0, 0, 0, port->rate, port->mtu);
- }
- acm_log(1, "joins for device %s port %d complete\n", dev->verbs->device->name,
- port->port_num);
-}
-
-static void acm_process_timeouts(void)
-{
- DLIST_ENTRY *entry;
- struct acm_send_msg *msg;
- struct acm_resolve_rec *rec;
- struct acm_mad *mad;
-
- while (!DListEmpty(&timeout_list)) {
- entry = timeout_list.Next;
- DListRemove(entry);
-
- msg = container_of(entry, struct acm_send_msg, entry);
- mad = (struct acm_mad *) &msg->data[0];
- rec = (struct acm_resolve_rec *) mad->data;
-
- acm_format_name(0, log_data, sizeof log_data,
- rec->dest_type, rec->dest, sizeof rec->dest);
- acm_log(0, "notice - dest %s\n", log_data);
- msg->resp_handler(msg, NULL, NULL);
- }
-}
-
-static void acm_process_wait_queue(struct acm_ep *ep, uint64_t *next_expire)
-{
- struct acm_send_msg *msg;
- DLIST_ENTRY *entry, *next;
- struct ibv_send_wr *bad_wr;
-
- for (entry = ep->wait_queue.Next; entry != &ep->wait_queue; entry = next) {
- next = entry->Next;
- msg = container_of(entry, struct acm_send_msg, entry);
- if (msg->expires < time_stamp_ms()) {
- DListRemove(entry);
- (void) atomic_dec(&wait_cnt);
- if (--msg->tries) {
- acm_log(1, "notice - retrying request\n");
- DListInsertTail(&msg->entry, &ep->active_queue);
- ibv_post_send(ep->qp, &msg->wr, &bad_wr);
- } else {
- acm_log(0, "notice - failing request\n");
- acm_send_available(ep, msg->req_queue);
- DListInsertTail(&msg->entry, &timeout_list);
- }
- } else {
- *next_expire = min(*next_expire, msg->expires);
- break;
- }
- }
-}
-
-static void CDECL_FUNC acm_retry_handler(void *context)
-{
- struct acm_device *dev;
- struct acm_port *port;
- struct acm_ep *ep;
- DLIST_ENTRY *dev_entry, *ep_entry;
- uint64_t next_expire;
- int i, wait;
-
- acm_log(0, "started\n");
- while (1) {
- while (!atomic_get(&wait_cnt))
- event_wait(&timeout_event, -1);
-
- next_expire = -1;
- for (dev_entry = dev_list.Next; dev_entry != &dev_list;
- dev_entry = dev_entry->Next) {
-
- dev = container_of(dev_entry, struct acm_device, entry);
-
- for (i = 0; i < dev->port_cnt; i++) {
- port = &dev->port[i];
-
- for (ep_entry = port->ep_list.Next;
- ep_entry != &port->ep_list;
- ep_entry = ep_entry->Next) {
-
- ep = container_of(ep_entry, struct acm_ep, entry);
- lock_acquire(&ep->lock);
- if (!DListEmpty(&ep->wait_queue))
- acm_process_wait_queue(ep, &next_expire);
- lock_release(&ep->lock);
- }
- }
- }
-
- acm_process_timeouts();
- wait = (int) (next_expire - time_stamp_ms());
- if (wait > 0 && atomic_get(&wait_cnt))
- event_wait(&timeout_event, wait);
- }
-}
-
-static void acm_init_server(void)
-{
- FILE *f;
- int i;
+ FILE *f;
+ int i;
for (i = 0; i < FD_SETSIZE - 1; i++) {
lock_init(&client_array[i].lock);
@@ -1764,7 +574,7 @@ static int acm_listen(void)
return 0;
}
-static void acm_disconnect_client(struct acm_client *client)
+static void acm_disconnect_client(struct acmc_client *client)
{
lock_acquire(&client->lock);
shutdown(client->sock, SHUT_RDWR);
@@ -1803,13 +613,13 @@ static void acm_svr_accept(void)
}
static int
-acm_is_path_from_port(struct acm_port *port, struct ibv_path_record *path)
+acm_is_path_from_port(struct acmc_port *port, struct ibv_path_record *path)
{
- union ibv_gid gid;
uint8_t i;
if (!ib_any_gid(&path->sgid)) {
- return (acm_gid_index(port, &path->sgid) < port->gid_cnt);
+ return (acm_gid_index(&port->port, &path->sgid) <
+ port->gid_cnt);
}
if (path->slid) {
@@ -1820,13 +630,13 @@ acm_is_path_from_port(struct acm_port *port, struct ibv_path_record *path)
return 1;
}
- if (acm_gid_index(port, &path->dgid) < port->gid_cnt) {
+ if (acm_gid_index(&port->port, &path->dgid) < port->gid_cnt) {
return 1;
}
for (i = 0; i < port->gid_cnt; i++) {
- ibv_query_gid(port->dev->verbs, port->port_num, i, &gid);
- if (gid.global.subnet_prefix == path->dgid.global.subnet_prefix) {
+ if (port->gid_tbl[i].global.subnet_prefix ==
+ path->dgid.global.subnet_prefix) {
return 1;
}
}
@@ -1834,11 +644,13 @@ acm_is_path_from_port(struct acm_port *port, struct ibv_path_record *path)
return 0;
}
-static struct acm_ep *
-acm_get_port_ep(struct acm_port *port, struct acm_ep_addr_data *data)
+static struct acmc_addr *
+acm_get_port_ep_address(struct acmc_port *port, struct acm_ep_addr_data *data)
{
- struct acm_ep *ep;
+ struct acmc_ep *ep;
+ struct acm_address *addr;
DLIST_ENTRY *ep_entry;
+ int i;
if (port->state != IBV_PORT_ACTIVE)
return NULL;
@@ -1848,28 +660,31 @@ acm_get_port_ep(struct acm_port *port, struct acm_ep_addr_data *data)
return NULL;
for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list;
- ep_entry = ep_entry->Next) {
-
- ep = container_of(ep_entry, struct acm_ep, entry);
- if (ep->state != ACM_READY)
- continue;
+ ep_entry = ep_entry->Next) {
+ ep = container_of(ep_entry, struct acmc_ep, entry);
if ((data->type == ACM_EP_INFO_PATH) &&
- (!data->info.path.pkey || (ntohs(data->info.path.pkey) == ep->pkey)))
- return ep;
+ (!data->info.path.pkey ||
+ (ntohs(data->info.path.pkey) == ep->endpoint.pkey))) {
+ for (i = 0; i < MAX_EP_ADDR; i++) {
+ if (ep->addr_info[i].addr.type)
+ return &ep->addr_info[i];
+ }
+ return NULL;
+ }
- if (acm_addr_index(ep, data->info.addr, (uint8_t) data->type) >= 0)
- return ep;
+ if ((addr = acm_addr_lookup(&ep->endpoint, data->info.addr,
+ (uint8_t) data->type)))
+ return container_of(addr, struct acmc_addr, addr);
}
return NULL;
}
-static struct acm_ep *
-acm_get_ep(struct acm_ep_addr_data *data)
+static struct acmc_addr *acm_get_ep_address(struct acm_ep_addr_data *data)
{
- struct acm_device *dev;
- struct acm_ep *ep;
+ struct acmc_device *dev;
+ struct acmc_addr *addr;
DLIST_ENTRY *dev_entry;
int i;
@@ -1879,13 +694,11 @@ acm_get_ep(struct acm_ep_addr_data *data)
for (dev_entry = dev_list.Next; dev_entry != &dev_list;
dev_entry = dev_entry->Next) {
- dev = container_of(dev_entry, struct acm_device, entry);
+ dev = container_of(dev_entry, struct acmc_device, entry);
for (i = 0; i < dev->port_cnt; i++) {
- lock_acquire(&dev->port[i].lock);
- ep = acm_get_port_ep(&dev->port[i], data);
- lock_release(&dev->port[i].lock);
- if (ep)
- return ep;
+ addr = acm_get_port_ep_address(&dev->port[i], data);
+ if (addr)
+ return addr;
}
}
@@ -1895,109 +708,58 @@ acm_get_ep(struct acm_ep_addr_data *data)
return NULL;
}
-static int
-acm_svr_query_path(struct acm_client *client, struct acm_msg *msg)
+static struct acmc_ep *acm_get_ep(int index)
{
- struct acm_request *req;
- struct acm_send_msg *sa_msg;
- struct ib_sa_mad *mad;
- struct acm_ep *ep;
- uint8_t status;
+ struct acmc_device *dev;
+ DLIST_ENTRY *dev_entry;
+ struct acmc_ep *ep;
+ DLIST_ENTRY *ep_entry;
+ int i, inx = 0;
- acm_log(2, "client %d\n", client->index);
- if (msg->hdr.length != ACM_MSG_HDR_LENGTH + ACM_MSG_EP_LENGTH) {
- acm_log(0, "ERROR - invalid length: 0x%x\n", msg->hdr.length);
- status = ACM_STATUS_EINVAL;
- goto resp;
+ acm_log(2, "ep index %d\n", index);
+ for (dev_entry = dev_list.Next; dev_entry != &dev_list;
+ dev_entry = dev_entry->Next) {
+ dev = container_of(dev_entry, struct acmc_device, entry);
+ for (i = 0; i < dev->port_cnt; i++) {
+ if (dev->port[i].state != IBV_PORT_ACTIVE)
+ continue;
+ for (ep_entry = dev->port[i].ep_list.Next;
+ ep_entry != &dev->port[i].ep_list;
+ ep_entry = ep_entry->Next, inx++) {
+ if (index == inx) {
+ ep = container_of(ep_entry,
+ struct acmc_ep,
+ entry);
+ return ep;
+ }
+ }
+ }
}
- ep = acm_get_ep(&msg->resolve_data[0]);
- if (!ep) {
- acm_log(1, "notice - could not find local end point\n");
- status = ACM_STATUS_ESRCADDR;
- goto resp;
- }
+ acm_log(1, "notice - could not find ep %d\n", index);
+ return NULL;
+}
- req = acm_alloc_req(client, msg);
- if (!req) {
- status = ACM_STATUS_ENOMEM;
- goto resp;
- }
+static int
+acm_svr_query_path(struct acmc_client *client, struct acm_msg *msg)
+{
+ struct acmc_addr *addr;
+ struct acmc_ep *ep;
- if (!acm_acquire_sa_dest(ep->port)) {
- acm_log(1, "cannot acquire SA destination\n");
- status = ACM_STATUS_EINVAL;
- goto free;
+ acm_log(2, "client %d\n", client->index);
+ if (msg->hdr.length != ACM_MSG_HDR_LENGTH + ACM_MSG_EP_LENGTH) {
+ acm_log(0, "ERROR - invalid length: 0x%x\n", msg->hdr.length);
+ return acmc_query_response(client->index, msg, ACM_STATUS_EINVAL);
}
- sa_msg = acm_alloc_send(ep, &ep->port->sa_dest, sizeof(*mad));
- acm_release_sa_dest(&ep->port->sa_dest);
- if (!sa_msg) {
- acm_log(0, "ERROR - cannot allocate send msg\n");
- status = ACM_STATUS_ENOMEM;
- goto free;
+ addr = acm_get_ep_address(&msg->resolve_data[0]);
+ if (!addr) {
+ acm_log(1, "notice - could not find local end point address\n");
+ return acmc_query_response(client->index, msg, ACM_STATUS_ESRCADDR);
}
- acm_init_send_req(sa_msg, (void *) req, acm_client_sa_resp);
- mad = (struct ib_sa_mad *) sa_msg->data;
- acm_init_path_query(mad);
-
- memcpy(mad->data, &msg->resolve_data[0].info.path,
- sizeof(struct ibv_path_record));
- mad->comp_mask = acm_path_comp_mask(&msg->resolve_data[0].info.path);
-
- atomic_inc(&counter[ACM_CNTR_ROUTE_QUERY]);
- acm_post_send(&ep->sa_queue, sa_msg);
- return ACM_STATUS_SUCCESS;
-
-free:
- acm_free_req(req);
-resp:
- return acm_client_query_resp(client, msg, status);
-}
-
-static uint8_t
-acm_send_resolve(struct acm_ep *ep, struct acm_dest *dest,
- struct acm_ep_addr_data *saddr)
-{
- struct acm_send_msg *msg;
- struct acm_mad *mad;
- struct acm_resolve_rec *rec;
- int i;
-
- acm_log(2, "\n");
- msg = acm_alloc_send(ep, &ep->mc_dest[0], sizeof(*mad));
- if (!msg) {
- acm_log(0, "ERROR - cannot allocate send msg\n");
- return ACM_STATUS_ENOMEM;
- }
-
- acm_init_send_req(msg, (void *) dest, acm_process_addr_resp);
- (void) atomic_inc(&dest->refcnt);
-
- mad = (struct acm_mad *) msg->data;
- mad->base_version = 1;
- mad->mgmt_class = ACM_MGMT_CLASS;
- mad->class_version = 1;
- mad->method = IB_METHOD_GET;
- mad->control = ACM_CTRL_RESOLVE;
- mad->tid = htonll((uint64_t) atomic_inc(&tid));
-
- rec = (struct acm_resolve_rec *) mad->data;
- rec->src_type = (uint8_t) saddr->type;
- rec->src_length = ACM_MAX_ADDRESS;
- memcpy(rec->src, saddr->info.addr, ACM_MAX_ADDRESS);
- rec->dest_type = dest->addr_type;
- rec->dest_length = ACM_MAX_ADDRESS;
- memcpy(rec->dest, dest->address, ACM_MAX_ADDRESS);
-
- rec->gid_cnt = (uint8_t) ep->mc_cnt;
- for (i = 0; i < ep->mc_cnt; i++)
- memcpy(&rec->gid[i], ep->mc_dest[i].address, 16);
-
- atomic_inc(&counter[ACM_CNTR_ADDR_QUERY]);
- acm_post_send(&ep->resolve_queue, msg);
- return 0;
+ ep = container_of(addr->addr.endpoint, struct acmc_ep, endpoint);
+ return ep->port->prov->query(addr->prov_addr_context, msg, client->index);
}
static int acm_svr_select_src(struct acm_ep_addr_data *src, struct acm_ep_addr_data *dst)
@@ -2007,9 +769,6 @@ static int acm_svr_select_src(struct acm_ep_addr_data *src, struct acm_ep_addr_d
int ret;
SOCKET s;
- if (src->type)
- return 0;
-
acm_log(2, "selecting source address\n");
memset(&addr, 0, sizeof addr);
switch (dst->type) {
@@ -2065,24 +824,24 @@ out:
* references to the source and destination addresses.
* The message buffer contains extra address data buffers. If a
* source address is not given, reference an empty address buffer,
- * and we'll resolve a source address later.
+ * and we'll resolve a source address later. Record the location of
+ * the source and destination addresses in the message header data
+ * to avoid further searches.
*/
-static uint8_t
-acm_svr_verify_resolve(struct acm_msg *msg,
- struct acm_ep_addr_data **saddr, struct acm_ep_addr_data **daddr)
+static uint8_t acm_svr_verify_resolve(struct acm_msg *msg)
{
- struct acm_ep_addr_data *src = NULL, *dst = NULL;
- int i, cnt;
+ int i, cnt, have_dst = 0;
if (msg->hdr.length < ACM_MSG_HDR_LENGTH) {
acm_log(0, "ERROR - invalid msg hdr length %d\n", msg->hdr.length);
return ACM_STATUS_EINVAL;
}
+ msg->hdr.src_out = 1;
cnt = (msg->hdr.length - ACM_MSG_HDR_LENGTH) / ACM_MSG_EP_LENGTH;
for (i = 0; i < cnt; i++) {
if (msg->resolve_data[i].flags & ACM_EP_FLAG_SOURCE) {
- if (src) {
+ if (!msg->hdr.src_out) {
acm_log(0, "ERROR - multiple sources specified\n");
return ACM_STATUS_ESRCADDR;
}
@@ -2091,10 +850,11 @@ acm_svr_verify_resolve(struct acm_msg *msg,
acm_log(0, "ERROR - unsupported source address type\n");
return ACM_STATUS_ESRCTYPE;
}
- src = &msg->resolve_data[i];
+ msg->hdr.src_out = 0;
+ msg->hdr.src_index = i;
}
if (msg->resolve_data[i].flags & ACM_EP_FLAG_DEST) {
- if (dst) {
+ if (have_dst) {
acm_log(0, "ERROR - multiple destinations specified\n");
return ACM_STATUS_EDESTADDR;
}
@@ -2103,145 +863,59 @@ acm_svr_verify_resolve(struct acm_msg *msg,
acm_log(0, "ERROR - unsupported destination address type\n");
return ACM_STATUS_EDESTTYPE;
}
- dst = &msg->resolve_data[i];
+ have_dst = 1;
+ msg->hdr.dst_index = i;
}
}
- if (!dst) {
+ if (!have_dst) {
acm_log(0, "ERROR - destination address required\n");
return ACM_STATUS_EDESTTYPE;
}
- if (!src) {
- msg->hdr.src_out = i;
- src = &msg->resolve_data[i];
- memset(src, 0, sizeof *src);
- }
- *saddr = src;
- *daddr = dst;
- return ACM_STATUS_SUCCESS;
-}
-
-/* Caller must hold dest lock */
-static uint8_t
-acm_svr_queue_req(struct acm_dest *dest, struct acm_client *client,
- struct acm_msg *msg)
-{
- struct acm_request *req;
-
- acm_log(2, "client %d\n", client->index);
- req = acm_alloc_req(client, msg);
- if (!req) {
- return ACM_STATUS_ENOMEM;
+ if (msg->hdr.src_out) {
+ msg->hdr.src_index = i;
+ memset(&msg->resolve_data[i], 0, sizeof(struct acm_ep_addr_data));
}
-
- DListInsertTail(&req->entry, &dest->req_queue);
return ACM_STATUS_SUCCESS;
}
-static int acm_dest_timeout(struct acm_dest *dest)
-{
- uint64_t timestamp = time_stamp_min();
-
- if (timestamp > dest->addr_timeout) {
- acm_log(2, "%s address timed out\n", dest->name);
- dest->state = ACM_INIT;
- return 1;
- } else if (timestamp > dest->route_timeout) {
- acm_log(2, "%s route timed out\n", dest->name);
- dest->state = ACM_ADDR_RESOLVED;
- return 1;
- }
- return 0;
-}
-
static int
-acm_svr_resolve_dest(struct acm_client *client, struct acm_msg *msg)
+acm_svr_resolve_dest(struct acmc_client *client, struct acm_msg *msg)
{
- struct acm_ep *ep;
- struct acm_dest *dest;
+ struct acmc_addr *addr;
+ struct acmc_ep *ep;
struct acm_ep_addr_data *saddr, *daddr;
uint8_t status;
- int ret;
acm_log(2, "client %d\n", client->index);
- status = acm_svr_verify_resolve(msg, &saddr, &daddr);
+ status = acm_svr_verify_resolve(msg);
if (status) {
acm_log(0, "notice - misformatted or unsupported request\n");
- return acm_client_resolve_resp(client, msg, NULL, status);
+ return acmc_resolve_response(client->index, msg, status);
}
- status = acm_svr_select_src(saddr, daddr);
- if (status) {
- acm_log(0, "notice - unable to select suitable source address\n");
- return acm_client_resolve_resp(client, msg, NULL, status);
+ saddr = &msg->resolve_data[msg->hdr.src_index];
+ daddr = &msg->resolve_data[msg->hdr.dst_index];
+ if (msg->hdr.src_out) {
+ status = acm_svr_select_src(saddr, daddr);
+ if (status) {
+ acm_log(0, "notice - unable to select suitable source address\n");
+ return acmc_resolve_response(client->index, msg, status);
+ }
}
acm_format_name(2, log_data, sizeof log_data,
saddr->type, saddr->info.addr, sizeof saddr->info.addr);
acm_log(2, "src %s\n", log_data);
- ep = acm_get_ep(saddr);
- if (!ep) {
- acm_log(0, "notice - unknown local end point\n");
- return acm_client_resolve_resp(client, msg, NULL, ACM_STATUS_ESRCADDR);
+ addr = acm_get_ep_address(saddr);
+ if (!addr) {
+ acm_log(0, "notice - unknown local end point address\n");
+ return acmc_resolve_response(client->index, msg, ACM_STATUS_ESRCADDR);
}
- acm_format_name(2, log_data, sizeof log_data,
- daddr->type, daddr->info.addr, sizeof daddr->info.addr);
- acm_log(2, "dest %s\n", log_data);
-
- dest = acm_acquire_dest(ep, daddr->type, daddr->info.addr);
- if (!dest) {
- acm_log(0, "ERROR - unable to allocate destination in client request\n");
- return acm_client_resolve_resp(client, msg, NULL, ACM_STATUS_ENOMEM);
- }
-
- lock_acquire(&dest->lock);
-test:
- switch (dest->state) {
- case ACM_READY:
- if (acm_dest_timeout(dest))
- goto test;
- acm_log(2, "request satisfied from local cache\n");
- atomic_inc(&counter[ACM_CNTR_ROUTE_CACHE]);
- status = ACM_STATUS_SUCCESS;
- break;
- case ACM_ADDR_RESOLVED:
- acm_log(2, "have address, resolving route\n");
- atomic_inc(&counter[ACM_CNTR_ADDR_CACHE]);
- status = acm_resolve_path(ep, dest, acm_dest_sa_resp);
- if (status) {
- break;
- }
- goto queue;
- case ACM_INIT:
- acm_log(2, "sending resolve msg to dest\n");
- status = acm_send_resolve(ep, dest, saddr);
- if (status) {
- break;
- }
- dest->state = ACM_QUERY_ADDR;
- /* fall through */
- default:
-queue:
- if (daddr->flags & ACM_FLAGS_NODELAY) {
- acm_log(2, "lookup initiated, but client wants no delay\n");
- status = ACM_STATUS_ENODATA;
- break;
- }
- status = acm_svr_queue_req(dest, client, msg);
- if (status) {
- break;
- }
- ret = 0;
- lock_release(&dest->lock);
- goto put;
- }
- lock_release(&dest->lock);
- ret = acm_client_resolve_resp(client, msg, dest, status);
-put:
- acm_put_dest(dest);
- return ret;
+ ep = container_of(addr->addr.endpoint, struct acmc_ep, endpoint);
+ return ep->port->prov->resolve(addr->prov_addr_context, msg, client->index);
}
/*
@@ -2250,94 +924,44 @@ put:
* lookup the destination by either LID or GID.
*/
static int
-acm_svr_resolve_path(struct acm_client *client, struct acm_msg *msg)
+acm_svr_resolve_path(struct acmc_client *client, struct acm_msg *msg)
{
- struct acm_ep *ep;
- struct acm_dest *dest;
+ struct acmc_addr *addr;
+ struct acmc_ep *ep;
struct ibv_path_record *path;
- uint8_t *addr;
- uint8_t status;
- int ret;
acm_log(2, "client %d\n", client->index);
if (msg->hdr.length < (ACM_MSG_HDR_LENGTH + ACM_MSG_EP_LENGTH)) {
acm_log(0, "notice - invalid msg hdr length %d\n", msg->hdr.length);
- return acm_client_resolve_resp(client, msg, NULL, ACM_STATUS_EINVAL);
+ return acmc_resolve_response(client->index, msg, ACM_STATUS_EINVAL);
}
path = &msg->resolve_data[0].info.path;
if (!path->dlid && ib_any_gid(&path->dgid)) {
acm_log(0, "notice - no destination specified\n");
- return acm_client_resolve_resp(client, msg, NULL, ACM_STATUS_EDESTADDR);
+ return acmc_resolve_response(client->index, msg,
+ ACM_STATUS_EDESTADDR);
}
acm_format_name(2, log_data, sizeof log_data, ACM_EP_INFO_PATH,
msg->resolve_data[0].info.addr, sizeof *path);
acm_log(2, "path %s\n", log_data);
- ep = acm_get_ep(&msg->resolve_data[0]);
- if (!ep) {
- acm_log(0, "notice - unknown local end point\n");
- return acm_client_resolve_resp(client, msg, NULL, ACM_STATUS_ESRCADDR);
+ addr = acm_get_ep_address(&msg->resolve_data[0]);
+ if (!addr) {
+ acm_log(0, "notice - unknown local end point address\n");
+ return acmc_resolve_response(client->index, msg,
+ ACM_STATUS_ESRCADDR);
}
- addr = msg->resolve_data[1].info.addr;
- memset(addr, 0, ACM_MAX_ADDRESS);
- if (path->dlid) {
- * ((uint16_t *) addr) = path->dlid;
- dest = acm_acquire_dest(ep, ACM_ADDRESS_LID, addr);
- } else {
- memcpy(addr, &path->dgid, sizeof path->dgid);
- dest = acm_acquire_dest(ep, ACM_ADDRESS_GID, addr);
- }
- if (!dest) {
- acm_log(0, "ERROR - unable to allocate destination in client request\n");
- return acm_client_resolve_resp(client, msg, NULL, ACM_STATUS_ENOMEM);
- }
-
- lock_acquire(&dest->lock);
-test:
- switch (dest->state) {
- case ACM_READY:
- if (acm_dest_timeout(dest))
- goto test;
- acm_log(2, "request satisfied from local cache\n");
- atomic_inc(&counter[ACM_CNTR_ROUTE_CACHE]);
- status = ACM_STATUS_SUCCESS;
- break;
- case ACM_INIT:
- acm_log(2, "have path, bypassing address resolution\n");
- acm_record_path_addr(ep, dest, path);
- /* fall through */
- case ACM_ADDR_RESOLVED:
- acm_log(2, "have address, resolving route\n");
- status = acm_resolve_path(ep, dest, acm_dest_sa_resp);
- if (status) {
- break;
- }
- /* fall through */
- default:
- if (msg->resolve_data[0].flags & ACM_FLAGS_NODELAY) {
- acm_log(2, "lookup initiated, but client wants no delay\n");
- status = ACM_STATUS_ENODATA;
- break;
- }
- status = acm_svr_queue_req(dest, client, msg);
- if (status) {
- break;
- }
- ret = 0;
- lock_release(&dest->lock);
- goto put;
- }
- lock_release(&dest->lock);
- ret = acm_client_resolve_resp(client, msg, dest, status);
-put:
- acm_put_dest(dest);
- return ret;
+ ep = container_of(addr->addr.endpoint, struct acmc_ep, endpoint);
+ return ep->port->prov->resolve(addr->prov_addr_context, msg,
+ client->index);
}
-static int acm_svr_resolve(struct acm_client *client, struct acm_msg *msg)
+static int acm_svr_resolve(struct acmc_client *client, struct acm_msg *msg)
{
+ (void) atomic_inc(&client->refcnt);
+
if (msg->resolve_data[0].type == ACM_EP_INFO_PATH) {
if (msg->resolve_data[0].flags & ACM_FLAGS_QUERY_SA) {
return acm_svr_query_path(client, msg);
@@ -2349,22 +973,95 @@ static int acm_svr_resolve(struct acm_client *client, struct acm_msg *msg)
}
}
-static int acm_svr_perf_query(struct acm_client *client, struct acm_msg *msg)
+static int acm_svr_perf_query(struct acmc_client *client, struct acm_msg *msg)
{
int ret, i;
uint16_t len;
+ struct acmc_addr *addr;
+ struct acmc_ep *ep = NULL;
+ int index;
acm_log(2, "client %d\n", client->index);
+ index = msg->hdr.data[1];
msg->hdr.opcode |= ACM_OP_ACK;
msg->hdr.status = ACM_STATUS_SUCCESS;
- msg->hdr.data[0] = ACM_MAX_COUNTER;
- msg->hdr.data[1] = 0;
msg->hdr.data[2] = 0;
- len = ACM_MSG_HDR_LENGTH + (ACM_MAX_COUNTER * sizeof(uint64_t));
+
+ if ((ntohs(msg->hdr.length) < (ACM_MSG_HDR_LENGTH + ACM_MSG_EP_LENGTH)
+ && index < 1) ||
+ ((ntohs(msg->hdr.length) >= (ACM_MSG_HDR_LENGTH + ACM_MSG_EP_LENGTH)
+ && !(msg->resolve_data[0].flags & ACM_EP_FLAG_SOURCE)))) {
+ for (i = 0; i < ACM_MAX_COUNTER; i++)
+ msg->perf_data[i] = htonll((uint64_t) atomic_get(&counter[i]));
+
+ msg->hdr.data[0] = ACM_MAX_COUNTER;
+ len = ACM_MSG_HDR_LENGTH + (ACM_MAX_COUNTER * sizeof(uint64_t));
+ } else {
+ if (index >= 1) {
+ ep = acm_get_ep(index - 1);
+ } else {
+ addr = acm_get_ep_address(&msg->resolve_data[0]);
+ if (addr)
+ ep = container_of(addr->addr.endpoint,
+ struct acmc_ep, endpoint);
+ }
+
+ if (ep) {
+ ep->port->prov->query_perf(ep->prov_ep_context,
+ msg->perf_data, &msg->hdr.data[0]);
+ len = ACM_MSG_HDR_LENGTH + (msg->hdr.data[0] * sizeof(uint64_t));
+ } else {
+ msg->hdr.status = ACM_STATUS_ESRCADDR;
+ len = ACM_MSG_HDR_LENGTH;
+ }
+ }
msg->hdr.length = htons(len);
- for (i = 0; i < ACM_MAX_COUNTER; i++)
- msg->perf_data[i] = htonll((uint64_t) atomic_get(&counter[i]));
+ ret = send(client->sock, (char *) msg, len, 0);
+ if (ret != len)
+ acm_log(0, "ERROR - failed to send response\n");
+ else
+ ret = 0;
+
+ return ret;
+}
+
+static int acm_svr_ep_query(struct acmc_client *client, struct acm_msg *msg)
+{
+ int ret, i;
+ uint16_t len;
+ struct acmc_ep *ep;
+ int index, cnt = 0;
+
+ acm_log(2, "client %d\n", client->index);
+ index = msg->hdr.data[0];
+ ep = acm_get_ep(index - 1);
+ if (ep) {
+ msg->hdr.status = ACM_STATUS_SUCCESS;
+ msg->ep_data[0].dev_guid = ep->port->dev->device.dev_guid;
+ msg->ep_data[0].port_num = ep->port->port.port_num;
+ msg->ep_data[0].pkey = htons(ep->endpoint.pkey);
+ strncpy((char *)msg->ep_data[0].prov_name, ep->port->prov->name,
+ ACM_MAX_PROV_NAME - 1);
+ msg->ep_data[0].prov_name[ACM_MAX_PROV_NAME - 1] = '\0';
+ len = ACM_MSG_HDR_LENGTH + sizeof(struct acm_ep_config_data);
+ for (i = 0; i < MAX_EP_ADDR; i++) {
+ if (ep->addr_info[i].addr.type != ACM_ADDRESS_INVALID) {
+ memcpy(msg->ep_data[0].addrs[cnt++].name,
+ ep->addr_info[i].string_buf,
+ ACM_MAX_ADDRESS);
+ }
+ }
+ msg->ep_data[0].addr_cnt = htons(cnt);
+ len += cnt * ACM_MAX_ADDRESS;
+ } else {
+ msg->hdr.status = ACM_STATUS_EINVAL;
+ len = ACM_MSG_HDR_LENGTH;
+ }
+ msg->hdr.opcode |= ACM_OP_ACK;
+ msg->hdr.data[1] = 0;
+ msg->hdr.data[2] = 0;
+ msg->hdr.length = htons(len);
ret = send(client->sock, (char *) msg, len, 0);
if (ret != len)
@@ -2381,7 +1078,7 @@ static int acm_msg_length(struct acm_msg *msg)
msg->hdr.length : ntohs(msg->hdr.length);
}
-static void acm_svr_receive(struct acm_client *client)
+static void acm_svr_receive(struct acmc_client *client)
{
struct acm_msg msg;
int ret;
@@ -2407,6 +1104,9 @@ static void acm_svr_receive(struct acm_client *client)
case ACM_OP_PERF_QUERY:
ret = acm_svr_perf_query(client, &msg);
break;
+ case ACM_OP_EP_QUERY:
+ ret = acm_svr_ep_query(client, &msg);
+ break;
default:
acm_log(0, "ERROR - unknown opcode 0x%x\n", msg.hdr.opcode);
break;
@@ -2417,12 +1117,6 @@ out:
acm_disconnect_client(client);
}
-static struct acm_device *
-acm_get_device_from_gid(union ibv_gid *sgid, uint8_t *port);
-static struct acm_ep *acm_find_ep(struct acm_port *port, uint16_t pkey);
-static int
-acm_ep_insert_addr(struct acm_ep *ep, uint8_t *addr, size_t addr_len, uint8_t addr_type);
-
static int acm_nl_to_addr_data(struct acm_ep_addr_data *ad,
int af_family, uint8_t *addr, size_t addr_len)
{
@@ -2444,19 +1138,18 @@ static int acm_nl_to_addr_data(struct acm_ep_addr_data *ad,
return 0;
}
-static void acm_add_ep_ip(struct acm_ep_addr_data *data, char *ifname)
+static void acm_add_ep_ip(char *ifname, struct acm_ep_addr_data *data, char *ip_str)
{
- struct acm_ep *ep;
- struct acm_device *dev;
+ struct acmc_ep *ep;
+ struct acmc_device *dev;
uint8_t port_num;
uint16_t pkey;
union ibv_gid sgid;
+ struct acmc_addr *addr;
- ep = acm_get_ep(data);
- if (ep) {
- acm_format_name(1, log_data, sizeof log_data,
- data->type, data->info.addr, sizeof data->info.addr);
- acm_log(1, "Address '%s' already available\n", log_data);
+ addr = acm_get_ep_address(data);
+ if (addr) {
+ acm_log(1, "Address '%s' already available\n", ip_str);
return;
}
@@ -2470,25 +1163,26 @@ static void acm_add_ep_ip(struct acm_ep_addr_data *data, char *ifname)
if (acm_if_get_pkey(ifname, &pkey))
return;
- acm_format_name(0, log_data, sizeof log_data,
- data->type, data->info.addr, sizeof data->info.addr);
- acm_log(0, " %s\n", log_data);
+ acm_log(0, " %s\n", ip_str);
- ep = acm_find_ep(&dev->port[port_num-1], pkey);
+ ep = acm_find_ep(&dev->port[port_num - 1], pkey);
if (ep) {
- if (acm_ep_insert_addr(ep, data->info.addr, sizeof data->info.addr, data->type))
- acm_log(0, "Failed to add '%s' to EP\n", log_data);
+ if (acm_ep_insert_addr(ep, ip_str, data->info.addr,
+ sizeof data->info.addr, data->type))
+ acm_log(0, "Failed to add '%s' to EP\n", ip_str);
} else {
- acm_log(0, "Failed to add '%s' no EP for pkey\n", log_data);
+ acm_log(0, "Failed to add '%s' no EP for pkey\n", ip_str);
}
}
static void acm_rm_ep_ip(struct acm_ep_addr_data *data)
{
- struct acm_ep *ep;
+ struct acmc_ep *ep;
+ struct acmc_addr *addr;
- ep = acm_get_ep(data);
- if (ep) {
+ addr = acm_get_ep_address(data);
+ if (addr) {
+ ep = container_of(addr->addr.endpoint, struct acmc_ep, endpoint);
acm_format_name(0, log_data, sizeof log_data,
data->type, data->info.addr, sizeof data->info.addr);
acm_log(0, " %s\n", log_data);
@@ -2519,27 +1213,25 @@ static int acm_ipnl_create(void)
static void acm_ip_iter_cb(char *ifname, union ibv_gid *gid, uint16_t pkey,
uint8_t addr_type, uint8_t *addr, size_t addr_len,
- char *addr_name, void *ctx)
+ char *ip_str, void *ctx)
{
int ret = EINVAL;
- struct acm_device *dev = NULL;
- struct acm_ep *ep = NULL;
+ struct acmc_device *dev;
+ struct acmc_ep *ep;
uint8_t port_num;
char gid_str[INET6_ADDRSTRLEN];
dev = acm_get_device_from_gid(gid, &port_num);
- if (dev)
- ep = acm_find_ep(&dev->port[port_num-1], pkey);
-
- if (ep)
- ret = acm_ep_insert_addr(ep, addr, addr_len, addr_type);
+ if (dev) {
+ ep = acm_find_ep(&dev->port[port_num - 1], pkey);
+ if (ep)
+ ret = acm_ep_insert_addr(ep, ip_str, addr, addr_len, addr_type);
+ }
if (ret) {
- acm_format_name(2, log_data, sizeof log_data,
- addr_type, addr, addr_len);
inet_ntop(AF_INET6, gid->raw, gid_str, sizeof(gid_str));
acm_log(0, "Failed to add '%s' (gid %s; pkey 0x%x)\n",
- log_data, gid_str, pkey);
+ ip_str, gid_str, pkey);
}
}
@@ -2550,31 +1242,30 @@ static void acm_ip_iter_cb(char *ifname, union ibv_gid *gid, uint16_t pkey,
static int resync_system_ips(void)
{
DLIST_ENTRY *dev_entry;
- struct acm_device *dev;
- int cnt;
+ struct acmc_device *dev;
+ struct acmc_port *port;
+ struct acmc_ep *ep;
+ DLIST_ENTRY *entry;
+ int i, cnt;
acm_log(0, "Resyncing all IP's\n");
/* mark all IP's invalid */
for (dev_entry = dev_list.Next; dev_entry != &dev_list;
dev_entry = dev_entry->Next) {
- struct acm_ep *ep = NULL;
- DLIST_ENTRY *entry;
-
- dev = container_of(dev_entry, struct acm_device, entry);
+ dev = container_of(dev_entry, struct acmc_device, entry);
for (cnt = 0; cnt < dev->port_cnt; cnt++) {
- struct acm_port *port = &dev->port[cnt];
+ port = &dev->port[cnt];
for (entry = port->ep_list.Next; entry != &port->ep_list;
entry = entry->Next) {
- int i;
+ ep = container_of(entry, struct acmc_ep, entry);
- ep = container_of(entry, struct acm_ep, entry);
for (i = 0; i < MAX_EP_ADDR; i++) {
- if (ep->addr_type[i] == ACM_ADDRESS_IP
- || ep->addr_type[i] == ACM_ADDRESS_IP6)
- ep->addr_type[i] = ACM_ADDRESS_INVALID;
+ if (ep->addr_info[i].addr.type == ACM_ADDRESS_IP ||
+ ep->addr_info[i].addr.type == ACM_ADDRESS_IP6)
+ ep->addr_info[i].addr.type = ACM_ADDRESS_INVALID;
}
}
}
@@ -2583,13 +1274,12 @@ static int resync_system_ips(void)
return acm_if_iter_sys(acm_ip_iter_cb, NULL);
}
-#define NL_MSG_BUF_SIZE 4096
static void acm_ipnl_handler(void)
{
int len;
char buffer[NL_MSG_BUF_SIZE];
struct nlmsghdr *nlh;
- char name[IFNAMSIZ];
+ char ifname[IFNAMSIZ];
char ip_str[INET6_ADDRSTRLEN];
struct acm_ep_addr_data ad;
@@ -2603,30 +1293,27 @@ static void acm_ipnl_handler(void)
switch (nlh->nlmsg_type) {
case RTM_NEWADDR:
- {
- if_indextoname(ifa->ifa_index, name);
+ if_indextoname(ifa->ifa_index, ifname);
while (rtl && RTA_OK(rth, rtl)) {
if (rth->rta_type == IFA_LOCAL) {
acm_log(1, "New system address available %s : %s\n",
- name, inet_ntop(ifa->ifa_family, RTA_DATA(rth),
+ ifname, inet_ntop(ifa->ifa_family, RTA_DATA(rth),
ip_str, sizeof(ip_str)));
if (!acm_nl_to_addr_data(&ad, ifa->ifa_family,
RTA_DATA(rth),
RTA_PAYLOAD(rth))) {
- acm_add_ep_ip(&ad, name);
+ acm_add_ep_ip(ifname, &ad, ip_str);
}
}
rth = RTA_NEXT(rth, rtl);
}
break;
- }
case RTM_DELADDR:
- {
- if_indextoname(ifa->ifa_index, name);
+ if_indextoname(ifa->ifa_index, ifname);
while (rtl && RTA_OK(rth, rtl)) {
if (rth->rta_type == IFA_LOCAL) {
acm_log(1, "System address removed %s : %s\n",
- name, inet_ntop(ifa->ifa_family, RTA_DATA(rth),
+ ifname, inet_ntop(ifa->ifa_family, RTA_DATA(rth),
ip_str, sizeof(ip_str)));
if (!acm_nl_to_addr_data(&ad, ifa->ifa_family,
RTA_DATA(rth),
@@ -2637,17 +1324,14 @@ static void acm_ipnl_handler(void)
rth = RTA_NEXT(rth, rtl);
}
break;
- }
case RTM_NEWLINK:
- {
- acm_log(2, "Link added : %s\n", if_indextoname(ifi->ifi_index, name));
+ acm_log(2, "Link added : %s\n",
+ if_indextoname(ifi->ifi_index, ifname));
break;
- }
case RTM_DELLINK:
- {
- acm_log(2, "Link removed : %s\n", if_indextoname(ifi->ifi_index, name));
+ acm_log(2, "Link removed : %s\n",
+ if_indextoname(ifi->ifi_index, ifname));
break;
- }
default:
acm_log(2, "unknown netlink message\n");
break;
@@ -2666,6 +1350,8 @@ static void acm_server(void)
{
fd_set readfds;
int i, n, ret;
+ struct acmc_device *dev;
+ DLIST_ENTRY *dev_entry;
acm_log(0, "started\n");
acm_init_server();
@@ -2690,481 +1376,132 @@ static void acm_server(void)
}
}
- ret = select(n + 1, &readfds, NULL, NULL, NULL);
- if (ret == SOCKET_ERROR) {
- acm_log(0, "ERROR - server select error\n");
- continue;
- }
-
- if (FD_ISSET(listen_socket, &readfds))
- acm_svr_accept();
-
- if (FD_ISSET(ip_mon_socket, &readfds))
- acm_ipnl_handler();
-
- for (i = 0; i < FD_SETSIZE - 1; i++) {
- if (client_array[i].sock != INVALID_SOCKET &&
- FD_ISSET(client_array[i].sock, &readfds)) {
- acm_log(2, "receiving from client %d\n", i);
- acm_svr_receive(&client_array[i]);
- }
- }
- }
-}
-
-static enum acm_addr_prot acm_convert_addr_prot(char *param)
-{
- if (!stricmp("acm", param))
- return ACM_ADDR_PROT_ACM;
-
- return addr_prot;
-}
-
-static enum acm_route_prot acm_convert_route_prot(char *param)
-{
- if (!stricmp("acm", param))
- return ACM_ROUTE_PROT_ACM;
- else if (!stricmp("sa", param))
- return ACM_ROUTE_PROT_SA;
-
- return route_prot;
-}
-
-static enum acm_loopback_prot acm_convert_loopback_prot(char *param)
-{
- if (!stricmp("none", param))
- return ACM_LOOPBACK_PROT_NONE;
- else if (!stricmp("local", param))
- return ACM_LOOPBACK_PROT_LOCAL;
-
- return loopback_prot;
-}
-
-static enum acm_route_preload acm_convert_route_preload(char *param)
-{
- if (!stricmp("none", param) || !stricmp("no", param))
- return ACM_ROUTE_PRELOAD_NONE;
- else if (!stricmp("opensm_full_v1", param))
- return ACM_ROUTE_PRELOAD_OSM_FULL_V1;
-
- return route_preload;
-}
-
-static enum acm_route_preload acm_convert_addr_preload(char *param)
-{
- if (!stricmp("none", param) || !stricmp("no", param))
- return ACM_ADDR_PRELOAD_NONE;
- else if (!stricmp("acm_hosts", param))
- return ACM_ADDR_PRELOAD_HOSTS;
-
- return addr_preload;
-}
-
-static enum ibv_rate acm_get_rate(uint8_t width, uint8_t speed)
-{
- switch (width) {
- case 1:
- switch (speed) {
- case 1: return IBV_RATE_2_5_GBPS;
- case 2: return IBV_RATE_5_GBPS;
- case 4: return IBV_RATE_10_GBPS;
- default: return IBV_RATE_MAX;
- }
- case 2:
- switch (speed) {
- case 1: return IBV_RATE_10_GBPS;
- case 2: return IBV_RATE_20_GBPS;
- case 4: return IBV_RATE_40_GBPS;
- default: return IBV_RATE_MAX;
- }
- case 4:
- switch (speed) {
- case 1: return IBV_RATE_20_GBPS;
- case 2: return IBV_RATE_40_GBPS;
- case 4: return IBV_RATE_80_GBPS;
- default: return IBV_RATE_MAX;
- }
- case 8:
- switch (speed) {
- case 1: return IBV_RATE_30_GBPS;
- case 2: return IBV_RATE_60_GBPS;
- case 4: return IBV_RATE_120_GBPS;
- default: return IBV_RATE_MAX;
- }
- default:
- acm_log(0, "ERROR - unknown link width 0x%x\n", width);
- return IBV_RATE_MAX;
- }
-}
-
-static enum ibv_mtu acm_convert_mtu(int mtu)
-{
- switch (mtu) {
- case 256: return IBV_MTU_256;
- case 512: return IBV_MTU_512;
- case 1024: return IBV_MTU_1024;
- case 2048: return IBV_MTU_2048;
- case 4096: return IBV_MTU_4096;
- default: return IBV_MTU_2048;
- }
-}
-
-static enum ibv_rate acm_convert_rate(int rate)
-{
- switch (rate) {
- case 2: return IBV_RATE_2_5_GBPS;
- case 5: return IBV_RATE_5_GBPS;
- case 10: return IBV_RATE_10_GBPS;
- case 20: return IBV_RATE_20_GBPS;
- case 30: return IBV_RATE_30_GBPS;
- case 40: return IBV_RATE_40_GBPS;
- case 60: return IBV_RATE_60_GBPS;
- case 80: return IBV_RATE_80_GBPS;
- case 120: return IBV_RATE_120_GBPS;
- default: return IBV_RATE_10_GBPS;
- }
-}
-
-static int acm_post_recvs(struct acm_ep *ep)
-{
- int i, size;
-
- size = recv_depth * ACM_RECV_SIZE;
- ep->recv_bufs = malloc(size);
- if (!ep->recv_bufs) {
- acm_log(0, "ERROR - unable to allocate receive buffer\n");
- return ACM_STATUS_ENOMEM;
- }
-
- ep->mr = ibv_reg_mr(ep->port->dev->pd, ep->recv_bufs, size,
- IBV_ACCESS_LOCAL_WRITE);
- if (!ep->mr) {
- acm_log(0, "ERROR - unable to register receive buffer\n");
- goto err;
- }
-
- for (i = 0; i < recv_depth; i++) {
- acm_post_recv(ep, (uintptr_t) (ep->recv_bufs + ACM_RECV_SIZE * i));
- }
- return 0;
-
-err:
- free(ep->recv_bufs);
- return -1;
-}
-
-static FILE *acm_open_addr_file(void)
-{
- FILE *f;
-
- if ((f = fopen(addr_file, "r")))
- return f;
-
- acm_log(0, "notice - generating %s file\n", addr_file);
- if (!(f = popen(acme, "r"))) {
- acm_log(0, "ERROR - cannot generate %s\n", addr_file);
- return NULL;
- }
- pclose(f);
- return fopen(addr_file, "r");
-}
-
-/* Parse "opensm full v1" file to build LID to GUID table */
-static void acm_parse_osm_fullv1_lid2guid(FILE *f, uint64_t *lid2guid)
-{
- char s[128];
- char *p, *ptr, *p_guid, *p_lid;
- uint64_t guid;
- uint16_t lid;
-
- while (fgets(s, sizeof s, f)) {
- if (s[0] == '#')
- continue;
- if (!(p = strtok_r(s, " \n", &ptr)))
- continue; /* ignore blank lines */
-
- if (strncmp(p, "Switch", sizeof("Switch") - 1) &&
- strncmp(p, "Channel", sizeof("Channel") - 1) &&
- strncmp(p, "Router", sizeof("Router") - 1))
- continue;
-
- if (!strncmp(p, "Channel", sizeof("Channel") - 1)) {
- p = strtok_r(NULL, " ", &ptr); /* skip 'Adapter' */
- if (!p)
- continue;
- }
-
- p_guid = strtok_r(NULL, ",", &ptr);
- if (!p_guid)
- continue;
-
- guid = (uint64_t) strtoull(p_guid, NULL, 16);
-
- ptr = strstr(ptr, "base LID");
- if (!ptr)
- continue;
- ptr += sizeof("base LID");
- p_lid = strtok_r(NULL, ",", &ptr);
- if (!p_lid)
- continue;
-
- lid = (uint16_t) strtoul(p_lid, NULL, 0);
- if (lid >= IB_LID_MCAST_START)
- continue;
- if (lid2guid[lid])
- acm_log(0, "ERROR - duplicate lid %u\n", lid);
- else
- lid2guid[lid] = htonll(guid);
- }
-}
-
-/* Parse 'opensm full v1' file to populate PR cache */
-static int acm_parse_osm_fullv1_paths(FILE *f, uint64_t *lid2guid, struct acm_ep *ep)
-{
- union ibv_gid sgid, dgid;
- struct ibv_port_attr attr = { 0 };
- struct acm_dest *dest;
- char s[128];
- char *p, *ptr, *p_guid, *p_lid;
- uint64_t guid;
- uint16_t lid, dlid, net_dlid;
- int sl, mtu, rate;
- int ret = 1, i;
- uint8_t addr[ACM_MAX_ADDRESS];
- uint8_t addr_type;
-
- ibv_query_gid(ep->port->dev->verbs, ep->port->port_num, 0, &sgid);
-
- /* Search for endpoint's SLID */
- while (fgets(s, sizeof s, f)) {
- if (s[0] == '#')
- continue;
- if (!(p = strtok_r(s, " \n", &ptr)))
- continue; /* ignore blank lines */
-
- if (strncmp(p, "Switch", sizeof("Switch") - 1) &&
- strncmp(p, "Channel", sizeof("Channel") - 1) &&
- strncmp(p, "Router", sizeof("Router") - 1))
- continue;
-
- if (!strncmp(p, "Channel", sizeof("Channel") - 1)) {
- p = strtok_r(NULL, " ", &ptr); /* skip 'Adapter' */
- if (!p)
- continue;
- }
-
- p_guid = strtok_r(NULL, ",", &ptr);
- if (!p_guid)
- continue;
-
- guid = (uint64_t) strtoull(p_guid, NULL, 16);
- if (guid != ntohll(sgid.global.interface_id))
- continue;
-
- ptr = strstr(ptr, "base LID");
- if (!ptr)
- continue;
- ptr += sizeof("base LID");
- p_lid = strtok_r(NULL, ",", &ptr);
- if (!p_lid)
- continue;
-
- lid = (uint16_t) strtoul(p_lid, NULL, 0);
- if (lid != ep->port->lid)
- continue;
-
- ibv_query_port(ep->port->dev->verbs, ep->port->port_num, &attr);
- ret = 0;
- break;
- }
-
- while (fgets(s, sizeof s, f)) {
- if (s[0] == '#')
- continue;
- if (!(p = strtok_r(s, " \n", &ptr)))
- continue; /* ignore blank lines */
-
- if (!strncmp(p, "Switch", sizeof("Switch") - 1) ||
- !strncmp(p, "Channel", sizeof("Channel") - 1) ||
- !strncmp(p, "Router", sizeof("Router") - 1))
- break;
-
- dlid = strtoul(p, NULL, 0);
- net_dlid = htons(dlid);
-
- p = strtok_r(NULL, ":", &ptr);
- if (!p)
- continue;
- if (strcmp(p, "UNREACHABLE") == 0)
- continue;
- sl = atoi(p);
-
- p = strtok_r(NULL, ":", &ptr);
- if (!p)
- continue;
- mtu = atoi(p);
-
- p = strtok_r(NULL, ":", &ptr);
- if (!p)
- continue;
- rate = atoi(p);
+ for (dev_entry = dev_list.Next; dev_entry != &dev_list;
+ dev_entry = dev_entry->Next) {
+ dev = container_of(dev_entry, struct acmc_device, entry);
+ FD_SET(dev->device.verbs->async_fd, &readfds);
+ n = max(n, (int) dev->device.verbs->async_fd);
+ }
- if (!lid2guid[dlid]) {
- acm_log(0, "ERROR - dlid %u not found in lid2guid table\n", dlid);
+ ret = select(n + 1, &readfds, NULL, NULL, NULL);
+ if (ret == SOCKET_ERROR) {
+ acm_log(0, "ERROR - server select error\n");
continue;
}
- dgid.global.subnet_prefix = sgid.global.subnet_prefix;
- dgid.global.interface_id = lid2guid[dlid];
-
- for (i = 0; i < 2; i++) {
- memset(addr, 0, ACM_MAX_ADDRESS);
- if (i == 0) {
- addr_type = ACM_ADDRESS_LID;
- memcpy(addr, &net_dlid, sizeof net_dlid);
- } else {
- addr_type = ACM_ADDRESS_GID;
- memcpy(addr, &dgid, sizeof(dgid));
- }
- dest = acm_acquire_dest(ep, addr_type, addr);
- if (!dest) {
- acm_log(0, "ERROR - unable to create dest\n");
- break;
+ if (FD_ISSET(listen_socket, &readfds))
+ acm_svr_accept();
+
+ if (FD_ISSET(ip_mon_socket, &readfds))
+ acm_ipnl_handler();
+
+ for (i = 0; i < FD_SETSIZE - 1; i++) {
+ if (client_array[i].sock != INVALID_SOCKET &&
+ FD_ISSET(client_array[i].sock, &readfds)) {
+ acm_log(2, "receiving from client %d\n", i);
+ acm_svr_receive(&client_array[i]);
}
+ }
- dest->path.sgid = sgid;
- dest->path.slid = htons(ep->port->lid);
- dest->path.dgid = dgid;
- dest->path.dlid = net_dlid;
- dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE;
- dest->path.pkey = htons(ep->pkey);
- dest->path.mtu = (uint8_t) mtu;
- dest->path.rate = (uint8_t) rate;
- dest->path.qosclass_sl = htons((uint16_t) sl & 0xF);
- if (dlid == ep->port->lid) {
- dest->path.packetlifetime = 0;
- dest->addr_timeout = (uint64_t)~0ULL;
- dest->route_timeout = (uint64_t)~0ULL;
- } else {
- dest->path.packetlifetime = attr.subnet_timeout;
- dest->addr_timeout = time_stamp_min() + (unsigned) addr_timeout;
- dest->route_timeout = time_stamp_min() + (unsigned) route_timeout;
+ for (dev_entry = dev_list.Next; dev_entry != &dev_list;
+ dev_entry = dev_entry->Next) {
+ dev = container_of(dev_entry, struct acmc_device, entry);
+ if (FD_ISSET(dev->device.verbs->async_fd, &readfds)) {
+ acm_log(2, "handling event from %s\n",
+ dev->device.verbs->device->name);
+ acm_event_handler(dev);
}
- dest->remote_qpn = 1;
- dest->state = ACM_READY;
- acm_put_dest(dest);
- acm_log(1, "added cached dest %s\n", dest->name);
}
}
- return ret;
}
-static int acm_parse_osm_fullv1(struct acm_ep *ep)
+enum ibv_rate acm_get_rate(uint8_t width, uint8_t speed)
{
- FILE *f;
- uint64_t *lid2guid;
- int ret = 1;
-
- if (!(f = fopen(route_data_file, "r"))) {
- acm_log(0, "ERROR - couldn't open %s\n", route_data_file);
- return ret;
+ switch (width) {
+ case 1:
+ switch (speed) {
+ case 1: return IBV_RATE_2_5_GBPS;
+ case 2: return IBV_RATE_5_GBPS;
+ case 4: return IBV_RATE_10_GBPS;
+ default: return IBV_RATE_MAX;
+ }
+ case 2:
+ switch (speed) {
+ case 1: return IBV_RATE_10_GBPS;
+ case 2: return IBV_RATE_20_GBPS;
+ case 4: return IBV_RATE_40_GBPS;
+ default: return IBV_RATE_MAX;
+ }
+ case 4:
+ switch (speed) {
+ case 1: return IBV_RATE_20_GBPS;
+ case 2: return IBV_RATE_40_GBPS;
+ case 4: return IBV_RATE_80_GBPS;
+ default: return IBV_RATE_MAX;
+ }
+ case 8:
+ switch (speed) {
+ case 1: return IBV_RATE_30_GBPS;
+ case 2: return IBV_RATE_60_GBPS;
+ case 4: return IBV_RATE_120_GBPS;
+ default: return IBV_RATE_MAX;
+ }
+ default:
+ acm_log(0, "ERROR - unknown link width 0x%x\n", width);
+ return IBV_RATE_MAX;
}
+}
- lid2guid = calloc(IB_LID_MCAST_START, sizeof(*lid2guid));
- if (!lid2guid) {
- acm_log(0, "ERROR - no memory for path record parsing\n");
- goto err;
+enum ibv_mtu acm_convert_mtu(int mtu)
+{
+ switch (mtu) {
+ case 256: return IBV_MTU_256;
+ case 512: return IBV_MTU_512;
+ case 1024: return IBV_MTU_1024;
+ case 2048: return IBV_MTU_2048;
+ case 4096: return IBV_MTU_4096;
+ default: return IBV_MTU_2048;
}
+}
- acm_parse_osm_fullv1_lid2guid(f, lid2guid);
- rewind(f);
- ret = acm_parse_osm_fullv1_paths(f, lid2guid, ep);
- free(lid2guid);
-err:
- fclose(f);
- return ret;
+enum ibv_rate acm_convert_rate(int rate)
+{
+ switch (rate) {
+ case 2: return IBV_RATE_2_5_GBPS;
+ case 5: return IBV_RATE_5_GBPS;
+ case 10: return IBV_RATE_10_GBPS;
+ case 20: return IBV_RATE_20_GBPS;
+ case 30: return IBV_RATE_30_GBPS;
+ case 40: return IBV_RATE_40_GBPS;
+ case 60: return IBV_RATE_60_GBPS;
+ case 80: return IBV_RATE_80_GBPS;
+ case 120: return IBV_RATE_120_GBPS;
+ default: return IBV_RATE_10_GBPS;
+ }
}
-static void acm_parse_hosts_file(struct acm_ep *ep)
+static FILE *acm_open_addr_file(void)
{
FILE *f;
- char s[120];
- char addr[INET6_ADDRSTRLEN], gid[INET6_ADDRSTRLEN];
- uint8_t name[ACM_MAX_ADDRESS];
- struct in6_addr ip_addr, ib_addr;
- struct acm_dest *dest, *gid_dest;
- uint8_t addr_type;
-
- if (!(f = fopen(addr_data_file, "r"))) {
- acm_log(0, "ERROR - couldn't open %s\n", addr_data_file);
- return;
- }
-
- while (fgets(s, sizeof s, f)) {
- if (s[0] == '#')
- continue;
-
- if (sscanf(s, "%46s%46s", addr, gid) != 2)
- continue;
-
- acm_log(2, "%s", s);
- if (inet_pton(AF_INET6, gid, &ib_addr) <= 0) {
- acm_log(0, "ERROR - %s is not IB GID\n", gid);
- continue;
- }
- memset(name, 0, ACM_MAX_ADDRESS);
- if (inet_pton(AF_INET, addr, &ip_addr) > 0) {
- addr_type = ACM_ADDRESS_IP;
- memcpy(name, &ip_addr, 4);
- } else if (inet_pton(AF_INET6, addr, &ip_addr) > 0) {
- addr_type = ACM_ADDRESS_IP6;
- memcpy(name, &ip_addr, sizeof(ip_addr));
- } else {
- addr_type = ACM_ADDRESS_NAME;
- strncpy((char *)name, addr, ACM_MAX_ADDRESS);
- }
-
- dest = acm_acquire_dest(ep, addr_type, name);
- if (!dest) {
- acm_log(0, "ERROR - unable to create dest %s\n", addr);
- continue;
- }
- memset(name, 0, ACM_MAX_ADDRESS);
- memcpy(name, &ib_addr, sizeof(ib_addr));
- gid_dest = acm_get_dest(ep, ACM_ADDRESS_GID, name);
- if (gid_dest) {
- dest->path = gid_dest->path;
- dest->state = ACM_READY;
- acm_put_dest(gid_dest);
- } else {
- memcpy(&dest->path.dgid, &ib_addr, 16);
- //ibv_query_gid(ep->port->dev->verbs, ep->port->port_num,
- // 0, &dest->path.sgid);
- dest->path.slid = htons(ep->port->lid);
- dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE;
- dest->path.pkey = htons(ep->pkey);
- dest->state = ACM_ADDR_RESOLVED;
- }
+ if ((f = fopen(addr_file, "r")))
+ return f;
- dest->remote_qpn = 1;
- dest->addr_timeout = time_stamp_min() + (unsigned) addr_timeout;
- dest->route_timeout = time_stamp_min() + (unsigned) route_timeout;
- acm_put_dest(dest);
- acm_log(1, "added host %s address type %d IB GID %s\n",
- addr, addr_type, gid);
+ acm_log(0, "notice - generating %s file\n", addr_file);
+ if (!(f = popen(acme, "r"))) {
+ acm_log(0, "ERROR - cannot generate %s\n", addr_file);
+ return NULL;
}
-
- fclose(f);
+ pclose(f);
+ return fopen(addr_file, "r");
}
static int
-acm_ep_insert_addr(struct acm_ep *ep, uint8_t *addr, size_t addr_len, uint8_t addr_type)
+acm_ep_insert_addr(struct acmc_ep *ep, const char *name, uint8_t *addr,
+ size_t addr_len, uint8_t addr_type)
{
- int i;
- int ret = ENOMEM;
+ int i, ret = -1;
uint8_t tmp[ACM_MAX_ADDRESS];
- char name_str[INET6_ADDRSTRLEN];
if (addr_len > ACM_MAX_ADDRESS)
return EINVAL;
@@ -3172,69 +1509,62 @@ acm_ep_insert_addr(struct acm_ep *ep, uint8_t *addr, size_t addr_len, uint8_t ad
memset(tmp, 0, sizeof tmp);
memcpy(tmp, addr, addr_len);
- lock_acquire(&ep->lock);
- if (acm_addr_index(ep, tmp, addr_type) < 0) {
- for (i = 0; i < MAX_EP_ADDR; i++) {
- if (ep->addr_type[i] == ACM_ADDRESS_INVALID) {
-
- ep->addr_type[i] = addr_type;
- memcpy(ep->addr[i].addr, tmp, ACM_MAX_ADDRESS);
-
- switch (addr_type) {
- case ACM_ADDRESS_IP:
- inet_ntop(AF_INET, addr, name_str, sizeof name_str);
- strncpy(ep->name[i], name_str, ACM_MAX_ADDRESS);
- break;
- case ACM_ADDRESS_IP6:
- inet_ntop(AF_INET6, addr, name_str, sizeof name_str);
- strncpy(ep->name[i], name_str, ACM_MAX_ADDRESS);
- break;
- case ACM_ADDRESS_NAME:
- strncpy(ep->name[i], (const char *)addr, ACM_MAX_ADDRESS);
- break;
- }
+ if (!acm_addr_lookup(&ep->endpoint, addr, addr_type)) {
+ for (i = 0; (i < MAX_EP_ADDR) &&
+ (ep->addr_info[i].addr.type != ACM_ADDRESS_INVALID); i++)
+ ;
+ if (i == MAX_EP_ADDR) {
+ ret = ENOMEM;
+ goto out;
+ }
- ret = 0;
- break;
+ /* Open the provider endpoint only if at least a name or
+ address is found */
+ if (!ep->prov_ep_context) {
+ ret = ep->port->prov->open_endpoint(&ep->endpoint,
+ ep->port->prov_port_context,
+ &ep->prov_ep_context);
+ if (ret) {
+ acm_log(0, "Error: failed to open prov ep\n");
+ goto out;
}
}
- } else {
- ret = 0;
+ ep->addr_info[i].addr.type = addr_type;
+ strncpy(ep->addr_info[i].string_buf, name, ACM_MAX_ADDRESS);
+ memcpy(ep->addr_info[i].addr.info.addr, tmp, ACM_MAX_ADDRESS);
+ ret = ep->port->prov->add_address(&ep->addr_info[i].addr,
+ ep->prov_ep_context,
+ &ep->addr_info[i].prov_addr_context);
+ if (ret) {
+ acm_log(0, "Error: failed to add addr to provider\n");
+ ep->addr_info[i].addr.type = ACM_ADDRESS_INVALID;
+ goto out;
+ }
}
- lock_release(&ep->lock);
+ ret = 0;
+out:
return ret;
}
-static struct acm_device *
+static struct acmc_device *
acm_get_device_from_gid(union ibv_gid *sgid, uint8_t *port)
{
DLIST_ENTRY *dev_entry;
- struct acm_device *dev;
- struct ibv_device_attr dev_attr;
- struct ibv_port_attr port_attr;
- union ibv_gid gid;
- int ret, i;
+ struct acmc_device *dev;
+ int i;
for (dev_entry = dev_list.Next; dev_entry != &dev_list;
dev_entry = dev_entry->Next) {
- dev = container_of(dev_entry, struct acm_device, entry);
-
- ret = ibv_query_device(dev->verbs, &dev_attr);
- if (ret)
- continue;
+ dev = container_of(dev_entry, struct acmc_device, entry);
- for (*port = 1; *port <= dev_attr.phys_port_cnt; (*port)++) {
- ret = ibv_query_port(dev->verbs, *port, &port_attr);
- if (ret)
- continue;
+ for (*port = 1; *port <= dev->port_cnt; (*port)++) {
- for (i = 0; i < port_attr.gid_tbl_len; i++) {
- ret = ibv_query_gid(dev->verbs, *port, i, &gid);
- if (ret || !gid.global.interface_id)
- break;
+ for (i = 0; i < dev->port[*port - 1].gid_cnt; i++) {
- if (!memcmp(sgid->raw, gid.raw, sizeof gid))
+ if (!memcmp(sgid->raw,
+ dev->port[*port - 1].gid_tbl[i].raw,
+ sizeof(*sgid)))
return dev;
}
}
@@ -3244,76 +1574,76 @@ acm_get_device_from_gid(union ibv_gid *sgid, uint8_t *port)
static void acm_ep_ip_iter_cb(char *ifname, union ibv_gid *gid, uint16_t pkey,
uint8_t addr_type, uint8_t *addr, size_t addr_len,
- char *addr_name, void *ctx)
+ char *ip_str, void *ctx)
{
uint8_t port_num;
- struct acm_device *dev;
- struct acm_ep *ep = (struct acm_ep *)ctx;
+ struct acmc_device *dev;
+ struct acmc_ep *ep = ctx;
dev = acm_get_device_from_gid(gid, &port_num);
if (dev && ep->port->dev == dev
- && ep->port->port_num == port_num && ep->pkey == pkey) {
- if (!acm_ep_insert_addr(ep, addr, addr_len, addr_type)) {
- acm_log(0, "Added %s %s %d 0x%x from %s\n", addr_name,
- dev->verbs->device->name, port_num, pkey,
+ && ep->port->port.port_num == port_num && ep->endpoint.pkey == pkey) {
+ if (!acm_ep_insert_addr(ep, ip_str, addr, addr_len, addr_type)) {
+ acm_log(0, "Added %s %s %d 0x%x from %s\n", ip_str,
+ dev->device.verbs->device->name, port_num, pkey,
ifname);
}
}
}
-static int acm_get_system_ips(struct acm_ep *ep)
+static int acm_get_system_ips(struct acmc_ep *ep)
{
- return acm_if_iter_sys(acm_ep_ip_iter_cb, (void *)ep);
+ return acm_if_iter_sys(acm_ep_ip_iter_cb, ep);
}
-static int acm_assign_ep_names(struct acm_ep *ep)
+static int acm_assign_ep_names(struct acmc_ep *ep)
{
FILE *faddr;
char *dev_name;
char s[120];
- char dev[32], addr[INET6_ADDRSTRLEN], pkey_str[8];
+ char dev[32], name[ACM_MAX_ADDRESS], pkey_str[8];
uint16_t pkey;
- uint8_t type;
- int port, ret = 0;
- struct in6_addr ip_addr;
+ uint8_t addr[ACM_MAX_ADDRESS], type;
+ int port;
size_t addr_len;
- dev_name = ep->port->dev->verbs->device->name;
+ dev_name = ep->port->dev->device.verbs->device->name;
acm_log(1, "device %s, port %d, pkey 0x%x\n",
- dev_name, ep->port->port_num, ep->pkey);
+ dev_name, ep->port->port.port_num, ep->endpoint.pkey);
acm_get_system_ips(ep);
if (!(faddr = acm_open_addr_file())) {
acm_log(0, "ERROR - address file not found\n");
- return -1;
+ goto out;
}
while (fgets(s, sizeof s, faddr)) {
if (s[0] == '#')
continue;
- if (sscanf(s, "%46s%32s%d%8s", addr, dev, &port, pkey_str) != 4)
+ if (sscanf(s, "%46s%32s%d%8s", name, dev, &port, pkey_str) != 4)
continue;
acm_log(2, "%s", s);
- if (inet_pton(AF_INET, addr, &ip_addr) > 0) {
+ if (inet_pton(AF_INET, name, addr) > 0) {
if (!support_ips_in_addr_cfg) {
acm_log(0, "ERROR - IP's are not configured to be read from ibacm_addr.cfg\n");
continue;
}
type = ACM_ADDRESS_IP;
addr_len = 4;
- } else if (inet_pton(AF_INET6, addr, &ip_addr) > 0) {
+ } else if (inet_pton(AF_INET6, name, addr) > 0) {
if (!support_ips_in_addr_cfg) {
acm_log(0, "ERROR - IP's are not configured to be read from ibacm_addr.cfg\n");
continue;
}
type = ACM_ADDRESS_IP6;
- addr_len = ACM_MAX_ADDRESS;
+ addr_len = 16;
} else {
type = ACM_ADDRESS_NAME;
- addr_len = strlen(addr);
+ addr_len = strlen(name);
+ memcpy(addr, name, addr_len);
}
if (stricmp(pkey_str, "default")) {
@@ -3325,11 +1655,11 @@ static int acm_assign_ep_names(struct acm_ep *ep)
pkey = 0xFFFF;
}
- if (!stricmp(dev_name, dev) && (ep->port->port_num == (uint8_t) port) &&
- (ep->pkey == pkey)) {
-
- acm_log(1, "assigning %s\n", addr);
- if ((ret = acm_ep_insert_addr(ep, (uint8_t *)&ip_addr, addr_len, type)) != 0) {
+ if (!stricmp(dev_name, dev) &&
+ (ep->port->port.port_num == (uint8_t) port) &&
+ (ep->endpoint.pkey == pkey)) {
+ acm_log(1, "assigning %s\n", name);
+ if (acm_ep_insert_addr(ep, name, addr, addr_len, type)) {
acm_log(1, "maximum number of names assigned to EP\n");
break;
}
@@ -3337,96 +1667,52 @@ static int acm_assign_ep_names(struct acm_ep *ep)
}
fclose(faddr);
- return ret;
-}
-
-/*
- * We currently require that the routing data be preloaded in order to
- * load the address data. This is backwards from normal operation, which
- * usually resolves the address before the route.
- */
-static void acm_ep_preload(struct acm_ep *ep)
-{
- switch (route_preload) {
- case ACM_ROUTE_PRELOAD_OSM_FULL_V1:
- if (acm_parse_osm_fullv1(ep))
- acm_log(0, "ERROR - failed to preload EP\n");
- break;
- default:
- break;
- }
-
- switch (addr_preload) {
- case ACM_ADDR_PRELOAD_HOSTS:
- acm_parse_hosts_file(ep);
- break;
- default:
- break;
- }
-}
-
-static int acm_init_ep_loopback(struct acm_ep *ep)
-{
- struct acm_dest *dest;
- int i;
-
- acm_log(2, "\n");
- if (loopback_prot != ACM_LOOPBACK_PROT_LOCAL)
- return 0;
-
- for (i = 0; i < MAX_EP_ADDR && ep->addr_type[i]; i++) {
- dest = acm_acquire_dest(ep, ep->addr_type[i], ep->addr[i].addr);
- if (!dest) {
- acm_format_name(0, log_data, sizeof log_data,
- ep->addr_type[i], ep->addr[i].addr,
- sizeof ep->addr[i].addr);
- acm_log(0, "ERROR - unable to create loopback dest %s\n", log_data);
- return -1;
- }
-
- ibv_query_gid(ep->port->dev->verbs, ep->port->port_num,
- 0, &dest->path.sgid);
-
- dest->path.dgid = dest->path.sgid;
- dest->path.dlid = dest->path.slid = htons(ep->port->lid);
- dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE;
- dest->path.pkey = htons(ep->pkey);
- dest->path.mtu = (uint8_t) ep->port->mtu;
- dest->path.rate = (uint8_t) ep->port->rate;
-
- dest->remote_qpn = ep->qp->qp_num;
- dest->addr_timeout = (uint64_t) ~0ULL;
- dest->route_timeout = (uint64_t) ~0ULL;
- dest->state = ACM_READY;
- acm_put_dest(dest);
- acm_log(1, "added loopback dest %s\n", dest->name);
- }
- return 0;
+out:
+ return (ep->addr_info[0].addr.type == ACM_ADDRESS_INVALID);
}
-static struct acm_ep *acm_find_ep(struct acm_port *port, uint16_t pkey)
+static struct acmc_ep *acm_find_ep(struct acmc_port *port, uint16_t pkey)
{
- struct acm_ep *ep, *res = NULL;
+ struct acmc_ep *ep, *res = NULL;
DLIST_ENTRY *entry;
acm_log(2, "pkey 0x%x\n", pkey);
- lock_acquire(&port->lock);
for (entry = port->ep_list.Next; entry != &port->ep_list; entry = entry->Next) {
- ep = container_of(entry, struct acm_ep, entry);
- if (ep->pkey == pkey) {
+ ep = container_of(entry, struct acmc_ep, entry);
+ if (ep->endpoint.pkey == pkey) {
res = ep;
break;
}
}
- lock_release(&port->lock);
return res;
}
-static struct acm_ep *
-acm_alloc_ep(struct acm_port *port, uint16_t pkey, uint16_t pkey_index)
+static void acm_ep_down(struct acmc_ep *ep)
+{
+ int i;
+
+ acm_log(1, "%s %d pkey 0x%04x\n",
+ ep->port->dev->device.verbs->device->name,
+ ep->port->port.port_num, ep->endpoint.pkey);
+ for (i = 0; i < MAX_EP_ADDR; i++) {
+ if (ep->addr_info[i].addr.type &&
+ ep->addr_info[i].prov_addr_context)
+ ep->port->prov->remove_address(ep->addr_info[i].
+ prov_addr_context);
+ }
+
+ if (ep->prov_ep_context)
+ ep->port->prov->close_endpoint(ep->prov_ep_context);
+
+ free(ep);
+}
+
+static struct acmc_ep *
+acm_alloc_ep(struct acmc_port *port, uint16_t pkey)
{
- struct acm_ep *ep;
+ struct acmc_ep *ep;
+ int i;
acm_log(1, "\n");
ep = calloc(1, sizeof *ep);
@@ -3434,140 +1720,137 @@ acm_alloc_ep(struct acm_port *port, uint16_t pkey, uint16_t pkey_index)
return NULL;
ep->port = port;
- ep->pkey = pkey;
- ep->pkey_index = pkey_index;
- ep->resolve_queue.credits = resolve_depth;
- ep->sa_queue.credits = sa_depth;
- ep->resp_queue.credits = send_depth;
- DListInit(&ep->resolve_queue.pending);
- DListInit(&ep->sa_queue.pending);
- DListInit(&ep->resp_queue.pending);
- DListInit(&ep->active_queue);
- DListInit(&ep->wait_queue);
- lock_init(&ep->lock);
+ ep->endpoint.port = &port->port;
+ ep->endpoint.pkey = pkey;
+
+ for (i = 0; i < MAX_EP_ADDR; i++) {
+ ep->addr_info[i].addr.endpoint = &ep->endpoint;
+ ep->addr_info[i].addr.id_string = ep->addr_info[i].string_buf;
+ }
return ep;
}
-static void acm_ep_up(struct acm_port *port, uint16_t pkey_index)
+static void acm_ep_up(struct acmc_port *port, uint16_t pkey)
{
- struct acm_ep *ep;
- struct ibv_qp_init_attr init_attr;
- struct ibv_qp_attr attr;
- int ret, sq_size;
- uint16_t pkey;
+ struct acmc_ep *ep;
+ int ret;
acm_log(1, "\n");
- ret = ibv_query_pkey(port->dev->verbs, port->port_num, pkey_index, &pkey);
- if (ret)
- return;
-
- pkey = ntohs(pkey); /* ibv_query_pkey returns pkey in network order */
if (acm_find_ep(port, pkey)) {
acm_log(2, "endpoint for pkey 0x%x already exists\n", pkey);
return;
}
acm_log(2, "creating endpoint for pkey 0x%x\n", pkey);
- ep = acm_alloc_ep(port, pkey, pkey_index);
+ ep = acm_alloc_ep(port, pkey);
if (!ep)
return;
ret = acm_assign_ep_names(ep);
if (ret) {
acm_log(0, "ERROR - unable to assign EP name for pkey 0x%x\n", pkey);
- goto err0;
+ goto ep_close;
}
- sq_size = resolve_depth + sa_depth + send_depth;
- ep->cq = ibv_create_cq(port->dev->verbs, sq_size + recv_depth,
- ep, port->dev->channel, 0);
- if (!ep->cq) {
- acm_log(0, "ERROR - failed to create CQ\n");
- goto err0;
- }
+ DListInsertHead(&ep->entry, &port->ep_list);
+ return;
- ret = ibv_req_notify_cq(ep->cq, 0);
- if (ret) {
- acm_log(0, "ERROR - failed to arm CQ\n");
- goto err1;
- }
+ep_close:
+ if (ep->prov_ep_context)
+ port->prov->close_endpoint(ep->prov_ep_context);
- memset(&init_attr, 0, sizeof init_attr);
- init_attr.cap.max_send_wr = sq_size;
- init_attr.cap.max_recv_wr = recv_depth;
- init_attr.cap.max_send_sge = 1;
- init_attr.cap.max_recv_sge = 1;
- init_attr.qp_context = ep;
- init_attr.sq_sig_all = 1;
- init_attr.qp_type = IBV_QPT_UD;
- init_attr.send_cq = ep->cq;
- init_attr.recv_cq = ep->cq;
- ep->qp = ibv_create_qp(ep->port->dev->pd, &init_attr);
- if (!ep->qp) {
- acm_log(0, "ERROR - failed to create QP\n");
- goto err1;
- }
+ free(ep);
+}
- attr.qp_state = IBV_QPS_INIT;
- attr.port_num = port->port_num;
- attr.pkey_index = pkey_index;
- attr.qkey = ACM_QKEY;
- ret = ibv_modify_qp(ep->qp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX |
- IBV_QP_PORT | IBV_QP_QKEY);
- if (ret) {
- acm_log(0, "ERROR - failed to modify QP to init\n");
- goto err2;
+static void acm_assign_provider(struct acmc_port *port)
+{
+ DLIST_ENTRY *entry;
+ struct acmc_prov *prov;
+ DLIST_ENTRY *sub_entry;
+ struct acmc_subnet *subnet;
+
+ acm_log(2, "port %s/%d\n", port->port.dev->verbs->device->name,
+ port->port.port_num);
+ for (entry = provider_list.Next; entry != &provider_list;
+ entry = entry->Next) {
+ prov = container_of(entry, struct acmc_prov, entry);
+
+ for (sub_entry = prov->subnet_list.Next;
+ sub_entry != &prov->subnet_list;
+ sub_entry = sub_entry->Next) {
+ subnet = container_of(sub_entry, struct acmc_subnet,
+ entry);
+ if (subnet->subnet_prefix ==
+ port->gid_tbl[0].global.subnet_prefix) {
+ acm_log(2, "Found provider %s for port %s/%d\n",
+ prov->prov->name,
+ port->port.dev->verbs->device->name,
+ port->port.port_num);
+ port->prov = prov->prov;
+ return;
+ }
+ }
}
- attr.qp_state = IBV_QPS_RTR;
- ret = ibv_modify_qp(ep->qp, &attr, IBV_QP_STATE);
- if (ret) {
- acm_log(0, "ERROR - failed to modify QP to rtr\n");
- goto err2;
- }
+ /* If no provider is found, assign the default provider*/
+ if (!port->prov) {
+ acm_log(2, "No prov found, assign default prov %s to %s/%d\n",
+ def_provider ? def_provider->prov->name: "NULL",
+ port->port.dev->verbs->device->name,
+ port->port.port_num);
+ port->prov = def_provider ? def_provider->prov : NULL;
+ }
+}
- attr.qp_state = IBV_QPS_RTS;
- attr.sq_psn = 0;
- ret = ibv_modify_qp(ep->qp, &attr, IBV_QP_STATE | IBV_QP_SQ_PSN);
- if (ret) {
- acm_log(0, "ERROR - failed to modify QP to rts\n");
- goto err2;
+static void acm_port_get_gid_tbl(struct acmc_port *port)
+{
+ union ibv_gid gid;
+ int i, j, ret;
+
+ for (i = 0;; i++) {
+ ret = ibv_query_gid(port->port.dev->verbs, port->port.port_num,
+ i, &gid);
+ if (ret || !gid.global.interface_id)
+ break;
}
- ret = acm_post_recvs(ep);
- if (ret)
- goto err2;
+ if (i > 0) {
+ port->gid_tbl = calloc(i, sizeof(union ibv_gid));
+ if (!port->gid_tbl) {
+ acm_log(0, "Error: failed to allocate gid table\n");
+ port->gid_cnt = 0;
+ return;
+ }
- ret = acm_init_ep_loopback(ep);
- if (ret) {
- acm_log(0, "ERROR - unable to init loopback\n");
- goto err2;
+ for (j = 0; j < i; j++) {
+ ret = ibv_query_gid(port->port.dev->verbs,
+ port->port.port_num, j,
+ &port->gid_tbl[j]);
+ if (ret || !port->gid_tbl[j].global.interface_id)
+ break;
+ acm_log(2, "guid %d: 0x%llx %llx\n", j,
+ port->gid_tbl[j].global.subnet_prefix,
+ port->gid_tbl[j].global.interface_id);
+ }
+ port->gid_cnt = j;
}
- lock_acquire(&port->lock);
- DListInsertHead(&ep->entry, &port->ep_list);
- lock_release(&port->lock);
- acm_ep_preload(ep);
- return;
-
-err2:
- ibv_destroy_qp(ep->qp);
-err1:
- ibv_destroy_cq(ep->cq);
-err0:
- free(ep);
+ acm_log(2, "port %d gid_cnt %d\n", port->port.port_num,
+ port->gid_cnt);
}
-static void acm_port_up(struct acm_port *port)
+static void acm_port_up(struct acmc_port *port)
{
struct ibv_port_attr attr;
- union ibv_gid gid;
uint16_t pkey;
int i, ret;
- int is_full_default_pkey_set = 0;
+ struct acmc_prov_context *dev_ctx;
+ int index = -1;
- acm_log(1, "%s %d\n", port->dev->verbs->device->name, port->port_num);
- ret = ibv_query_port(port->dev->verbs, port->port_num, &attr);
+ acm_log(1, "%s %d\n", port->dev->device.verbs->device->name,
+ port->port.port_num);
+ ret = ibv_query_port(port->dev->device.verbs, port->port.port_num,
+ &attr);
if (ret) {
acm_log(0, "ERROR - unable to get port state\n");
return;
@@ -3577,176 +1860,232 @@ static void acm_port_up(struct acm_port *port)
return;
}
- port->mtu = attr.active_mtu;
- port->rate = acm_get_rate(attr.active_width, attr.active_speed);
- if (attr.subnet_timeout >= 8)
- port->subnet_timeout = 1 << (attr.subnet_timeout - 8);
- for (port->gid_cnt = 0;; port->gid_cnt++) {
- ret = ibv_query_gid(port->dev->verbs, port->port_num, port->gid_cnt, &gid);
- if (ret || !gid.global.interface_id)
- break;
- }
-
+ acm_port_get_gid_tbl(port);
port->lid = attr.lid;
port->lid_mask = 0xffff - ((1 << attr.lmc) - 1);
-
- port->sa_dest.av.src_path_bits = 0;
- port->sa_dest.av.dlid = attr.sm_lid;
- port->sa_dest.av.sl = attr.sm_sl;
- port->sa_dest.av.port_num = port->port_num;
- port->sa_dest.remote_qpn = 1;
- attr.sm_lid = htons(attr.sm_lid);
- acm_set_dest_addr(&port->sa_dest, ACM_ADDRESS_LID,
- (uint8_t *) &attr.sm_lid, sizeof(attr.sm_lid));
-
- port->sa_dest.ah = ibv_create_ah(port->dev->pd, &port->sa_dest.av);
- if (!port->sa_dest.ah)
+ port->sa_addr.lid = attr.sm_lid;
+ port->sa_addr.sl = attr.sm_sl;
+ port->state = IBV_PORT_ACTIVE;
+ acm_assign_provider(port);
+ if (!port->prov) {
+ acm_log(1, "no provider assigned to port\n");
return;
+ }
+ dev_ctx = acm_acquire_prov_context(&port->dev->prov_dev_context_list,
+ port->prov);
+ if (!dev_ctx) {
+ acm_log(0, "Error -- failed to acquire dev context\n");
+ return;
+ }
+
+ if (atomic_get(&dev_ctx->refcnt) == 1) {
+ if (port->prov->open_device(&port->dev->device, &dev_ctx->context)) {
+ acm_log(0, "Error -- failed to open the prov device\n");
+ goto err1;
+ }
+ }
+
+ if (port->prov->open_port(&port->port, dev_ctx->context,
+ &port->prov_port_context)) {
+ acm_log(0, "Error -- failed to open the prov port\n");
+ goto err1;
+ }
+
+ /* Determine the default pkey first.
+ Order of preference: 0xffff, 0x7fff, first pkey
+ */
+ for (i = 0; i < attr.pkey_tbl_len; i++) {
+ ret = ibv_query_pkey(port->dev->device.verbs,
+ port->port.port_num, i, &pkey);
+ if (ret)
+ continue;
+ pkey = ntohs(pkey);
+ if (pkey == 0xffff) {
+ index = i;
+ break;
+ }
+ else if (pkey == 0x7fff) {
+ index = i;
+ }
+ }
+ port->default_pkey_index = index < 0 ? 0: index;
- atomic_set(&port->sa_dest.refcnt, 1);
for (i = 0; i < attr.pkey_tbl_len; i++) {
- ret = ibv_query_pkey(port->dev->verbs, port->port_num, i, &pkey);
+ ret = ibv_query_pkey(port->dev->device.verbs,
+ port->port.port_num, i, &pkey);
if (ret)
continue;
pkey = ntohs(pkey);
if (!(pkey & 0x7fff))
continue;
- /* Determine pkey index for default partition with preference for full membership */
- if (!is_full_default_pkey_set && (pkey & 0x7fff) == 0x7fff) {
- port->default_pkey_ix = i;
- if (pkey & 0x8000)
- is_full_default_pkey_set = 1;
+ acm_ep_up(port, pkey);
+ }
+ return;
+err1:
+ acm_release_prov_context(dev_ctx);
+}
+
+static void acm_shutdown_port(struct acmc_port *port)
+{
+ DLIST_ENTRY *entry;
+ struct acmc_ep *ep;
+ struct acmc_prov_context *dev_ctx;
+
+ while (!DListEmpty(&port->ep_list)) {
+ entry = port->ep_list.Next;
+ DListRemove(entry);
+ ep = container_of(entry, struct acmc_ep, entry);
+ acm_ep_down(ep);
+ }
+
+ if (port->prov_port_context) {
+ port->prov->close_port(port->prov_port_context);
+ port->prov_port_context = NULL;
+ dev_ctx = acm_get_prov_context(&port->dev->prov_dev_context_list,
+ port->prov);
+ if (dev_ctx) {
+ if (atomic_get(&dev_ctx->refcnt) == 1)
+ port->prov->close_device(dev_ctx->context);
+ acm_release_prov_context(dev_ctx);
}
- acm_ep_up(port, (uint16_t) i);
}
-
- acm_port_join(port);
- port->state = IBV_PORT_ACTIVE;
- acm_log(1, "%s %d is up\n", port->dev->verbs->device->name, port->port_num);
+ port->prov = NULL;
+ if (port->gid_tbl) {
+ free(port->gid_tbl);
+ port->gid_tbl = NULL;
+ }
+ port->gid_cnt = 0;
}
-static void acm_port_down(struct acm_port *port)
+static void acm_port_down(struct acmc_port *port)
{
struct ibv_port_attr attr;
int ret;
- acm_log(1, "%s %d\n", port->dev->verbs->device->name, port->port_num);
- ret = ibv_query_port(port->dev->verbs, port->port_num, &attr);
+ acm_log(1, "%s %d\n", port->port.dev->verbs->device->name, port->port.port_num);
+ ret = ibv_query_port(port->port.dev->verbs, port->port.port_num, &attr);
if (!ret && attr.state == IBV_PORT_ACTIVE) {
acm_log(1, "port active\n");
return;
}
port->state = attr.state;
+ acm_shutdown_port(port);
- /*
- * We wait for the SA destination to be released. We could use an
- * event instead of a sleep loop, but it's not worth it given how
- * infrequently we should be processing a port down event in practice.
- */
- atomic_dec(&port->sa_dest.refcnt);
- while (atomic_get(&port->sa_dest.refcnt))
- sleep(0);
- ibv_destroy_ah(port->sa_dest.ah);
- acm_log(1, "%s %d is down\n", port->dev->verbs->device->name, port->port_num);
+ acm_log(1, "%s %d is down\n", port->dev->device.verbs->device->name,
+ port->port.port_num);
}
-/*
- * There is one event handler thread per device. This is the only thread that
- * modifies the port state or a port endpoint list. Other threads which access
- * those must synchronize against changes accordingly, but this thread only
- * needs to lock when making modifications.
- */
-static void CDECL_FUNC acm_event_handler(void *context)
+static void acm_port_change(struct acmc_port *port)
+{
+ struct ibv_port_attr attr;
+ int ret;
+
+ acm_log(1, "%s %d\n", port->port.dev->verbs->device->name, port->port.port_num);
+ ret = ibv_query_port(port->port.dev->verbs, port->port.port_num, &attr);
+ if (ret || attr.state != IBV_PORT_ACTIVE) {
+ acm_log(1, "port not active: don't care\n");
+ return;
+ }
+
+ port->state = attr.state;
+ acm_shutdown_port(port);
+ acm_port_up(port);
+}
+
+static void acm_event_handler(struct acmc_device *dev)
{
- struct acm_device *dev = (struct acm_device *) context;
struct ibv_async_event event;
int i, ret;
- acm_log(1, "started\n");
- for (i = 0; i < dev->port_cnt; i++) {
- acm_port_up(&dev->port[i]);
- }
+ ret = ibv_get_async_event(dev->device.verbs, &event);
+ if (ret)
+ return;
- for (;;) {
- ret = ibv_get_async_event(dev->verbs, &event);
- if (ret)
- continue;
+ acm_log(2, "processing async event %s for %s\n",
+ ibv_event_type_str(event.event_type),
+ dev->device.verbs->device->name);
+ i = event.element.port_num - 1;
- acm_log(2, "processing async event %s\n",
- ibv_event_type_str(event.event_type));
- i = event.element.port_num - 1;
- switch (event.event_type) {
- case IBV_EVENT_PORT_ACTIVE:
- if (dev->port[i].state != IBV_PORT_ACTIVE)
- acm_port_up(&dev->port[i]);
- break;
- case IBV_EVENT_PORT_ERR:
- if (dev->port[i].state == IBV_PORT_ACTIVE)
- acm_port_down(&dev->port[i]);
- break;
- case IBV_EVENT_CLIENT_REREGISTER:
- if (dev->port[i].state == IBV_PORT_ACTIVE) {
- acm_port_join(&dev->port[i]);
- acm_log(1, "%s %d has reregistered\n",
- dev->verbs->device->name, i + 1);
- }
- break;
- default:
- break;
+ switch (event.event_type) {
+ case IBV_EVENT_PORT_ACTIVE:
+ if (dev->port[i].state != IBV_PORT_ACTIVE)
+ acm_port_up(&dev->port[i]);
+ break;
+ case IBV_EVENT_PORT_ERR:
+ if (dev->port[i].state == IBV_PORT_ACTIVE)
+ acm_port_down(&dev->port[i]);
+ break;
+ case IBV_EVENT_CLIENT_REREGISTER:
+ if ((dev->port[i].state == IBV_PORT_ACTIVE) &&
+ dev->port[i].prov_port_context) {
+ dev->port[i].prov->handle_event(dev->port[i].prov_port_context,
+ event.event_type);
+ acm_log(1, "%s %d has reregistered\n",
+ dev->device.verbs->device->name, i + 1);
}
-
- ibv_ack_async_event(&event);
+ break;
+ case IBV_EVENT_LID_CHANGE:
+ case IBV_EVENT_GID_CHANGE:
+ case IBV_EVENT_PKEY_CHANGE:
+ acm_port_change(&dev->port[i]);
+ break;
+ default:
+ break;
}
+
+ ibv_ack_async_event(&event);
}
static void acm_activate_devices()
{
- struct acm_device *dev;
+ struct acmc_device *dev;
DLIST_ENTRY *dev_entry;
+ int i;
acm_log(1, "\n");
for (dev_entry = dev_list.Next; dev_entry != &dev_list;
dev_entry = dev_entry->Next) {
- dev = container_of(dev_entry, struct acm_device, entry);
- beginthread(acm_event_handler, dev);
- beginthread(acm_comp_handler, dev);
+ dev = container_of(dev_entry, struct acmc_device, entry);
+ for (i = 0; i < dev->port_cnt; i++) {
+ acm_port_up(&dev->port[i]);
+ }
}
}
-static void acm_open_port(struct acm_port *port, struct acm_device *dev, uint8_t port_num)
+static void
+acm_open_port(struct acmc_port *port, struct acmc_device *dev, uint8_t port_num)
{
- acm_log(1, "%s %d\n", dev->verbs->device->name, port_num);
+ acm_log(1, "%s %d\n", dev->device.verbs->device->name, port_num);
port->dev = dev;
- port->port_num = port_num;
+ port->port.dev = &dev->device;
+ port->port.port_num = port_num;
lock_init(&port->lock);
DListInit(&port->ep_list);
- acm_init_dest(&port->sa_dest, ACM_ADDRESS_LID, NULL, 0);
-
- port->mad_portid = umad_open_port(dev->verbs->device->name, port->port_num);
- if (port->mad_portid < 0) {
+ DListInit(&port->sa_pending);
+ DListInit(&port->sa_wait);
+ port->sa_credits = sa.depth;
+ port->sa_addr.qpn = htonl(1);
+ port->sa_addr.qkey = htonl(ACM_QKEY);
+
+ port->mad_portid = umad_open_port(dev->device.verbs->device->name, port_num);
+ if (port->mad_portid < 0)
acm_log(0, "ERROR - unable to open MAD port\n");
- return;
- }
port->mad_agentid = umad_register(port->mad_portid,
- IB_MGMT_CLASS_SA, 1, 1, NULL);
- if (port->mad_agentid < 0) {
+ IB_MGMT_CLASS_SA, 1, 1, NULL);
+ if (port->mad_agentid < 0)
acm_log(0, "ERROR - unable to register MAD client\n");
- goto err;
- }
+ port->prov = NULL;
port->state = IBV_PORT_DOWN;
- return;
-err:
- umad_close_port(port->mad_portid);
}
static void acm_open_dev(struct ibv_device *ibdev)
{
- struct acm_device *dev;
+ struct acmc_device *dev;
struct ibv_device_attr attr;
struct ibv_context *verbs;
size_t size;
@@ -3765,39 +2104,25 @@ static void acm_open_dev(struct ibv_device *ibdev)
goto err1;
}
- size = sizeof(*dev) + sizeof(struct acm_port) * attr.phys_port_cnt;
- dev = (struct acm_device *) calloc(1, size);
+ size = sizeof(*dev) + sizeof(struct acmc_port) * attr.phys_port_cnt;
+ dev = (struct acmc_device *) calloc(1, size);
if (!dev)
goto err1;
- dev->verbs = verbs;
- dev->guid = ibv_get_device_guid(ibdev);
+ dev->device.verbs = verbs;
+ dev->device.dev_guid = ibv_get_device_guid(ibdev);
dev->port_cnt = attr.phys_port_cnt;
+ DListInit(&dev->prov_dev_context_list);
- dev->pd = ibv_alloc_pd(dev->verbs);
- if (!dev->pd) {
- acm_log(0, "ERROR - unable to allocate PD\n");
- goto err2;
- }
-
- dev->channel = ibv_create_comp_channel(dev->verbs);
- if (!dev->channel) {
- acm_log(0, "ERROR - unable to create comp channel\n");
- goto err3;
- }
-
- for (i = 0; i < dev->port_cnt; i++)
+ for (i = 0; i < dev->port_cnt; i++) {
acm_open_port(&dev->port[i], dev, i + 1);
+ }
DListInsertHead(&dev->entry, &dev_list);
acm_log(1, "%s opened\n", ibdev->name);
return;
-err3:
- ibv_dealloc_pd(dev->pd);
-err2:
- free(dev);
err1:
ibv_close_device(verbs);
}
@@ -3827,6 +2152,414 @@ static int acm_open_devices(void)
return 0;
}
+static void acm_load_prov_config(void)
+{
+ FILE *fd;
+ char s[128];
+ char *p, *ptr;
+ char prov_name[ACM_PROV_NAME_SIZE];
+ uint64_t prefix;
+ struct acmc_prov *prov;
+ DLIST_ENTRY *entry;
+ struct acmc_subnet *subnet;
+
+ if (!(fd = fopen(opts_file, "r")))
+ return;
+
+ while (fgets(s, sizeof s, fd)) {
+ if (s[0] == '#')
+ continue;
+
+ /* Ignore blank lines */
+ if (!(p = strtok_r(s, " \n", &ptr)))
+ continue;
+
+ if (strncasecmp(p, "provider", sizeof("provider") - 1))
+ continue;
+
+ p = strtok_r(NULL, " ", &ptr);
+ if (!p)
+ continue;
+
+ strncpy(prov_name, p, sizeof(prov_name));
+ prov_name[sizeof(prov_name) -1] = '\0';
+
+ p = strtok_r(NULL, " ", &ptr);
+ if (!p)
+ continue;
+ if (!strncasecmp(p, "default", sizeof("default") - 1)) {
+ strncpy(def_prov_name, prov_name, sizeof(def_prov_name));
+ def_prov_name[sizeof(def_prov_name) -1] = '\0';
+ acm_log(2, "default provider: %s\n", def_prov_name);
+ continue;
+ }
+ prefix = strtoull(p, NULL, 0);
+ acm_log(2, "provider %s subnet_prefix 0x%llx\n", prov_name,
+ prefix);
+ /* Convert it into network byte order */
+ prefix = htonll(prefix);
+
+ for (entry = provider_list.Next; entry != &provider_list;
+ entry = entry->Next) {
+ prov = container_of(entry, struct acmc_prov, entry);
+ if (!strcasecmp(prov->prov->name, prov_name)) {
+ subnet = calloc(1, sizeof (*subnet));
+ if (!subnet) {
+ acm_log(0, "Error: out of memory\n");
+ return;
+ }
+ subnet->subnet_prefix = prefix;
+ DListInsertTail(&subnet->entry,
+ &prov->subnet_list);
+ }
+ }
+ }
+
+ fclose(fd);
+
+ for (entry = provider_list.Next; entry != &provider_list;
+ entry = entry->Next) {
+ prov = container_of(entry, struct acmc_prov, entry);
+ if (!strcasecmp(prov->prov->name, def_prov_name)) {
+ def_provider = prov;
+ break;
+ }
+ }
+}
+
+static int acm_open_providers(void)
+{
+ DIR *shlib_dir;
+ struct dirent *dent;
+ char file_name[256];
+ struct stat buf;
+ void *handle;
+ struct acmc_prov *prov;
+ struct acm_provider *provider;
+ uint32_t version;
+ char *err_str;
+ int (*query)(struct acm_provider **, uint32_t *);
+
+ acm_log(1, "\n");
+ shlib_dir = opendir(prov_lib_path);
+ if (!shlib_dir) {
+ acm_log(0, "ERROR - could not open provider lib dir: %s\n",
+ prov_lib_path);
+ return -1;
+ }
+
+ while ((dent = readdir(shlib_dir))) {
+ if (!strstr(dent->d_name, ".so"))
+ continue;
+
+ snprintf(file_name, sizeof(file_name), "%s/%s", prov_lib_path,
+ dent->d_name);
+ if (lstat(file_name, &buf)) {
+ acm_log(0, "Error - could not stat: %s\n", file_name);
+ continue;
+ }
+ if (!S_ISREG(buf.st_mode))
+ continue;
+
+ acm_log(2, "Loading provider %s...\n", file_name);
+ if (!(handle = dlopen(file_name, RTLD_LAZY))) {
+ acm_log(0, "Error - could not load provider %s (%s)\n",
+ file_name, dlerror());
+ continue;
+ }
+
+ query = dlsym(handle, "provider_query");
+ if ((err_str = dlerror()) != NULL) {
+ acm_log(0, "Error -provider_query not found in %s (%s)\n",
+ file_name, err_str);
+ dlclose(handle);
+ continue;
+ }
+
+ if (query(&provider, &version)) {
+ acm_log(0, "Error - provider_query failed to %s\n", file_name);
+ dlclose(handle);
+ continue;
+ }
+
+ if (version != ACM_PROV_VERSION ||
+ provider->size != sizeof(struct acm_provider)) {
+ acm_log(0, "Error -unmatched provider version 0x%08x (size %d)"
+ " core 0x%08x (size %d)\n", version, provider->size,
+ ACM_PROV_VERSION, sizeof(struct acm_provider));
+ dlclose(handle);
+ continue;
+ }
+
+ acm_log(1, "Provider %s (%s) loaded\n", provider->name, file_name);
+
+ prov = calloc(1, sizeof(*prov));
+ if (!prov) {
+ acm_log(0, "Error -failed to allocate provider %s\n", file_name);
+ dlclose(handle);
+ continue;
+ }
+
+ prov->prov = provider;
+ prov->handle = handle;
+ DListInit(&prov->subnet_list);
+ DListInsertTail(&prov->entry, &provider_list);
+ if (!strcasecmp(provider->name, def_prov_name))
+ def_provider = prov;
+ }
+
+ closedir(shlib_dir);
+ acm_load_prov_config();
+ return 0;
+}
+
+static void acm_close_providers(void)
+{
+ struct acmc_prov *prov;
+ DLIST_ENTRY *entry;
+ DLIST_ENTRY *sub_entry;
+ struct acmc_subnet *subnet;
+
+ acm_log(1, "\n");
+ def_provider = NULL;
+ while (!DListEmpty(&provider_list)) {
+ entry = provider_list.Next;
+ DListRemove(entry);
+ prov = container_of(entry, struct acmc_prov, entry);
+ while (!DListEmpty(&prov->subnet_list)) {
+ sub_entry = prov->subnet_list.Next;
+ DListRemove(sub_entry);
+ subnet = container_of(sub_entry, struct acmc_subnet,
+ entry);
+ free(subnet);
+ }
+ dlclose(prov->handle);
+ free(prov);
+ }
+}
+
+static int acmc_init_sa_fds(void)
+{
+ struct acmc_device *dev;
+ DLIST_ENTRY *dev_entry;
+ int ret, p, i = 0;
+
+ for (dev_entry = dev_list.Next; dev_entry != &dev_list;
+ dev_entry = dev_entry->Next) {
+ dev = container_of(dev_entry, struct acmc_device, entry);
+ sa.nfds += dev->port_cnt;
+ }
+
+ sa.fds = calloc(sa.nfds, sizeof(*sa.fds));
+ sa.ports = calloc(sa.nfds, sizeof(*sa.ports));
+ if (!sa.fds || !sa.ports)
+ return -ENOMEM;
+
+ for (dev_entry = dev_list.Next; dev_entry != &dev_list;
+ dev_entry = dev_entry->Next) {
+ dev = container_of(dev_entry, struct acmc_device, entry);
+ for (p = 0; p < dev->port_cnt; p++) {
+ sa.fds[i].fd = umad_get_fd(dev->port[p].mad_portid);
+ sa.fds[i].events = POLLIN;
+ ret = fcntl(sa.fds[i].fd, F_SETFL, O_NONBLOCK);
+ if (ret)
+ acm_log(0, "WARNING - umad fd is blocking\n");
+
+ sa.ports[i++] = &dev->port[p];
+ }
+ }
+
+ return 0;
+}
+
+struct acm_sa_mad *
+acm_alloc_sa_mad(const struct acm_endpoint *endpoint, void *context,
+ void (*handler)(struct acm_sa_mad *))
+{
+ struct acmc_sa_req *req;
+
+ if (!endpoint) {
+ acm_log(0, "Error: NULL endpoint\n");
+ return NULL;
+ }
+ req = calloc(1, sizeof (*req));
+ if (!req) {
+ acm_log(0, "Error: failed to allocate sa request\n");
+ return NULL;
+ }
+
+ req->ep = container_of(endpoint, struct acmc_ep, endpoint);
+ req->mad.context = context;
+ req->resp_handler = handler;
+
+ acm_log(2, "%p\n", req);
+ return &req->mad;
+}
+
+void acm_free_sa_mad(struct acm_sa_mad *mad)
+{
+ struct acmc_sa_req *req;
+ req = container_of(mad, struct acmc_sa_req, mad);
+ acm_log(2, "%p\n", req);
+ free(req);
+}
+
+int acm_send_sa_mad(struct acm_sa_mad *mad)
+{
+ struct acmc_port *port;
+ struct acmc_sa_req *req;
+ int ret;
+
+ req = container_of(mad, struct acmc_sa_req, mad);
+ acm_log(2, "%p from %s\n", req, req->ep->addr_info[0].addr.id_string);
+
+ port = req->ep->port;
+ mad->umad.addr.qpn = port->sa_addr.qpn;
+ mad->umad.addr.qkey = port->sa_addr.qkey;
+ mad->umad.addr.lid = htons(port->sa_addr.lid);
+ mad->umad.addr.sl = port->sa_addr.sl;
+ mad->umad.addr.pkey_index = req->ep->port->default_pkey_index;
+
+ lock_acquire(&port->lock);
+ if (port->sa_credits && DListEmpty(&port->sa_wait)) {
+ ret = umad_send(port->mad_portid, port->mad_agentid, &mad->umad,
+ sizeof mad->sa_mad, sa.timeout, sa.retries);
+ if (!ret) {
+ port->sa_credits--;
+ DListInsertTail(&req->entry, &port->sa_pending);
+ }
+ } else {
+ ret = 0;
+ DListInsertTail(&req->entry, &port->sa_wait);
+ }
+ lock_release(&port->lock);
+ return ret;
+}
+
+static void acmc_send_queued_req(struct acmc_port *port)
+{
+ struct acmc_sa_req *req;
+ int ret;
+
+ lock_acquire(&port->lock);
+ if (DListEmpty(&port->sa_wait) || !port->sa_credits) {
+ lock_release(&port->lock);
+ return;
+ }
+
+ req = container_of(port->sa_wait.Next, struct acmc_sa_req, entry);
+ DListRemove(&req->entry);
+ ret = umad_send(port->mad_portid, port->mad_agentid, &req->mad.umad,
+ sizeof req->mad.sa_mad, sa.timeout, sa.retries);
+ if (!ret) {
+ port->sa_credits--;
+ DListInsertTail(&req->entry, &port->sa_pending);
+ }
+ lock_release(&port->lock);
+
+ if (ret) {
+ req->mad.umad.status = -ret;
+ req->resp_handler(&req->mad);
+ }
+}
+
+static void acmc_recv_mad(struct acmc_port *port)
+{
+ struct acmc_sa_req *req;
+ struct acm_sa_mad resp;
+ DLIST_ENTRY *entry;
+ int ret, len, found;
+ struct umad_hdr *hdr;
+
+ acm_log(2, "\n");
+ len = sizeof(resp.sa_mad);
+ ret = umad_recv(port->mad_portid, &resp.umad, &len, 0);
+ if (ret < 0) {
+ acm_log(1, "umad_recv error %d\n", ret);
+ return;
+ }
+
+ hdr = &resp.sa_mad.mad_hdr;
+ acm_log(2, "bv %x cls %x cv %x mtd %x st %d tid %llx at %x atm %x\n",
+ hdr->base_version, hdr->mgmt_class, hdr->class_version,
+ hdr->method, hdr->status, hdr->tid, hdr->attr_id, hdr->attr_mod);
+ found = 0;
+ lock_acquire(&port->lock);
+ for (entry = port->sa_pending.Next; entry != &port->sa_pending;
+ entry = entry->Next) {
+ req = container_of(entry, struct acmc_sa_req, entry);
+ /* The lower 32-bit of the tid is used for agentid in umad */
+ if (req->mad.sa_mad.mad_hdr.tid == (hdr->tid & 0xFFFFFFFF00000000)) {
+ found = 1;
+ DListRemove(entry);
+ port->sa_credits++;
+ break;
+ }
+ }
+ lock_release(&port->lock);
+
+ if (found) {
+ memcpy(&req->mad.umad, &resp.umad, sizeof(resp.umad) + len);
+ req->resp_handler(&req->mad);
+ }
+}
+
+static void *acm_sa_handler(void *context)
+{
+ int i, ret;
+
+ acm_log(0, "started\n");
+ ret = acmc_init_sa_fds();
+ if (ret) {
+ acm_log(0, "ERROR - failed to init fds\n");
+ return NULL;
+ }
+
+ if (pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL)) {
+ acm_log(0, "Error: failed to set cancel type \n");
+ return NULL;
+ }
+
+ if (pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL)) {
+ acm_log(0, "Error: failed to set cancel state\n");
+ return NULL;
+ }
+
+ for (;;) {
+ pthread_testcancel();
+ ret = poll(sa.fds, sa.nfds, -1);
+ if (ret < 0) {
+ acm_log(0, "ERROR - sa poll error: %d\n", errno);
+ continue;
+ }
+
+ for (i = 0; i < sa.nfds; i++) {
+ if (!sa.fds[i].revents)
+ continue;
+
+ if (sa.fds[i].revents & POLLIN) {
+ acmc_recv_mad(sa.ports[i]);
+ acmc_send_queued_req(sa.ports[i]);
+ }
+ sa.fds[i].revents = 0;
+ }
+ }
+ return NULL;
+}
+
+static void acm_stop_sa_handler(void)
+{
+ if (pthread_cancel(sa.thread_id)) {
+ acm_log(0, "Error: failed to cancel sa resp thread \n");
+ return;
+ }
+
+ if (pthread_join(sa.thread_id, NULL)) {
+ acm_log(0, "Error: failed to join sa resp thread\n");
+ return;
+ }
+}
+
static void acm_set_options(void)
{
FILE *f;
@@ -3849,44 +2582,18 @@ static void acm_set_options(void)
log_level = atoi(value);
else if (!stricmp("lock_file", opt))
strcpy(lock_file, value);
- else if (!stricmp("addr_prot", opt))
- addr_prot = acm_convert_addr_prot(value);
- else if (!stricmp("addr_timeout", opt))
- addr_timeout = atoi(value);
- else if (!stricmp("route_prot", opt))
- route_prot = acm_convert_route_prot(value);
- else if (!strcmp("route_timeout", opt))
- route_timeout = atoi(value);
- else if (!stricmp("loopback_prot", opt))
- loopback_prot = acm_convert_loopback_prot(value);
else if (!stricmp("server_port", opt))
server_port = (short) atoi(value);
+ else if (!stricmp("provider_lib_path", opt))
+ strcpy(prov_lib_path, value);
+ else if (!stricmp("support_ips_in_addr_cfg", opt))
+ support_ips_in_addr_cfg = atoi(value);
else if (!stricmp("timeout", opt))
- timeout = atoi(value);
+ sa.timeout = atoi(value);
else if (!stricmp("retries", opt))
- retries = atoi(value);
- else if (!stricmp("resolve_depth", opt))
- resolve_depth = atoi(value);
+ sa.retries = atoi(value);
else if (!stricmp("sa_depth", opt))
- sa_depth = atoi(value);
- else if (!stricmp("send_depth", opt))
- send_depth = atoi(value);
- else if (!stricmp("recv_depth", opt))
- recv_depth = atoi(value);
- else if (!stricmp("min_mtu", opt))
- min_mtu = acm_convert_mtu(atoi(value));
- else if (!stricmp("min_rate", opt))
- min_rate = acm_convert_rate(atoi(value));
- else if (!stricmp("route_preload", opt))
- route_preload = acm_convert_route_preload(value);
- else if (!stricmp("route_data_file", opt))
- strcpy(route_data_file, value);
- else if (!stricmp("addr_preload", opt))
- addr_preload = acm_convert_addr_preload(value);
- else if (!stricmp("addr_data_file", opt))
- strcpy(addr_data_file, value);
- else if (!stricmp("support_ips_in_addr_cfg", opt))
- support_ips_in_addr_cfg = atoi(value);
+ sa.depth = atoi(value);
}
fclose(f);
@@ -3894,26 +2601,16 @@ static void acm_set_options(void)
static void acm_log_options(void)
{
+ acm_log(0, "log file %s\n", log_file);
acm_log(0, "log level %d\n", log_level);
acm_log(0, "lock file %s\n", lock_file);
- acm_log(0, "address resolution %d\n", addr_prot);
- acm_log(0, "address timeout %d\n", addr_timeout);
- acm_log(0, "route resolution %d\n", route_prot);
- acm_log(0, "route timeout %d\n", route_timeout);
- acm_log(0, "loopback resolution %d\n", loopback_prot);
acm_log(0, "server_port %d\n", server_port);
- acm_log(0, "timeout %d ms\n", timeout);
- acm_log(0, "retries %d\n", retries);
- acm_log(0, "resolve depth %d\n", resolve_depth);
- acm_log(0, "sa depth %d\n", sa_depth);
- acm_log(0, "send depth %d\n", send_depth);
- acm_log(0, "receive depth %d\n", recv_depth);
- acm_log(0, "minimum mtu %d\n", min_mtu);
- acm_log(0, "minimum rate %d\n", min_rate);
- acm_log(0, "route preload %d\n", route_preload);
- acm_log(0, "route data file %s\n", route_data_file);
- acm_log(0, "address preload %d\n", addr_preload);
- acm_log(0, "address data file %s\n", addr_data_file);
+ acm_log(0, "timeout %d ms\n", sa.timeout);
+ acm_log(0, "retries %d\n", sa.retries);
+ acm_log(0, "sa depth %d\n", sa.depth);
+ acm_log(0, "options file %s\n", opts_file);
+ acm_log(0, "addr file %s\n", addr_file);
+ acm_log(0, "provider lib path %s\n", prov_lib_path);
acm_log(0, "support IP's in ibacm_addr.cfg %d\n", support_ips_in_addr_cfg);
}
@@ -3948,7 +2645,11 @@ static int acm_open_lock_file(void)
}
snprintf(pid, sizeof pid, "%d\n", getpid());
- write(lock_fd, pid, strlen(pid));
+ if (write(lock_fd, pid, strlen(pid)) != strlen(pid)){
+ lockf(lock_fd, F_ULOCK, 0);
+ close(lock_fd);
+ return -1;
+ }
return 0;
}
@@ -3967,9 +2668,12 @@ static void daemonize(void)
if (chdir("/"))
exit(1);
- freopen("/dev/null", "r", stdin);
- freopen("/dev/null", "w", stdout);
- freopen("/dev/null", "w", stderr);
+ if(!freopen("/dev/null", "r", stdin))
+ exit(1);
+ if(!freopen("/dev/null", "w", stdout))
+ exit(1);
+ if(!freopen("/dev/null", "w", stderr))
+ exit(1);
}
static void show_usage(char *program)
@@ -4023,15 +2727,17 @@ int CDECL_FUNC main(int argc, char **argv)
acm_log(0, "Assistant to the InfiniBand Communication Manager\n");
acm_log_options();
- atomic_init(&tid);
- atomic_init(&wait_cnt);
+ DListInit(&provider_list);
DListInit(&dev_list);
- DListInit(&timeout_list);
- event_init(&timeout_event);
for (i = 0; i < ACM_MAX_COUNTER; i++)
atomic_init(&counter[i]);
umad_init();
+ if (acm_open_providers()) {
+ acm_log(0, "ERROR - unable to open any providers\n");
+ return -1;
+ }
+
if (acm_open_devices()) {
acm_log(0, "ERROR - unable to open any devices\n");
return -1;
@@ -4040,13 +2746,19 @@ int CDECL_FUNC main(int argc, char **argv)
acm_log(1, "creating IP Netlink socket\n");
acm_ipnl_create();
+ acm_log(1, "starting sa response receiving thread\n");
+ if (pthread_create(&sa.thread_id, NULL, acm_sa_handler, NULL)) {
+ acm_log(0, "Error: failed to create sa resp rcving thread");
+ return -1;
+ }
acm_activate_devices();
- acm_log(1, "starting timeout/retry thread\n");
- beginthread(acm_retry_handler, NULL);
acm_log(1, "starting server\n");
acm_server();
acm_log(0, "shutting down\n");
+ acm_close_providers();
+ acm_stop_sa_handler();
+ umad_done();
fclose(flog);
return 0;
}
diff --git a/src/acm_util.c b/src/acm_util.c
index da6d058..d54f520 100644
--- a/src/acm_util.c
+++ b/src/acm_util.c
@@ -127,7 +127,7 @@ int acm_if_iter_sys(acm_if_iter_cb cb, void *ctx)
{
struct ifconf *ifc;
struct ifreq *ifr;
- char ip[INET6_ADDRSTRLEN];
+ char ip_str[INET6_ADDRSTRLEN];
int s, ret, i, len;
uint16_t pkey;
union ibv_gid sgid;
@@ -148,7 +148,7 @@ int acm_if_iter_sys(acm_if_iter_cb cb, void *ctx)
}
memset(ifc, 0, len);
- ifc->ifc_len = len;
+ ifc->ifc_len = len - sizeof(*ifc);
ifc->ifc_req = (struct ifreq *) (ifc + 1);
ret = ioctl(s, SIOCGIFCONF, ifc);
@@ -162,17 +162,21 @@ int acm_if_iter_sys(acm_if_iter_cb cb, void *ctx)
switch (ifr[i].ifr_addr.sa_family) {
case AF_INET:
addr_type = ACM_ADDRESS_IP;
- memcpy(&addr, &((struct sockaddr_in *) &ifr[i].ifr_addr)->sin_addr, sizeof addr);
+ memcpy(&addr, &((struct sockaddr_in *) &ifr[i].ifr_addr)->sin_addr,
+ sizeof addr);
addr_len = 4;
inet_ntop(ifr[i].ifr_addr.sa_family,
- &((struct sockaddr_in *) &ifr[i].ifr_addr)->sin_addr, ip, sizeof ip);
+ &((struct sockaddr_in *) &ifr[i].ifr_addr)->sin_addr,
+ ip_str, sizeof ip_str);
break;
case AF_INET6:
addr_type = ACM_ADDRESS_IP6;
- memcpy(&addr, &((struct sockaddr_in6 *) &ifr[i].ifr_addr)->sin6_addr, sizeof addr);
+ memcpy(&addr, &((struct sockaddr_in6 *) &ifr[i].ifr_addr)->sin6_addr,
+ sizeof addr);
addr_len = ACM_MAX_ADDRESS;
inet_ntop(ifr[i].ifr_addr.sa_family,
- &((struct sockaddr_in6 *) &ifr[i].ifr_addr)->sin6_addr, ip, sizeof ip);
+ &((struct sockaddr_in6 *) &ifr[i].ifr_addr)->sin6_addr,
+ ip_str, sizeof ip_str);
break;
default:
continue;
@@ -195,7 +199,7 @@ int acm_if_iter_sys(acm_if_iter_cb cb, void *ctx)
if (ret)
continue;
- cb(ifr[i].ifr_name, &sgid, pkey, addr_type, addr, addr_len, ip, ctx);
+ cb(ifr[i].ifr_name, &sgid, pkey, addr_type, addr, addr_len, ip_str, ctx);
}
ret = 0;
diff --git a/src/acm_util.h b/src/acm_util.h
index 4f111b6..2bcf280 100644
--- a/src/acm_util.h
+++ b/src/acm_util.h
@@ -50,7 +50,7 @@ int acm_if_get_sgid(char *ifname, union ibv_gid *sgid);
typedef void (*acm_if_iter_cb)(char *ifname, union ibv_gid *gid, uint16_t pkey,
uint8_t addr_type, uint8_t *addr, size_t addr_len,
- char *addr_name, void *ctx);
+ char *ip_str, void *ctx);
int acm_if_iter_sys(acm_if_iter_cb cb, void *ctx);
#endif /* ACM_IF_H */
diff --git a/src/acme.c b/src/acme.c
index c93ee1b..f1b0d01 100644
--- a/src/acme.c
+++ b/src/acme.c
@@ -58,11 +58,16 @@ static char addr_type = 'u';
static int verify;
static int nodelay;
static int repetitions = 1;
+static int ep_index;
+static int enum_ep;
enum perf_query_output {
PERF_QUERY_NONE,
PERF_QUERY_ROW,
- PERF_QUERY_COL
+ PERF_QUERY_COL,
+ PERF_QUERY_EP_INDEX,
+ PERF_QUERY_EP_ALL,
+ PERF_QUERY_EP_ADDR
};
static enum perf_query_output perf_query;
int verbose;
@@ -79,13 +84,22 @@ static void show_usage(char *program)
{
printf("usage 1: %s\n", program);
printf("Query specified ibacm service for data\n");
+ printf(" [-e [N]] - display one or all endpoints:\n");
+ printf(" No index: all endpoints\n");
+ printf(" N: endpoint N (N = 1, 2, ...)\n");
printf(" [-f addr_format] - i(p), n(ame), l(id), g(gid), or u(nspecified)\n");
printf(" address format for -s and -d options, default: 'u'\n");
printf(" [-s src_addr] - source address for path queries\n");
printf(" [-d dest_addr] - destination addresses for path queries\n");
printf(" [-v] - verify ACM response against SA query response\n");
printf(" [-c] - read ACM cached data only\n");
- printf(" [-P] - query performance data from destination service\n");
+ printf(" [-P [opt]] - query performance data from destination service:\n");
+ printf(" No option: output combined data in row format.\n");
+ printf(" col: output combined data in colum format.\n");
+ printf(" N: output data for endpoint N (N = 1, 2,...)\n");
+ printf(" all: output data for all endpoints\n");
+ printf(" s: output data for the endpoint with the\n");
+ printf(" address specified in -s option\n");
printf(" [-S svc_addr] - address of ACM service, default: local service\n");
printf(" [-C repetitions] - repeat count for resolution\n");
printf("usage 2: %s\n", program);
@@ -261,7 +275,7 @@ static void gen_opts_temp(FILE *f)
fprintf(f, "# Specifies the location of the route data file to use when preloading\n");
fprintf(f, "# the ACM cache. This option is only valid if route_preload\n");
fprintf(f, "# indicates that routing data should be read from a file.\n");
- fprintf(f, "# Default is ACM_CONF_DIR/ibacm_route.data\n");
+ fprintf(f, "# Default is %s/ibacm_route.data\n", ACM_CONF_DIR);
fprintf(f, "# route_data_file /etc/rdma/ibacm_route.data\n");
fprintf(f, "\n");
fprintf(f, "# addr_preload:\n");
@@ -277,13 +291,28 @@ static void gen_opts_temp(FILE *f)
fprintf(f, "# Specifies the location of the address data file to use when preloading\n");
fprintf(f, "# the ACM cache. This option is only valid if addr_preload\n");
fprintf(f, "# indicates that address data should be read from a file.\n");
- fprintf(f, "# Default is ACM_CONF_DIR/ibacm_hosts.data\n");
+ fprintf(f, "# Default is %s/ibacm_hosts.data\n", ACM_CONF_DIR);
fprintf(f, "# addr_data_file /etc/rdma/ibacm_hosts.data\n");
fprintf(f, "\n");
fprintf(f, "# support_ips_in_addr_cfg:\n");
fprintf(f, "# If 1 continue to read IP addresses from ibacm_addr.cfg\n");
fprintf(f, "# Default is 0 \"no\"\n");
fprintf(f, "# support_ips_in_addr_cfg 0\n");
+ fprintf(f, "\n");
+ fprintf(f, "# provider_lib_path:\n");
+ fprintf(f, "# Specifies the directory of the provider libraries\n");
+ fprintf(f, "\n");
+ fprintf(f, "# provider_lib_path %s\n", IBACM_LIB_PATH);
+ fprintf(f, "\n");
+ fprintf(f, "# provider:\n");
+ fprintf(f, "# Specifies the provider to assign to each subnet\n");
+ fprintf(f, "# ACM providers may override the address and route resolution\n");
+ fprintf(f, "# protocols with provider specific protocols.\n");
+ fprintf(f, "# provider name (prefix | default)\n");
+ fprintf(f, "# Example:\n");
+ fprintf(f, "# provider ibacmp 0xFE80000000000000\n");
+ fprintf(f, "# provider ibacmp default\n");
+ fprintf(f, "\n");
}
static int open_dir(void)
@@ -506,7 +535,7 @@ static int inet_any_pton(char *addr, struct sockaddr *sa)
if (ret <= 0) {
sin6 = (struct sockaddr_in6 *) sa;
sa->sa_family = AF_INET6;
- ret = inet_pton(AF_INET6, src_addr, &sin6->sin6_addr);
+ ret = inet_pton(AF_INET6, addr, &sin6->sin6_addr);
}
return ret;
@@ -728,24 +757,107 @@ static void resolve(char *svc)
free(dest_list);
}
-static void query_perf(char *svc)
+static int query_perf_ip(uint64_t **counters, int *cnt)
{
- static int lables;
+ union _sockaddr {
+ struct sockaddr_storage src;
+ struct sockaddr saddr;
+ } addr;
+ uint8_t type;
+ struct sockaddr_in *sin;
+ struct sockaddr_in6 *sin6;
+ int ret;
+
+ VPRINT("%s: src_addr %s\n", __FUNCTION__, src_addr);
+ addr.saddr.sa_family = AF_INET;
+ sin = (struct sockaddr_in *) &addr.saddr;
+ ret = inet_pton(AF_INET, src_addr, &sin->sin_addr);
+ if (ret <= 0) {
+ addr.saddr.sa_family = AF_INET6;
+ sin6 = (struct sockaddr_in6 *)&addr.saddr;
+ ret = inet_pton(AF_INET6, src_addr, &sin6->sin6_addr);
+ if (ret <= 0) {
+ printf("inet_pton error on src address (%s): 0x%x\n",
+ src_addr, ret);
+ return -1;
+ }
+ type = ACM_EP_INFO_ADDRESS_IP6;
+ } else {
+ type = ACM_EP_INFO_ADDRESS_IP;
+ }
+
+ ret = ib_acm_query_perf_ep_addr((uint8_t *)&addr.src, type, counters,
+ cnt);
+ if (ret) {
+ printf("ib_acm_query_perf failed: %s\n", strerror(errno));
+ return ret;
+ }
+
+ return 0;
+}
+
+static int query_perf_name(uint64_t **counters, int *cnt)
+{
+ int ret;
+
+ VPRINT("%s: src_addr %s\n", __FUNCTION__, src_addr);
+ ret = ib_acm_query_perf_ep_addr((uint8_t *)src_addr, ACM_EP_INFO_NAME,
+ counters, cnt);
+ if (ret) {
+ printf("ib_acm_query_perf failed: %s\n", strerror(errno));
+ return ret;
+ }
+
+ return 0;
+}
+
+static int query_perf_ep_addr(uint64_t **counters, int *cnt)
+{
+ int ret;
+ char src_type;
+
+ src_addr = get_dest(src_arg, &src_type);
+ switch (src_type) {
+ case 'i':
+ ret = query_perf_ip(counters, cnt);
+ break;
+ case 'n':
+ ret = query_perf_name(counters, cnt);
+ break;
+ default:
+ printf("Unsupported src_type %d\n", src_type);
+ return -1;
+ }
+
+ return ret;
+}
+
+static int query_perf_one(char *svc, int index)
+{
+ static int labels;
int ret, cnt, i;
uint64_t *counters;
- ret = ib_acm_query_perf(&counters, &cnt);
+ if (perf_query == PERF_QUERY_EP_ADDR)
+ ret = query_perf_ep_addr(&counters, &cnt);
+ else
+ ret = ib_acm_query_perf(index, &counters, &cnt);
+
if (ret) {
- printf("%s: Failed to query perf data %s\n", svc, strerror(errno));
- return;
+ if (perf_query != PERF_QUERY_EP_ALL) {
+ printf("%s: Failed to query perf data: %s\n", svc,
+ strerror(errno));
+ }
+ return ret;
}
- if (perf_query == PERF_QUERY_ROW) {
- if (!lables) {
+ if (perf_query != PERF_QUERY_COL) {
+ if (!labels) {
+ printf("svc,");
for (i = 0; i < cnt - 1; i++)
printf("%s,", ib_acm_cntr_name(i));
printf("%s\n", ib_acm_cntr_name(i));
- lables = 1;
+ labels = 1;
}
printf("%s,", svc);
for (i = 0; i < cnt - 1; i++)
@@ -759,6 +871,57 @@ static void query_perf(char *svc)
}
}
ib_acm_free_perf(counters);
+
+ return 0;
+}
+
+static void query_perf(char *svc)
+{
+ int index = 1;
+
+ if (perf_query != PERF_QUERY_EP_ALL) {
+ query_perf_one(svc, ep_index);
+ }
+ else {
+ while (!query_perf_one(svc, index++));
+ }
+}
+
+static int enumerate_ep(char *svc, int index)
+{
+ static int labels;
+ int ret, i;
+ struct acm_ep_config_data *ep_data;
+
+ ret = ib_acm_enum_ep(index, &ep_data);
+ if (ret)
+ return ret;
+
+ if (!labels) {
+ printf("svc,guid,port,pkey,ep_index,prov,addr_0,addresses\n");
+ labels = 1;
+ }
+
+ printf("%s,0x%016lx,%d,0x%04x,%d,%s", svc, ep_data->dev_guid,
+ ep_data->port_num, ep_data->pkey, index, ep_data->prov_name);
+ for (i = 0; i < ep_data->addr_cnt; i++)
+ printf(",%s", ep_data->addrs[i].name);
+ printf("\n");
+ ib_acm_free_ep_data(ep_data);
+
+ return 0;
+}
+
+static void enumerate_eps(char *svc)
+{
+ int index = 1;
+
+ if (ep_index > 0) {
+ if (enumerate_ep(svc, ep_index))
+ printf(" Endpoint %d is not available\n", ep_index);
+ } else {
+ while (!enumerate_ep(svc, index++));
+ }
}
static int query_svcs(void)
@@ -786,6 +949,9 @@ static int query_svcs(void)
if (perf_query)
query_perf(svc_list[i]);
+ if (enum_ep)
+ enumerate_eps(svc_list[i]);
+
ib_acm_disconnect();
}
@@ -804,6 +970,23 @@ char *opt_arg(int argc, char **argv)
return NULL;
}
+void parse_perf_arg(char *arg)
+{
+ if (!strnicmp("col", arg, 3)) {
+ perf_query = PERF_QUERY_COL;
+ } else if (!strnicmp("all", arg, 3)) {
+ perf_query = PERF_QUERY_EP_ALL;
+ } else if (!strcmp("s", arg)) {
+ perf_query = PERF_QUERY_EP_ADDR;
+ } else {
+ ep_index = atoi(arg);
+ if (ep_index > 0)
+ perf_query = PERF_QUERY_EP_INDEX;
+ else
+ perf_query = PERF_QUERY_ROW;
+ }
+}
+
int CDECL_FUNC main(int argc, char **argv)
{
int op, ret;
@@ -814,8 +997,13 @@ int CDECL_FUNC main(int argc, char **argv)
if (ret)
goto out;
- while ((op = getopt(argc, argv, "f:s:d:vcA::O::D:P::S:C:V")) != -1) {
+ while ((op = getopt(argc, argv, "e::f:s:d:vcA::O::D:P::S:C:V")) != -1) {
switch (op) {
+ case 'e':
+ enum_ep = 1;
+ if (opt_arg(argc, argv))
+ ep_index = atoi(opt_arg(argc, argv));
+ break;
case 'f':
addr_type = optarg[0];
if (addr_type != 'i' && addr_type != 'n' &&
@@ -848,8 +1036,8 @@ int CDECL_FUNC main(int argc, char **argv)
dest_dir = optarg;
break;
case 'P':
- if (opt_arg(argc, argv) && !strnicmp("col", opt_arg(argc, argv), 3))
- perf_query = PERF_QUERY_COL;
+ if (opt_arg(argc, argv))
+ parse_perf_arg(opt_arg(argc, argv));
else
perf_query = PERF_QUERY_ROW;
break;
@@ -869,11 +1057,13 @@ int CDECL_FUNC main(int argc, char **argv)
}
}
- if ((src_arg && !dest_arg) ||
- (!src_arg && !dest_arg && !perf_query && !make_addr && !make_opts))
+ if ((src_arg && (!dest_arg && perf_query != PERF_QUERY_EP_ADDR)) ||
+ (perf_query == PERF_QUERY_EP_ADDR && !src_arg) ||
+ (!src_arg && !dest_arg && !perf_query && !make_addr && !make_opts &&
+ !enum_ep))
goto show_use;
- if (dest_arg || perf_query)
+ if (dest_arg || perf_query || enum_ep)
ret = query_svcs();
if (!ret && make_addr)
diff --git a/src/libacm.c b/src/libacm.c
index 4e42233..95e562d 100644
--- a/src/libacm.c
+++ b/src/libacm.c
@@ -49,7 +49,8 @@ static void acm_set_server_port(void)
FILE *f;
if ((f = fopen("/var/run/ibacm.port", "r"))) {
- fscanf(f, "%hu", (unsigned short *) &server_port);
+ if (fscanf(f, "%hu", (unsigned short *) &server_port) != 1)
+ printf("Failed to read server port\n");
fclose(f);
}
}
@@ -304,7 +305,7 @@ out:
return ret;
}
-int ib_acm_query_perf(uint64_t **counters, int *count)
+int ib_acm_query_perf(int index, uint64_t **counters, int *count)
{
struct acm_msg msg;
int ret, i;
@@ -313,6 +314,7 @@ int ib_acm_query_perf(uint64_t **counters, int *count)
memset(&msg, 0, sizeof msg);
msg.hdr.version = ACM_VERSION;
msg.hdr.opcode = ACM_OP_PERF_QUERY;
+ msg.hdr.data[1] = index;
msg.hdr.length = htons(ACM_MSG_HDR_LENGTH);
ret = send(sock, (char *) &msg, ACM_MSG_HDR_LENGTH, 0);
@@ -345,6 +347,110 @@ out:
return ret;
}
+int ib_acm_enum_ep(int index, struct acm_ep_config_data **data)
+{
+ struct acm_msg msg;
+ int ret;
+ int len;
+ int cnt;
+ struct acm_ep_config_data *edata;
+
+ lock_acquire(&lock);
+ memset(&msg, 0, sizeof msg);
+ msg.hdr.version = ACM_VERSION;
+ msg.hdr.opcode = ACM_OP_EP_QUERY;
+ msg.hdr.data[0] = index;
+ msg.hdr.length = htons(ACM_MSG_HDR_LENGTH);
+
+ ret = send(sock, (char *) &msg, ACM_MSG_HDR_LENGTH, 0);
+ if (ret != ACM_MSG_HDR_LENGTH)
+ goto out;
+
+ ret = recv(sock, (char *) &msg, sizeof msg, 0);
+ if (ret < ACM_MSG_HDR_LENGTH || ret != ntohs(msg.hdr.length)) {
+ ret = ACM_STATUS_EINVAL;
+ goto out;
+ }
+
+ if (msg.hdr.status) {
+ ret = acm_error(msg.hdr.status);
+ goto out;
+ }
+
+ cnt = ntohs(msg.ep_data[0].addr_cnt);
+ len = sizeof(struct acm_ep_config_data) +
+ ACM_MAX_ADDRESS * cnt;
+ edata = malloc(len);
+ if (!edata) {
+ ret = ACM_STATUS_ENOMEM;
+ goto out;
+ }
+
+ memcpy(edata, &msg.ep_data[0], len);
+ edata->dev_guid = ntohll(msg.ep_data[0].dev_guid);
+ edata->pkey = ntohs(msg.ep_data[0].pkey);
+ edata->addr_cnt = cnt;
+ *data = edata;
+ ret = 0;
+out:
+ lock_release(&lock);
+ return ret;
+}
+
+int ib_acm_query_perf_ep_addr(uint8_t *src, uint8_t type,
+ uint64_t **counters, int *count)
+{
+ struct acm_msg msg;
+ int ret, i, len;
+
+ if (!src)
+ return -1;
+
+ lock_acquire(&lock);
+ memset(&msg, 0, sizeof msg);
+ msg.hdr.version = ACM_VERSION;
+ msg.hdr.opcode = ACM_OP_PERF_QUERY;
+
+ ret = acm_format_ep_addr(&msg.resolve_data[0], src, type,
+ ACM_EP_FLAG_SOURCE);
+ if (ret)
+ goto out;
+
+ len = ACM_MSG_HDR_LENGTH + ACM_MSG_EP_LENGTH;
+ msg.hdr.length = htons(len);
+
+ ret = send(sock, (char *) &msg, len, 0);
+ if (ret != len)
+ goto out;
+
+ ret = recv(sock, (char *) &msg, sizeof msg, 0);
+ if (ret < ACM_MSG_HDR_LENGTH || ret != ntohs(msg.hdr.length)) {
+ ret = ACM_STATUS_EINVAL;
+ goto out;
+ }
+
+ if (msg.hdr.status) {
+ ret = acm_error(msg.hdr.status);
+ goto out;
+ }
+
+ *counters = malloc(sizeof(uint64_t) * msg.hdr.data[0]);
+ if (!*counters) {
+ ret = ACM_STATUS_ENOMEM;
+ goto out;
+ }
+
+ *count = msg.hdr.data[0];
+ for (i = 0; i < *count; i++)
+ (*counters)[i] = ntohll(msg.perf_data[i]);
+
+ ret = 0;
+out:
+ lock_release(&lock);
+ return ret;
+}
+
+
const char *ib_acm_cntr_name(int index)
{
static const char *const cntr_name[] = {
diff --git a/src/libacm.h b/src/libacm.h
index 9a241aa..359a6af 100644
--- a/src/libacm.h
+++ b/src/libacm.h
@@ -45,9 +45,14 @@ int ib_acm_resolve_ip(struct sockaddr *src, struct sockaddr *dest,
int ib_acm_resolve_path(struct ibv_path_record *path, uint32_t flags);
#define ib_acm_free_paths(paths) free(paths)
-int ib_acm_query_perf(uint64_t **counters, int *count);
+int ib_acm_query_perf(int index, uint64_t **counters, int *count);
+int ib_acm_query_perf_ep_addr(uint8_t *src, uint8_t type,
+ uint64_t **counters, int *count);
#define ib_acm_free_perf(counters) free(counters)
const char *ib_acm_cntr_name(int index);
+int ib_acm_enum_ep(int index, struct acm_ep_config_data **data);
+#define ib_acm_free_ep_data(data) free(data)
+
#endif /* LIBACM_H */
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-ofed/ibacm.git
More information about the Pkg-ofed-commits
mailing list