[Pkg-ofed-commits] r743 - in /branches/ofed-1.5.1lenny/perftest/trunk: ./ debian/
gmpc-guest at alioth.debian.org
gmpc-guest at alioth.debian.org
Wed May 5 19:08:22 UTC 2010
Author: gmpc-guest
Date: Wed May 5 19:08:22 2010
New Revision: 743
URL: http://svn.debian.org/wsvn/pkg-ofed/?sc=1&rev=743
Log:
New upstream release
Added:
branches/ofed-1.5.1lenny/perftest/trunk/multicast_resources.c
- copied unchanged from r742, branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.c
branches/ofed-1.5.1lenny/perftest/trunk/multicast_resources.h
- copied unchanged from r742, branches/ofed-1.5.1lenny/perftest/branches/upstream/current/multicast_resources.h
Modified:
branches/ofed-1.5.1lenny/perftest/trunk/Makefile
branches/ofed-1.5.1lenny/perftest/trunk/README
branches/ofed-1.5.1lenny/perftest/trunk/debian/changelog
branches/ofed-1.5.1lenny/perftest/trunk/perftest.spec
branches/ofed-1.5.1lenny/perftest/trunk/read_bw.c
branches/ofed-1.5.1lenny/perftest/trunk/send_bw.c
Modified: branches/ofed-1.5.1lenny/perftest/trunk/Makefile
URL: http://svn.debian.org/wsvn/pkg-ofed/branches/ofed-1.5.1lenny/perftest/trunk/Makefile?rev=743&op=diff
==============================================================================
--- branches/ofed-1.5.1lenny/perftest/trunk/Makefile (original)
+++ branches/ofed-1.5.1lenny/perftest/trunk/Makefile Wed May 5 19:08:22 2010
@@ -1,26 +1,33 @@
RDMACM_TESTS = rdma_lat rdma_bw
-TESTS = write_bw_postlist send_lat send_bw write_lat write_bw read_lat read_bw
+MCAST_TESTS = send_bw
+TESTS = write_bw_postlist send_lat write_lat write_bw read_lat read_bw
UTILS = clock_test
-all: ${RDMACM_TESTS} ${TESTS} ${UTILS}
+all: ${RDMACM_TESTS} ${MCAST_TESTS} ${TESTS} ${UTILS}
CFLAGS += -Wall -g -D_GNU_SOURCE -O2
EXTRA_FILES = get_clock.c
-EXTRA_HEADERS = get_clock.h
+MCAST_FILES = multicast_resources.c
+EXTRA_HEADERS = get_clock.h
+MCAST_HEADERS = multicast_resources.h
#The following seems to help GNU make on some platforms
LOADLIBES +=
LDFLAGS +=
${RDMACM_TESTS}: LOADLIBES += -libverbs -lrdmacm
+${MCAST_TESTS}: LOADLIBES += -libverbs -lpthread -libumad
${TESTS} ${UTILS}: LOADLIBES += -libverbs
${RDMACM_TESTS}: %: %.c ${EXTRA_FILES} ${EXTRA_HEADERS}
$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $< ${EXTRA_FILES} $(LOADLIBES) $(LDLIBS) -o $@
+${MCAST_TESTS}: %: %.c ${EXTRA_FILES} ${MCAST_FILES} ${EXTRA_HEADERS} ${MCAST_HEADERS}
+ $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $< ${EXTRA_FILES} ${MCAST_FILES} $(LOADLIBES) $(LDLIBS) -o ib_$@
${TESTS} ${UTILS}: %: %.c ${EXTRA_FILES} ${EXTRA_HEADERS}
$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $< ${EXTRA_FILES} $(LOADLIBES) $(LDLIBS) -o ib_$@
clean:
$(foreach fname,${RDMACM_TESTS}, rm -f ${fname})
+ $(foreach fname,${MCAST_TESTS}, rm -f ib_${fname})
$(foreach fname,${TESTS} ${UTILS}, rm -f ib_${fname})
.DELETE_ON_ERROR:
.PHONY: all clean
Modified: branches/ofed-1.5.1lenny/perftest/trunk/README
URL: http://svn.debian.org/wsvn/pkg-ofed/branches/ofed-1.5.1lenny/perftest/trunk/README?rev=743&op=diff
==============================================================================
--- branches/ofed-1.5.1lenny/perftest/trunk/README (original)
+++ branches/ofed-1.5.1lenny/perftest/trunk/README Wed May 5 19:08:22 2010
@@ -1,7 +1,7 @@
Open Fabrics Enterprise Distribution (OFED)
- Performance Tests README for OFED 1.3
+ Performance Tests README for OFED 1.5
- March 2008
+ December 2009
@@ -9,7 +9,7 @@
Table of Contents
===============================================================================
1. Overview
-2. Notes on Testing Method
+2. Notes on Testing Methodology
3. Test Descriptions
4. Running Tests
@@ -18,30 +18,42 @@
===============================================================================
This is a collection of tests written over uverbs intended for use as a
performance micro-benchmark. As an example, the tests can be used for
-hardware or software tuning and/or functional testing.
+HW or SW tuning and/or functional testing.
-Please post results and observations to the openib-general mailing list.
+The collection conatains a set of BW and latency benchmark such as :
+
+ * Read - ib_read_bw and ib_read_lat.
+ * Write - ib_write_bw and ib_wriet_lat.
+ * Send - ib_send_bw and ib_send_lat.
+ * RDMA - rdma_bw and rdma_lat.
+ * In additional : ib_write_bw_postlist and ib_clock_test.
+
+Please post results/observations to the openib-general mailing list.
See "Contact Us" at http://openib.org/mailman/listinfo/openib-general and
http://www.openib.org.
===============================================================================
-2. Notes on Testing Method
+2. Notes on Testing Methodology
===============================================================================
-- The benchmark uses the CPU cycle counter to get time stamps without a context
- switch. Some CPU architectures (e.g., Intel's 80486 or older PPC) do NOT have
- such capability.
+- The benchmark used the CPU cycle counter to get time stamps without context
+ switch. Some CPU architectures (e.g., Intel's 80486 or older PPC) do NOT
+ have such capability.
- The benchmark measures round-trip time but reports half of that as one-way
latency. This means that it may not be sufficiently accurate for asymmetrical
configurations.
-- Min/Median/Max results are reported.
- The Median (vs average) is less sensitive to extreme scores.
- Typically, the Max value is the first value measured.
+- On Bw benchmarks , We calculate the BW on send side only, as he calculates
+ the Bw after collecting completion from the receive side.
+ In case we use the bidirectional flag , BW is calculated on both sides
-- Larger samples only help marginally. The default (1000) is very satisfactory.
- Note that an array of cycles_t (typically an unsigned long) is allocated
+- Min/Median/Max result is reported.
+ The median (vs average) is less sensitive to extreme scores.
+ Typically, the "Max" value is the first value measured.
+
+- Larger samples help marginally only. The default (1000) is pretty good.
+ Note that an array of cycles_t (typically unsigned long) is allocated
once to collect samples and again to store the difference between them.
Really big sample sizes (e.g., 1 million) might expose other problems
with the program.
@@ -54,32 +66,29 @@
===============================================================================
-3. Test Descriptions
+4. Test Descriptions
===============================================================================
+rdma_lat.c latency test with RDMA write transactions
+rdma_bw.c streaming BW test with RDMA write transactions
-The following tests are mainly useful for hardware/software benchmarking.
+The following tests are mainly useful for HW/SW benchmarking.
+They are not intended as actual usage examples.
+send_lat.c latency test with send transactions
+send_bw.c BW test with send transactions
write_lat.c latency test with RDMA write transactions
-write_bw.c bandwidth test with RDMA write transactions
-send_lat.c latency test with send transactions
-send_bw.c bandwidth test with send transactions
+write_bw.c BW test with RDMA write transactions
read_lat.c latency test with RDMA read transactions
-read_bw.c bandwidth test with RDMA read transactions
+read_bw.c BW test with RDMA read transactions
+The executable name of each test starts with the general prefix "ib_",
+e.g., ib_write_lat , exept of those of RDMA tests , in their case
+their excutable have the same name except of the .c.
-Legacy tests: (To be removed in the next release)
-rdma_lat.c latency test with RDMA write transactions
-rdma_bw.c streaming bandwidth test with RDMA write transactions
-
-The executable name of each test starts with the general prefix "ib_";
-for example, ib_write_lat.
-
-
-===============================================================================
-4. Running Tests
-===============================================================================
+Running Tests
+-------------
Prerequisites:
kernel 2.6
@@ -90,25 +99,6 @@
Client: ./<test name> <options> <server IP address>
o <server address> is IPv4 or IPv6 address. You can use the IPoIB
-diags_release_notes.txt
-mpi-selector_release_notes.txt
-rdma_cm_release_notes.txt
-MSTFLINT_README.txt
-open_mpi_release_notes.txt RDS_README.txt
-ib-bonding.txt
-mthca_release_notes.txt
-opensm_release_notes.txt
-rds_release_notes.txt
-ibutils_release_notes.txt*
-mvapich_release_notes.txt
-PERF_TEST_README.txt
-sdp_release_notes.txt
-ipoib_release_notes.txt
-srp_release_notes.txt
-QoS_in_OFED.txt
-SRPT_README.txt
-mlx4_release_notes.txt
-QoS_management_in_OpenSM.
address if IPoIB is configured.
o --help lists the available <options>
@@ -116,30 +106,38 @@
Common Options to all tests:
- -p, --port=<port> listen on/connect to port <port> (default: 18515)
- -m, --mtu=<mtu> mtu size (default: 1024)
- -d, --ib-dev=<dev> use IB device <dev> (default: first device found)
- -i, --ib-port=<port> use port <port> of IB device (default: 1)
- -s, --size=<size> size of message to exchange (default: 1)
- -a, --all run sizes from 2 till 2^23
- -t, --tx-depth=<dep> size of tx queue (default: 50)
- -n, --iters=<iters> number of exchanges (at least 100, default: 1000)
- -S, --sl=<sl> SL (default 0)
- -C, --report-cycles report times in cpu cycle units
- (default: microseconds)
- -H, --report-histogram print out all results
- (default: print summary only)
- -U, --report-unsorted (implies -H) print out unsorted results
- (default: sorted)
- -V, --version display version number
+ -p, --port=<port> Listen on/connect to port <port> (default: 18515).
+ -m, --mtu=<mtu> Mtu size (default: 1024).
+ -d, --ib-dev=<dev> Use IB device <dev> (default: first device found).
+ -i, --ib-port=<port> Use port <port> of IB device (default: 1).
+ -s, --size=<size> Size of message to exchange (default: 1).
+ -a, --all Run sizes from 2 till 2^23.
+ -t, --tx-depth=<dep> Size of tx queue (default: 50).
+ -r, --rx-depth=<dep> Make rx queue bigger than tx (default 600).
+ -n, --iters=<iters> Number of exchanges (at least 100, default: 1000).
+ -I, --inline_size=<size> Max size of message to be sent in inline mode.
+ On Bw tests default is 1,latency tests is 400.
+ -C, --report-cycles Report times in cpu cycle units.
+ -u, --qp-timeout=<timeout> QP timeout, timeout value is 4 usec*2 ^(timeout).
+ Default is 14.
+ -S, --sl=<sl> SL (default 0).
+ -H, --report-histogram Print out all results (Default: summary only).
+ Only on Latnecy tests.
+ -x, --gid-index=<index> Test uses GID with GID index taken from command
+ Line (for RDMAoE index should be 0).
+ -b, --bidirectional Measure bidirectional bandwidth (default uni).
+ On BW tests only (Implicit on latency tests).
+ -V, --version Display version number.
+ -e, --events Sleep on CQ events (default poll).
+ -N, --no peak-bw Cancel peak-bw calculation (default with peak-bw)
+ -F, --CPU-freq Do not fail even if cpufreq_ondemand module.
*** IMPORTANT NOTE: You need to be running a Subnet Manager on the switch or
on one of the nodes in your fabric.
Example:
-Run "ib_write_lat -a" on the server side.
-Then run "ib_write_lat -a <server IP address>" on the client side.
+Run "ib_rdma_lat -C" on the server side.
+Then run "ib_rdma_lat -C <server IP address>" on the client.
-ib_write_lat will exit on both server and client after printing results.
+ib_rdma_lat will exit on both server and client after printing results.
-
Modified: branches/ofed-1.5.1lenny/perftest/trunk/debian/changelog
URL: http://svn.debian.org/wsvn/pkg-ofed/branches/ofed-1.5.1lenny/perftest/trunk/debian/changelog?rev=743&op=diff
==============================================================================
--- branches/ofed-1.5.1lenny/perftest/trunk/debian/changelog (original)
+++ branches/ofed-1.5.1lenny/perftest/trunk/debian/changelog Wed May 5 19:08:22 2010
@@ -1,3 +1,9 @@
+perftest (1.2.3-0.10.g90b10d8-1) unstable; urgency=low
+
+ * New upstream release (OFED 1.5.1)
+
+ -- Guy Coates <gmpc at sanger.ac.uk> Wed, 05 May 2010 20:05:55 +0100
+
perftest (1.2.3-0.8.g196d994-1) unstable; urgency=low
* New upstream release (OFED 1.5)
Modified: branches/ofed-1.5.1lenny/perftest/trunk/perftest.spec
URL: http://svn.debian.org/wsvn/pkg-ofed/branches/ofed-1.5.1lenny/perftest/trunk/perftest.spec?rev=743&op=diff
==============================================================================
--- branches/ofed-1.5.1lenny/perftest/trunk/perftest.spec (original)
+++ branches/ofed-1.5.1lenny/perftest/trunk/perftest.spec Wed May 5 19:08:22 2010
@@ -1,10 +1,10 @@
Name: perftest
Summary: IB Performance tests
Version: 1.2.3
-Release: 0.8.g196d994
+Release: 0.10.g90b10d8
License: BSD 3-Clause, GPL v2 or later
Group: Productivity/Networking/Diagnostic
-Source: http://www.openfabrics.org/downloads/perftest-1.2.3-0.8.g196d994.tar.gz
+Source: http://www.openfabrics.org/downloads/perftest-1.2.3-0.10.g90b10d8.tar.gz
Url: http://www.openfabrics.org
BuildRoot: %{_tmppath}/%{name}-%{version}-build
BuildRequires: libibverbs-devel librdmacm-devel
Modified: branches/ofed-1.5.1lenny/perftest/trunk/read_bw.c
URL: http://svn.debian.org/wsvn/pkg-ofed/branches/ofed-1.5.1lenny/perftest/trunk/read_bw.c?rev=743&op=diff
==============================================================================
--- branches/ofed-1.5.1lenny/perftest/trunk/read_bw.c (original)
+++ branches/ofed-1.5.1lenny/perftest/trunk/read_bw.c Wed May 5 19:08:22 2010
@@ -56,20 +56,22 @@
#include "get_clock.h"
-#define MAX_OUTSTANDING_READ 16
+#define MAX_OUT_READ_HERMON 16
+#define MAX_OUT_READ 4
#define PINGPONG_READ_WRID 1
-#define VERSION 1.2
+#define VERSION 1.3
#define ALL 1
#define RC 0
struct user_parameters {
- const char *servername;
+ const char *servername;
int connection_type;
int mtu;
int all; /* run all msg size */
int iters;
int tx_depth;
int max_out_read;
+ int use_out_read;
int use_event;
int qp_timeout;
int gid_index; /* if value not negative, we use gid AND gid_index=value */
@@ -103,6 +105,37 @@
};
+/*
+ *
+ */
+static int set_max_out_read(struct user_parameters *param,struct ibv_device_attr *attr) {
+
+ int is_hermon = 0;
+ int max_out_reads;
+
+ // Checks the devide type for setting the max outstanding reads.
+ if (attr->vendor_part_id == 25408 || attr->vendor_part_id == 25418 ||
+ attr->vendor_part_id == 25448 || attr->vendor_part_id == 26418 ||
+ attr->vendor_part_id == 26428 || attr->vendor_part_id == 26438 ||
+ attr->vendor_part_id == 26448 || attr->vendor_part_id == 26458 ||
+ attr->vendor_part_id == 26468 || attr->vendor_part_id == 26478) {
+ is_hermon = 1;
+ }
+
+ max_out_reads = (is_hermon == 1) ? MAX_OUT_READ_HERMON : MAX_OUT_READ;
+
+ if (param->use_out_read) {
+ if (param->max_out_read <= 0 || param->max_out_read > max_out_reads) {
+ fprintf(stderr,"Max outstanding reads for this device is %d\n",max_out_reads);
+ return 1;
+ }
+ }
+ else {
+ param->max_out_read = max_out_reads;
+ }
+ return 0;
+}
+
static uint16_t pp_get_local_lid(struct pingpong_context *ctx, int port)
{
struct ibv_port_attr attr;
@@ -417,11 +450,18 @@
ibv_get_device_name(ib_dev));
return NULL;
}
- if (user_parm->mtu == 0) {/*user did not ask for specific mtu */
- if (ibv_query_device(ctx->context, &device_attr)) {
- fprintf(stderr, "Failed to query device props");
- return NULL;
- }
+
+ if (ibv_query_device(ctx->context, &device_attr)) {
+ fprintf(stderr, "Failed to query device props");
+ return NULL;
+ }
+
+ if (set_max_out_read(user_parm,&device_attr)) {
+ fprintf(stderr, "Failed to set the number of outstanding reads\n");
+ return NULL;
+ }
+
+ if (user_parm->mtu == 0) {/*user did not ask for specific mtu */
if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1)
user_parm->mtu = 1024;
else
@@ -578,7 +618,7 @@
printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
printf(" -m, --mtu=<mtu> mtu size (256 - 4096. default for hermon is 2048)\n");
- printf(" -o, --outs=<num> num of outstanding read/atom(default 16)\n");
+ printf(" -o, --outs=<num> num of outstanding read/atom(default for hermon 16 (others 4)\n");
printf(" -s, --size=<size> size of message to exchange (default 65536)\n");
printf(" -a, --all Run sizes from 2 till 2^23\n");
printf(" -t, --tx-depth=<dep> size of tx queue (default 100)\n");
@@ -726,11 +766,11 @@
/* init default values to user's parameters */
memset(&user_param, 0, sizeof(struct user_parameters));
user_param.mtu = 0;
+ user_param.use_out_read = 0;
user_param.iters = 1000;
user_param.tx_depth = 100;
user_param.servername = NULL;
user_param.use_event = 0;
- user_param.max_out_read = MAX_OUTSTANDING_READ; /* the device capability on gen2 */
user_param.qp_timeout = 14;
user_param.gid_index = -1; /*gid will not be used*/
/* Parameter parsing. */
@@ -781,6 +821,7 @@
break;
case 'o':
user_param.max_out_read = strtol(optarg, NULL, 0);
+ user_param.use_out_read = 1;
break;
case 'a':
user_param.all = ALL;
Modified: branches/ofed-1.5.1lenny/perftest/trunk/send_bw.c
URL: http://svn.debian.org/wsvn/pkg-ofed/branches/ofed-1.5.1lenny/perftest/trunk/send_bw.c?rev=743&op=diff
==============================================================================
--- branches/ofed-1.5.1lenny/perftest/trunk/send_bw.c (original)
+++ branches/ofed-1.5.1lenny/perftest/trunk/send_bw.c Wed May 5 19:08:22 2010
@@ -57,6 +57,7 @@
#include <infiniband/verbs.h>
#include "get_clock.h"
+#include "multicast_resources.h"
#define PINGPONG_SEND_WRID 1
#define PINGPONG_RECV_WRID 2
@@ -69,11 +70,6 @@
#define SIGNAL 1
#define MAX_INLINE 400
#define ALL 1
-#define MAX_POLL_ITERATION_TIMEOUT 100000000
-#define QPNUM_MCAST 0xffffff
-#define DEF_QKEY 0x11111111
-#define MCG_LID 0xc001
-#define MCG_GID {255,1,0,0,0,2,201,133,0,0,1,1,0,0,0,0}
struct user_parameters {
const char *servername;
@@ -87,9 +83,13 @@
int rx_depth;
int duplex;
int use_event;
+ int ib_port;
+ // Ido
+ int num_of_mcg_group;
+ int num_of_qp_on_group;
+ // End Ido
int use_mcg;
int use_user_mgid;
- int num_of_clients_mcg;
int inline_size;
int qp_timeout;
int gid_index; /* if value not negative, we use gid AND gid_index=value */
@@ -102,23 +102,26 @@
int post_recv;
struct pingpong_context {
- struct ibv_context *context;
+ struct ibv_context *context;
struct ibv_comp_channel *channel;
- struct ibv_pd *pd;
- struct ibv_mr *mr;
- struct ibv_cq *cq;
- struct ibv_qp *qp;
- struct mcg_qp *qp_mcg;
- void *buf;
- unsigned size;
- int tx_depth;
- int rx_depth;
- struct ibv_sge list;
- struct ibv_sge recv_list;
- struct ibv_send_wr wr;
- struct ibv_recv_wr rwr;
- struct ibv_ah *ah;
- union ibv_gid dgid;
+ struct ibv_pd *pd;
+ struct ibv_mr *mr;
+ struct ibv_cq *cq;
+ struct ibv_qp *qp;
+ // Ido
+ struct mcast_group *mcg_group;
+ struct mcast_parameters *mcg_params;
+ //End Ido
+ void *buf;
+ unsigned size;
+ int tx_depth;
+ int rx_depth;
+ struct ibv_sge list;
+ struct ibv_sge recv_list;
+ struct ibv_send_wr wr;
+ struct ibv_recv_wr rwr;
+ struct ibv_ah *ah;
+ union ibv_gid dgid;
};
struct pingpong_dest {
@@ -130,53 +133,87 @@
union ibv_gid dgid;
};
-struct mcg_qp {
-
- struct ibv_qp *qp;
- unsigned int package_counter;
- cycles_t *completion_array;
-};
-
/*
*
*/
-static int set_multicast_gid(struct pingpong_context *ctx,struct user_parameters *param) {
-
- uint8_t mcg_gid[16] = MCG_GID;
- const char *pstr = param->user_mgid;
- char *term = NULL;
- char tmp[20];
- int i;
-
- if (param->use_user_mgid) {
- term = strpbrk(pstr, ":");
- memcpy(tmp, pstr, term - pstr+1);
- tmp[term - pstr] = 0;
- mcg_gid[0] = (unsigned char)strtoll(tmp, NULL, 0);
- for (i = 1; i < 15; ++i) {
- pstr += term - pstr + 1;
- term = strpbrk(pstr, ":");
- memcpy(tmp, pstr, term - pstr+1);
- tmp[term - pstr] = 0;
- mcg_gid[i] = (unsigned char)strtoll(tmp, NULL, 0);
- }
- pstr += term - pstr + 1;
- strcpy(tmp, pstr);
- mcg_gid[15] = (unsigned char)strtoll(tmp, NULL, 0);
- }
- memcpy(ctx->dgid.raw,mcg_gid,16);
- return 0;
+static int create_mcast(struct pingpong_context *ctx,
+ struct user_parameters *param,
+ struct ibv_device *ib_dev) {
+
+
+ ALLOCATE(ctx->mcg_params,struct mcast_parameters,1);
+ ctx->mcg_params->num_of_groups = param->num_of_mcg_group;
+ ctx->mcg_params->num_qps_on_group = param->num_of_qp_on_group;
+ ctx->mcg_params->iterations = param->iters;
+ ctx->mcg_params->tx_depth = ctx->tx_depth;
+ ctx->mcg_params->rx_depth = ctx->rx_depth;
+ ctx->mcg_params->pd = ctx->pd;
+ ctx->mcg_params->ctx = ctx->context;
+ ctx->mcg_params->ib_dev = ib_dev;
+ ctx->mcg_params->ib_port = param->ib_port;
+ ctx->mcg_params->is_client = param->servername ? TRUE : FALSE;
+ ctx->mcg_params->is_user_mgid = param->use_user_mgid ? TRUE : FALSE;
+ ctx->mcg_params->inline_size = param->inline_size;
+ ctx->mcg_params->user_mgid = param->user_mgid;
+
+ // Query pkey.
+ if (ibv_query_pkey(ctx->context,param->ib_port,DEF_PKEY_IDX,&ctx->mcg_params->pkey)) {
+ fprintf(stderr, "failed to query PKey table of port %d\n",param->ib_port);
+ return 1;
+ }
+
+ // Query port gid.
+ if( ibv_query_gid(ctx->context,param->ib_port,0,&ctx->mcg_params->port_gid)) {
+ fprintf(stderr, "failed to query GID table of port %d.\n",param->ib_port);
+ return 1;
+ }
+
+ // Create the Mcast entities.
+ ctx->mcg_group = mcast_create_resources(ctx->mcg_params);
+ if (ctx->mcg_group == NULL) {
+ fprintf(stderr,"Couldn't create Multicast resources\n");
+ return 1;
+ }
+ return 0;
}
+
/*
*
*/
-static int
-qp_create(struct pingpong_context *ctx, struct user_parameters *param) {
+static int cq_create(struct pingpong_context *ctx,struct user_parameters *param) {
+
+ int i,rx_depth;
+
+ rx_depth = ctx->rx_depth;
+
+ if (param->use_mcg) {
+ rx_depth *= param->num_of_qp_on_group;
+ for (i=0; i < param->num_of_mcg_group; i++) {
+ ctx->mcg_group[i].mcg_cq = ibv_create_cq(ctx->context,rx_depth,NULL,ctx->channel,0);
+ if (!ctx->mcg_group[i].mcg_cq) {
+ fprintf(stderr, "Couldn't create CQ\n");
+ return 1;
+ }
+ }
+ }
+ else {
+ ctx->cq = ibv_create_cq(ctx->context,rx_depth,NULL,ctx->channel,0);
+ if (!ctx->cq) {
+ fprintf(stderr, "Couldn't create CQ --- \n");
+ return 1;
+ }
+ }
+ return 0;
+}
+
+/*
+ *
+ */
+static int qp_create(struct pingpong_context *ctx, struct user_parameters *param) {
struct ibv_qp_init_attr attr;
- int i;
memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
attr.send_cq = ctx->cq;
@@ -200,21 +237,15 @@
default:
fprintf(stderr, "Unknown connection type \n");
return 1;
- }
- if (param->use_mcg && !param->servername) {
- ctx->qp_mcg = malloc(sizeof(struct mcg_qp)*param->num_of_clients_mcg);
- if(ctx->qp_mcg == NULL) {
- fprintf(stdout, "Couldn't create QP\n");
- return 1;
- }
- for (i=0; i < param->num_of_clients_mcg; i++ ) {
- ctx->qp_mcg[i].qp = ibv_create_qp(ctx->pd, &attr);
- if (!ctx->qp_mcg[i].qp) {
- fprintf(stderr, "Couldn't create QP\n");
- return 1;
- }
- }
- }
+ }
+ // Ido
+ if (param->use_mcg) {
+ if (mcast_create_qps(ctx->mcg_group,ctx->mcg_params,&attr)){
+ fprintf(stderr,"Couldn't create QPs for Multicast group\n");
+ return 1;
+ }
+ }
+ // End Ido
else {
ctx->qp = ibv_create_qp(ctx->pd, &attr);
if (!ctx->qp) {
@@ -225,95 +256,94 @@
return 0;
}
-/*
+/*
*
*/
-static int modify_qp_to_init(struct ibv_qp *qp,int port,int connection_type) {
-
- struct ibv_qp_attr attr;
- int flags;
-
- memset(&attr, 0, sizeof(attr));
- attr.qp_state = IBV_QPS_INIT;
- attr.pkey_index = 0;
- attr.port_num = port;
-
- if (connection_type == UD) {
-
+static int modify_qp_to_init(struct pingpong_context *ctx,struct user_parameters *param) {
+
+ struct ibv_qp_attr attr;
+ int flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT;
+
+ memset(&attr, 0, sizeof(struct ibv_qp_attr));
+ attr.qp_state = IBV_QPS_INIT;
+ attr.pkey_index = 0;
+ attr.port_num = param->ib_port;
+
+ if (param->connection_type == UD) {
attr.qkey = DEF_QKEY;
- flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_QKEY;
- }
- else {
-
+ flags |= IBV_QP_QKEY;
+ } else {
attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE;
- flags = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS;
- }
- if (ibv_modify_qp(qp,&attr,flags)) {
-
- fprintf(stderr, "Failed to modify UD QP to INIT\n");
- return 1;
+ flags |= IBV_QP_ACCESS_FLAGS;
+ }
+
+ if (param->use_mcg) {
+ if (mcast_init_resources(ctx->mcg_group,ctx->mcg_params,&attr,flags)){
+ fprintf(stderr,"Couldn't init Multicast resources\n");
+ return 1;
+ }
+ }
+ else {
+ if (ibv_modify_qp(ctx->qp,&attr,flags)) {
+ fprintf(stderr, "Failed to modify UD QP to INIT\n");
+ return 1;
+ }
}
return 0;
}
-/*
- *
- */
+
static int destroy_ctx_resources(struct pingpong_context *ctx,
struct user_parameters *param) {
- int i;
int test_result = 0;
- if (ctx->ah) {
- if (ibv_destroy_ah(ctx->ah)) {
- fprintf(stderr, "failed to destroy AH\n");
+
+ if (param->use_mcg) {
+ if (mcast_destroy_resources(ctx->mcg_group,ctx->mcg_params)) {
+ fprintf(stderr, "failed to destroy MultiCast resources\n");
test_result = 1;
}
- }
- if (param->use_mcg && !param->servername) {
- for(i=0; i < param->num_of_clients_mcg; i++) {
-
- if (ibv_detach_mcast(ctx->qp_mcg[i].qp, &ctx->dgid,MCG_LID)) {
- fprintf(stderr, "failed to deatttached QP\n");
+ free(ctx->mcg_params);
+ }
+ else {
+ if (ctx->ah) {
+ if (ibv_destroy_ah(ctx->ah)) {
+ fprintf(stderr, "failed to destroy AH\n");
test_result = 1;
}
- if (ibv_destroy_qp(ctx->qp_mcg[i].qp)) {
- fprintf(stderr, "failed to destroy QP\n");
- test_result = 1;
- }
- free(ctx->qp_mcg[i].completion_array);
- }
- free(ctx->qp_mcg);
+ }
- } else {
if (ibv_destroy_qp(ctx->qp)) {
fprintf(stderr, "failed to destroy QP\n");
test_result = 1;
}
- }
+
+ if (ctx->cq) {
+ if (ibv_destroy_cq(ctx->cq)) {
+ fprintf(stderr, "failed to destroy CQ\n");
+ test_result = 1;
+ }
+ }
+ }
+
if (ctx->mr) {
if (ibv_dereg_mr(ctx->mr)) {
fprintf(stderr, "failed to deregister MR\n");
test_result = 1;
}
}
-
+
if (ctx->buf)
free(ctx->buf);
-
- if (ctx->cq) {
- if (ibv_destroy_cq(ctx->cq)) {
- fprintf(stderr, "failed to destroy CQ\n");
- test_result = 1;
- }
- }
+
if (ctx->channel) {
if (ibv_destroy_comp_channel(ctx->channel)) {
fprintf(stderr, "failed to destroy channel \n");
test_result = 1;
}
}
+
if (ctx->pd) {
if (ibv_dealloc_pd(ctx->pd)) {
fprintf(stderr, "failed to deallocate PD\n");
@@ -326,32 +356,22 @@
test_result = 1;
}
}
+
if(!param->use_mcg || param->servername) {
free(tposted);
free(tcompleted);
}
-
return test_result;
}
-/*
- *
- */
+
static int
alloc_time_res(struct pingpong_context *ctx, struct user_parameters *param ) {
- int i;
-
- if(param->use_mcg && !param->servername ) {
- for(i=0; i < param->num_of_clients_mcg; i++){
- ctx->qp_mcg[i].completion_array = malloc(sizeof(cycles_t)*param->iters);
- if(ctx->qp_mcg[i].completion_array == NULL) {
- perror("malloc");
- return 1;
- }
- }
- }
- else {
+ if(param->use_mcg) {
+ ;
+ }
+ else if (param->servername || param->duplex){
tposted = malloc(sizeof(cycles_t)*param->iters);
if (!tposted) {
perror("malloc");
@@ -660,12 +680,10 @@
static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev,
unsigned size,int tx_depth,
- int rx_depth, int port,
- struct user_parameters *user_parm)
+ int rx_depth,struct user_parameters *user_parm)
{
struct pingpong_context *ctx;
struct ibv_device_attr device_attr;
- int i,cq_rx_depth;
ctx = malloc(sizeof *ctx);
if (!ctx)
@@ -675,8 +693,7 @@
ctx->channel = NULL;
ctx->size = size;
ctx->tx_depth = tx_depth;
- ctx->rx_depth = rx_depth + tx_depth;
- cq_rx_depth = ctx->rx_depth;
+ ctx->rx_depth = tx_depth + rx_depth;
/* in case of UD need space for the GRH */
if (user_parm->connection_type==UD) {
ctx->buf = memalign(page_size, ( size + 40 ) * 2);
@@ -743,49 +760,36 @@
return NULL;
}
}
- if (user_parm->use_mcg && !user_parm->servername) {
- cq_rx_depth *= user_parm->num_of_clients_mcg;
- }
- ctx->cq = ibv_create_cq(ctx->context,cq_rx_depth, NULL, ctx->channel, 0);
- if (!ctx->cq) {
- fprintf(stderr, "Couldn't create CQ\n");
+
+ // Ido.
+ if (user_parm->use_mcg) {
+ if (create_mcast(ctx,user_parm,ib_dev)) {
+ return NULL;
+ }
+ }
+ // Ido.
+ if (cq_create(ctx,user_parm)) {
+ fprintf(stderr, "Couldn't Create CQ as requested\n");
return NULL;
}
+ // Ido.
if (qp_create(ctx,user_parm)) {
fprintf(stderr, "Couldn't create QP\n");
return NULL;
}
-
- if (user_parm->use_mcg && !user_parm->servername) {
- if (set_multicast_gid(ctx,user_parm)) {
- fprintf(stderr, "Couldn't Set the Multicast M_GID\n");
- return NULL;
- }
-
- for(i=0; i < user_parm->num_of_clients_mcg; i++) {
- if (modify_qp_to_init(ctx->qp_mcg[i].qp,port,user_parm->connection_type)) {
- fprintf(stderr, "Failed to modify UD QP to INIT\n");
- return NULL;
- }
- if (ibv_attach_mcast(ctx->qp_mcg[i].qp,&ctx->dgid,MCG_LID)) {
- fprintf(stderr, "Couldn't attach QP to mcg\n");
- return NULL;
- }
- }
- printf(" Successfully created and attached %d QPs to a Mcast group\n\n",i);
- } else {
- if (modify_qp_to_init(ctx->qp,port,user_parm->connection_type)) {
- fprintf(stderr, "Failed to modify UD QP to INIT\n");
- return NULL;
- }
- }
+ // Ido.
+ if (modify_qp_to_init(ctx,user_parm)) {
+ fprintf(stderr, "Couldn't modify to init\n");
+ return NULL;
+ }
+ // End Ido.
return ctx;
}
/*
*
*/
-static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
+static int pp_connect_ctx(struct pingpong_context *ctx,int my_psn,
struct pingpong_dest *dest,
struct user_parameters *user_parm)
{
@@ -811,9 +815,8 @@
attr.path_mtu = IBV_MTU_4096;
break;
}
- printf(" Mtu : %d\n", user_parm->mtu);
- attr.dest_qp_num = (user_parm->use_mcg && user_parm->servername) ?
- QPNUM_MCAST : dest->qpn;
+ printf("Mtu : %d\n", user_parm->mtu);
+ attr.dest_qp_num = dest->qpn;
attr.rq_psn = dest->psn;
if (user_parm->connection_type == RC) {
attr.max_dest_rd_atomic = 1;
@@ -830,16 +833,8 @@
attr.ah_attr.sl = 0;
}
attr.ah_attr.src_path_bits = 0;
- attr.ah_attr.port_num = port;
- if (user_parm->use_mcg && user_parm->servername) {
- attr.ah_attr.grh.hop_limit = 1;
- attr.ah_attr.sl = 0;
- attr.ah_attr.dlid = MCG_LID;
- attr.ah_attr.is_global = 1;
- attr.ah_attr.grh.sgid_index = 0;
- memcpy(attr.ah_attr.grh.dgid.raw,dest->dgid.raw, 16);
- }
-
+ attr.ah_attr.port_num = user_parm->ib_port;
+
if (user_parm->connection_type == RC) {
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
@@ -865,29 +860,27 @@
fprintf(stderr, "Failed to modify UC QP to RTR\n");
return 1;
}
- }
+ }
+ // Ido
+ else if (user_parm->use_mcg) {
+ if (mcast_modify_qp_to_rtr(ctx->mcg_group,ctx->mcg_params,&attr)) {
+ fprintf(stderr,"Couldn't modify Multicast Qps to RTR\n");
+ return 1;
+ }
+ }
+ // End Ido.
else {
-
- if (user_parm->use_mcg && !user_parm->servername) {
- for(i=0; i < user_parm->num_of_clients_mcg; i++) {
- if (ibv_modify_qp(ctx->qp_mcg[i].qp, &attr, IBV_QP_STATE )) {
- printf("Failed to modify UD QP to RTR\n");
- return 1;
- }
- }
- }
- else {
- if (ibv_modify_qp(ctx->qp,&attr, IBV_QP_STATE )) {
- fprintf(stderr, "Failed to modify UD QP to RTR\n");
- return 1;
- }
- }
- ctx->ah = ibv_create_ah(ctx->pd, &attr.ah_attr);
- if (!ctx->ah) {
- fprintf(stderr, "Failed to create AH for UD\n");
- return 1;
- }
- }
+ if (ibv_modify_qp(ctx->qp,&attr, IBV_QP_STATE )) {
+ fprintf(stderr, "Failed to modify UD QP to RTR\n");
+ return 1;
+ }
+ ctx->ah = ibv_create_ah(ctx->pd, &attr.ah_attr);
+ if (!ctx->ah) {
+ fprintf(stderr, "Failed to create AH for UD\n");
+ return 1;
+ }
+ }
+
attr.qp_state = IBV_QPS_RTS;
attr.sq_psn = my_psn;
attr.max_rd_atomic = 1;
@@ -903,17 +896,25 @@
fprintf(stderr, "Failed to modify RC QP to RTS\n");
return 1;
}
- } else { /*both UC and UD */
- if(!user_parm->use_mcg || user_parm->servername) {
- if(ibv_modify_qp(ctx->qp, &attr,IBV_QP_STATE |IBV_QP_SQ_PSN)) {
+ }
+ else if (!user_parm->use_mcg) {
+ if(ibv_modify_qp(ctx->qp, &attr,IBV_QP_STATE |IBV_QP_SQ_PSN)) {
fprintf(stderr, "Failed to modify UC QP to RTS\n");
return 1;
- }
- }
- }
+ }
+ }
+ else if (user_parm->servername) {
+ for(i=0; i < user_parm->num_of_mcg_group; i++) {
+ if (ibv_modify_qp(ctx->mcg_group[i].send_qp,&attr,IBV_QP_STATE |IBV_QP_SQ_PSN)) {
+ printf("Failed to modify UD QP to RTS\n");
+ return 1;
+ }
+ }
+ }
+
/* post recieve max msg size*/
{
- int i,j;
+ int i;
struct ibv_recv_wr *bad_wr_recv;
//recieve
ctx->rwr.wr_id = PINGPONG_RECV_WRID;
@@ -927,16 +928,15 @@
ctx->recv_list.length = ctx->size;
}
ctx->recv_list.lkey = ctx->mr->lkey;
- if ( user_parm->use_mcg && !user_parm->servername) {
- for (i=0; i < user_parm->num_of_clients_mcg; i++) {
- for (j = 0; j < ctx->rx_depth; ++j) {
- if (ibv_post_recv(ctx->qp_mcg[i].qp, &ctx->rwr, &bad_wr_recv)) {
- fprintf(stderr, "Couldn't post recv: counter=%d\n", j);
- return 14;
- }
- }
- }
- } else {
+
+ if (user_parm->use_mcg) {
+ ctx->mcg_params->rwr = &ctx->rwr;
+ if (mcast_post_receive(ctx->mcg_group,ctx->mcg_params)) {
+ fprintf(stderr,"Couldn't post receive\n");
+ return 1;
+ }
+ }
+ else {
for (i = 0; i < ctx->rx_depth; ++i) {
if (ibv_post_recv(ctx->qp, &ctx->rwr, &bad_wr_recv)) {
fprintf(stderr, "Couldn't post recv: counter=%d\n", i);
@@ -964,7 +964,8 @@
printf(" -s, --size=<size> Size of message to exchange (default 65536)\n");
printf(" -a, --all Run sizes from 2 till 2^23\n");
printf(" -t, --tx-depth=<dep> Size of tx queue (default 300)\n");
- printf(" -g, --mcg=<num_of_qps> Send messages to multicast group with <num_of_qps> qps attached to it.\n");
+ printf(" -g, --mcg=<num_of_mcg> Send messages to <num_of_mcg> multicast groups .\n");
+ printf(" -q, --qp_num=<num_of_qps> Attach <num_of_qps> QPs to each multicast group (default 1).\n");
printf(" -r, --rx-depth=<dep> Make rx queue bigger than tx (default 600)\n");
printf(" -n, --iters=<iters> Number of exchanges (at least 2, default 1000)\n");
printf(" -I, --inline_size=<size> Max size of message to be sent in inline mode (default 0)\n");
@@ -1019,52 +1020,7 @@
tsize * iters * cycles_to_units /(tcompleted[iters - 1] - tposted[0]) / 0x100000);
}
-/*
- *
- */
-static void print_mcg_report(struct pingpong_context *ctx,
- struct user_parameters *param,
- unsigned size,
- int no_cpu_freq_fail) {
-
- int i,packets_loss = 0;
- unsigned long packets_sum = 0;
- unsigned int lower , upper;
- int last_pos;
- double cycles_to_units;
- cycles_t max_first_comp, min_last_comp;
-
- cycles_to_units = get_cpu_mhz(no_cpu_freq_fail) * 1000000;
-
- last_pos = (ctx->qp_mcg[0].package_counter) ? ctx->qp_mcg[0].package_counter -1 : 0;
- max_first_comp = ctx->qp_mcg[0].completion_array[0];
- min_last_comp = ctx->qp_mcg[0].completion_array[last_pos];
-
- for (i=0; i < param->num_of_clients_mcg; i++) {
- if (ctx->qp_mcg[i].completion_array[0] > max_first_comp) {
- max_first_comp = ctx->qp_mcg[i].completion_array[0];
- }
- last_pos = (ctx->qp_mcg[i].package_counter) ? ctx->qp_mcg[i].package_counter -1 : 0;
- if (ctx->qp_mcg[i].completion_array[last_pos] < min_last_comp) {
- min_last_comp = ctx->qp_mcg[i].completion_array[last_pos];
- }
- }
- for (i=0; i < param->num_of_clients_mcg; i++ ) {
-
- upper = lower = 0;
- last_pos = (ctx->qp_mcg[i].package_counter) ? ctx->qp_mcg[i].package_counter -1 : 0;
- while (ctx->qp_mcg[i].completion_array[lower] < max_first_comp) {
- lower++;
- }
- while (ctx->qp_mcg[i].completion_array[last_pos] > min_last_comp) {
- last_pos++;
- }
- packets_sum += ctx->qp_mcg[i].package_counter - upper -lower;
- packets_loss += (param->iters - ctx->qp_mcg[i].package_counter);
- }
- printf("%7d %d %d %7.2f\n",size,param->iters,packets_loss,
- packets_sum*size*cycles_to_units/(min_last_comp - max_first_comp) / 0x100000);
-}
+
/*
*
@@ -1201,11 +1157,9 @@
struct pingpong_dest *rem_dest,
int size)
{
- int i,ne,num_of_iters;
int scnt, ccnt, rcnt;
struct ibv_recv_wr *bad_wr_recv;
struct ibv_send_wr *bad_wr;
- struct ibv_wc wc;
if (user_param->connection_type == UD) {
@@ -1232,18 +1186,30 @@
scnt = 0;
ccnt = 0;
rcnt = 0;
- if (user_param->use_mcg && !user_param->servername ) {
- for (i=0; i < user_param->num_of_clients_mcg; i++) {
- ctx->qp_mcg[i].package_counter = 0;
- }
- }
+ // Ido
+ if (user_param->use_mcg) {
+
+ ctx->mcg_params->rwr = &ctx->rwr;
+ ctx->mcg_params->size = size;
+ if (user_param->servername) {
+ if(mcast_post_send(ctx->mcg_group,ctx->mcg_params,&ctx->list)) {
+ fprintf(stderr,"Cannot post send\n");
+ return 1;
+ }
+ }
+ // Need to add more things.
+ if (mcast_run_iter_uni(ctx->mcg_group,ctx->mcg_params)) {
+ fprintf(stderr,"Cannot run the benchmark iteration\n");
+ return 1;
+ }
+ }
+ else {
// Server side.
if (!user_param->servername) {
-
- num_of_iters = user_param->use_mcg ?
- user_param->iters*user_param->num_of_clients_mcg : user_param->iters;
- while (rcnt < num_of_iters && scnt < MAX_POLL_ITERATION_TIMEOUT) {
-
+ while (rcnt < user_param->iters) {
+ int ne;
+ struct ibv_wc wc;
+ /*Server is polling on recieve first */
if (user_param->use_event) {
struct ibv_cq *ev_cq;
void *ev_ctx;
@@ -1261,44 +1227,25 @@
}
}
do {
- ne = ibv_poll_cq(ctx->cq, 1 , &wc);
- if (ne > 0) {
+ ne = ibv_poll_cq(ctx->cq, 1, &wc);
+ if (ne) {
+ // tcompleted[ccnt] = get_cycles();
if (wc.status != IBV_WC_SUCCESS) {
- fprintf(stderr, "Completion wth error at server :\n");
+ fprintf(stderr, "Completion wth error at %s:\n",
+ user_param->servername ? "client" : "server");
fprintf(stderr, "Failed status %d: wr_id %d syndrom 0x%x\n",
- wc.status, (int) wc.wr_id, wc.vendor_err);
+ wc.status, (int) wc.wr_id, wc.vendor_err);
fprintf(stderr, "scnt=%d, ccnt=%d\n",
- scnt, ccnt);
+ scnt, ccnt);
return 1;
}
++rcnt;
- scnt = 0;
- if (user_param->use_mcg) {
- for (i=0; i < user_param->num_of_clients_mcg; i++) {
- if ( wc.qp_num == ctx->qp_mcg[i].qp->qp_num) {
-
- ccnt = ctx->qp_mcg[i].package_counter;
- ctx->qp_mcg[i].completion_array[ccnt] = get_cycles();
- ctx->qp_mcg[i].package_counter++;
-
- if (ibv_post_recv(ctx->qp_mcg[i].qp, &ctx->rwr, &bad_wr_recv)) {
- fprintf(stderr, "Couldn't post recv: rcnt=%d\n",rcnt);
- return 15;
- }
- break;
- }
- }
+ if (ibv_post_recv(ctx->qp, &ctx->rwr, &bad_wr_recv)) {
+ fprintf(stderr, "Couldn't post recv: rcnt=%d\n",
+ rcnt);
+ return 15;
}
- else {
- if (ibv_post_recv(ctx->qp, &ctx->rwr, &bad_wr_recv)) {
- fprintf(stderr, "Couldn't post recv: rcnt=%d\n",rcnt);
- return 15;
- }
- }
- }
- else {
- scnt++;
- //deal with stat ne = 0
+
}
} while (ne > 0 );
@@ -1307,8 +1254,7 @@
return 12;
}
}
- }
- else {
+ } else {
/* client is posting and not receiving. */
while (scnt < user_param->iters || ccnt < user_param->iters) {
while (scnt < user_param->iters && (scnt - ccnt) < user_param->tx_depth ) {
@@ -1364,21 +1310,21 @@
}
}
}
+ }
return 0;
}
int main(int argc, char *argv[])
{
- struct ibv_device **dev_list;
- struct ibv_device *ib_dev;
+ struct ibv_device **dev_list;
+ struct ibv_device *ib_dev;
struct pingpong_context *ctx;
struct pingpong_dest my_dest;
struct pingpong_dest *rem_dest;
struct user_parameters user_param;
- struct ibv_device_attr device_attribute;
+ struct ibv_device_attr device_attribute;
char *ib_devname = NULL;
int port = 18515;
- int ib_port = 1;
long long size = 65536;
int sockfd;
int i = 0;
@@ -1392,9 +1338,11 @@
/* init default values to user's parameters */
memset(&user_param, 0, sizeof(struct user_parameters));
user_param.mtu = 0;
- user_param.num_of_clients_mcg = 1;
+ user_param.ib_port = 1;
+ user_param.num_of_qp_on_group = 1;
user_param.iters = 1000;
user_param.tx_depth = 300;
+ user_param.rx_depth = 300;
user_param.servername = NULL;
user_param.user_mgid = NULL;
user_param.use_event = 0;
@@ -1421,6 +1369,7 @@
{ .name = "sl", .has_arg = 1, .val = 'S' },
{ .name = "gid-index", .has_arg = 1, .val = 'x' },
{ .name = "mcg", .has_arg = 1, .val = 'g' },
+ { .name = "qp_num", .has_arg = 1, .val = 'q' },
{ .name = "MGID", .has_arg = 1, .val = 'M' },
{ .name = "all", .has_arg = 0, .val = 'a' },
{ .name = "bidirectional", .has_arg = 0, .val = 'b' },
@@ -1431,7 +1380,7 @@
{ 0 }
};
- c = getopt_long(argc, argv, "p:d:i:m:c:s:n:t:I:r:u:S:x:g:M:ebaVNF", long_options, NULL);
+ c = getopt_long(argc, argv, "p:d:i:m:c:s:n:t:I:r:u:S:x:g:q:M:ebaVNF", long_options, NULL);
if (c == -1)
break;
@@ -1448,8 +1397,15 @@
break;
case 'g':
++user_param.use_mcg;
- user_param.num_of_clients_mcg = strtol(optarg, NULL, 0);
- if (user_param.num_of_clients_mcg < 0) {
+ user_param.num_of_mcg_group = strtol(optarg, NULL, 0);
+ if (user_param.num_of_mcg_group <= 0) {
+ usage(argv[0]);
+ return 1;
+ }
+ break;
+ case 'q':
+ user_param.num_of_qp_on_group = strtol(optarg, NULL, 0);
+ if (user_param.num_of_mcg_group <= 0) {
usage(argv[0]);
return 1;
}
@@ -1478,8 +1434,8 @@
return 0;
break;
case 'i':
- ib_port = strtol(optarg, NULL, 0);
- if (ib_port < 0) {
+ user_param.ib_port = strtol(optarg, NULL, 0);
+ if (user_param.ib_port < 0) {
usage(argv[0]);
return 1;
}
@@ -1517,9 +1473,9 @@
case 'r':
errno = 0;
- user_param.rx_depth = strtol(optarg, NULL, 0);
- if (errno) { usage(argv[0]); return 1; }
- break;
+ user_param.rx_depth = strtol(optarg, NULL, 0);
+ if (errno) { usage(argv[0]); return 1; }
+ break;
case 'n':
user_param.iters = strtol(optarg, NULL, 0);
@@ -1641,23 +1597,22 @@
user_param.inline_size = 0;
}
- printf("Inline data is used up to %d bytes message\n\n", user_param.inline_size);
-
- ctx = pp_init_ctx(ib_dev, size, user_param.tx_depth, user_param.rx_depth,ib_port, &user_param);
+ printf("Inline data is used up to %d bytes message\n", user_param.inline_size);
+
+ ctx = pp_init_ctx(ib_dev, size, user_param.tx_depth, user_param.rx_depth, &user_param);
if (!ctx)
return 1;
/* Create connection between client and server.
* We do it by exchanging data over a TCP socket connection. */
- my_dest.lid = pp_get_local_lid(ctx, ib_port);
+ my_dest.lid = pp_get_local_lid(ctx, user_param.ib_port);
my_dest.psn = lrand48() & 0xffffff;
- my_dest.qpn = (user_param.use_mcg && !user_param.servername) ?
- ctx->qp_mcg[0].qp->qp_num : ctx->qp->qp_num;
+ my_dest.qpn = (user_param.use_mcg) ? QPNUM_MCAST : ctx->qp->qp_num;
if (user_param.gid_index != -1) {
int err=0;
- err = ibv_query_gid (ctx->context, ib_port, user_param.gid_index, &gid);
+ err = ibv_query_gid (ctx->context,user_param.ib_port, user_param.gid_index, &gid);
if (err) {
return -1;
}
@@ -1670,19 +1625,21 @@
return 1;
}
}
+ if (user_param.use_mcg) {
+ ctx->dgid = ctx->mcg_group[0].mgid;
+ }
my_dest.dgid = ctx->dgid;
my_dest.rkey = ctx->mr->rkey;
my_dest.vaddr = (uintptr_t)ctx->buf + size;
- printf(" local address: LID %#04x, PSN %#06x " ,my_dest.lid, my_dest.psn);
+ printf(" local address: LID %#04x, QPN %#06x, PSN %#06x\n" ,my_dest.lid,my_dest.qpn,my_dest.psn);
if (user_param.use_mcg && !user_param.servername) {
- printf("\n M_gid adress: M_ ");
- } else {
- printf(" QPN %#06x\n" ,my_dest.qpn);
- }
+ printf(" M_");
+ }
+
if (user_param.gid_index > -1 || (user_param.use_mcg && !user_param.servername) ) {
- printf("GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ printf("GID %02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d\n",
my_dest.dgid.raw[0],my_dest.dgid.raw[1],
my_dest.dgid.raw[2], my_dest.dgid.raw[3], my_dest.dgid.raw[4],
my_dest.dgid.raw[5], my_dest.dgid.raw[6], my_dest.dgid.raw[7],
@@ -1706,17 +1663,13 @@
if (!rem_dest)
return 1;
- printf(" remote address: LID %#04x, PSN %#06x ", rem_dest->lid, rem_dest->psn);
+ printf(" remote address: LID %#04x, QPN %#06x, PSN %#06x\n",rem_dest->lid,rem_dest->qpn,rem_dest->psn);
if (user_param.use_mcg && user_param.servername) {
- printf("\n M_gid adress: M_");
- }
- else {
- printf(" QPN %#06x\n",rem_dest->qpn);
- }
-
+ printf(" M_");
+ }
if (user_param.gid_index > -1 || (user_param.use_mcg && user_param.servername) ) {
- printf("GID %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+ printf("GID %02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d:%02d\n",
rem_dest->dgid.raw[0],rem_dest->dgid.raw[1],rem_dest->dgid.raw[2], rem_dest->dgid.raw[3], rem_dest->dgid.raw[4],
rem_dest->dgid.raw[5],rem_dest->dgid.raw[6], rem_dest->dgid.raw[7],rem_dest->dgid.raw[8], rem_dest->dgid.raw[9],
rem_dest->dgid.raw[10],rem_dest->dgid.raw[11], rem_dest->dgid.raw[12], rem_dest->dgid.raw[13],
@@ -1724,7 +1677,7 @@
}
- if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest, &user_param))
+ if (pp_connect_ctx(ctx,my_dest.psn, rem_dest, &user_param))
return 1;
/* An additional handshake is required *after* moving qp to RTR.
@@ -1742,18 +1695,18 @@
return 1;
}
}
- printf("--------------------------------------------------------------------------\n");
- if (user_param.servername || user_param.duplex) {
- printf(" #bytes #iterations BW peak[MB/sec] BW average[MB/sec] \n");
- }
- else if (user_param.use_mcg) {
- printf(" #bytes #iterations Packets Loss Aggregated BW [MB/s] \n");
+ printf("------------------------------------------------------------------\n");
+ if (user_param.use_mcg) {
+ printf(" #bytes #iterations #Pcg Success BW Aggreg.[MB/sec]\n");
+ }
+ else if (user_param.servername || user_param.duplex) {
+ printf(" #bytes #iterations BW peak[MB/sec] BW average[MB/sec] \n");
}
else {
printf(" Results are in the send side , Have a good day \n");
}
-
+
if(alloc_time_res(ctx,&user_param)) {
fprintf(stderr, "Couldn't allocate the resources for the tests\n");
return 1;
@@ -1773,12 +1726,18 @@
ctx->list.addr = (uintptr_t)ctx->buf;
}
- ctx->list.lkey = ctx->mr->lkey;
+ ctx->list.lkey = ctx->mr->lkey;
ctx->wr.wr_id = PINGPONG_SEND_WRID;
ctx->wr.sg_list = &ctx->list;
ctx->wr.num_sge = 1;
ctx->wr.opcode = IBV_WR_SEND;
ctx->wr.next = NULL;
+ if (user_param.use_mcg) {
+ if (mcast_post_send(ctx->mcg_group,ctx->mcg_params,&ctx->list)) {
+ fprintf(stderr,"Couldn't create Sending Wqe\n");
+ return 1;
+ }
+ }
/* recieve */
ctx->rwr.wr_id = PINGPONG_RECV_WRID;
@@ -1787,6 +1746,10 @@
ctx->rwr.next = NULL;
ctx->recv_list.addr = (uintptr_t) ctx->buf;
ctx->recv_list.lkey = ctx->mr->lkey;
+ if (user_param.use_mcg) {
+ ctx->mcg_params->rwr = &ctx->rwr;
+ }
+
if (user_param.all == ALL) {
if (user_param.connection_type == UD) {
@@ -1806,14 +1769,18 @@
if(run_iter_uni(ctx, &user_param, rem_dest, size))
return 17;
}
- if (user_param.servername) {
+ if (user_param.use_mcg) {
+ if (user_param.servername) {
+ mcast_print_client(ctx->mcg_group,ctx->mcg_params,no_cpu_freq_fail);
+ }
+ else {
+ mcast_print_server(ctx->mcg_group,ctx->mcg_params,no_cpu_freq_fail);
+ }
+ }
+ else if (user_param.servername) {
print_report(user_param.iters, size, user_param.duplex, tposted, tcompleted, noPeak, no_cpu_freq_fail);
}
- else {
- if (user_param.use_mcg) {
- print_mcg_report(ctx,&user_param,size,no_cpu_freq_fail);
- }
- }
+
if (user_param.servername) {
rem_dest = pp_client_exch_dest(sockfd, &my_dest, &user_param);
} else {
@@ -1831,12 +1798,17 @@
return 18;
}
- if (user_param.use_mcg && !user_param.servername) {
- print_mcg_report(ctx,&user_param,size,no_cpu_freq_fail);
- }
-
- if (user_param.servername || user_param.duplex)
- print_report(user_param.iters, size, user_param.duplex, tposted, tcompleted, noPeak, no_cpu_freq_fail);
+ if (user_param.use_mcg) {
+ if (user_param.servername) {
+ mcast_print_client(ctx->mcg_group,ctx->mcg_params,no_cpu_freq_fail);
+ }
+ else {
+ mcast_print_server(ctx->mcg_group,ctx->mcg_params,no_cpu_freq_fail);
+ }
+ }
+ else if (user_param.servername) {
+ print_report(user_param.iters, size, user_param.duplex, tposted, tcompleted, noPeak, no_cpu_freq_fail);
+ }
}
/* close sockets */
More information about the Pkg-ofed-commits
mailing list