From 0aca280172674bf26c5de988b15c902f71f3d603 Mon Sep 17 00:00:00 2001 From: Bogdan Eugen Cristea Date: Fri, 19 Mar 2021 10:14:13 +0200 Subject: [PATCH 1/5] add debug string --- koet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/koet.py b/koet.py index c6eed94..af0fa12 100755 --- a/koet.py +++ b/koet.py @@ -1312,6 +1312,7 @@ def load_multiple_fping(logdir, hosts_dictionary): latencies_list = latencies.split(' ') # our mean calculation expect strings. Need to change this when # optimizing + print(latencies_list) mean_all.append(str(mean_list(latencies_list))) max_all.append(max(latencies_list)) min_all.append(min(latencies_list)) From 9e99bf45ea60bbba83bfdc161d62b506d94de1ca Mon Sep 17 00:00:00 2001 From: Bogdan Eugen Cristea Date: Mon, 22 Mar 2021 15:07:14 +0200 Subject: [PATCH 2/5] makefile for RDMA --- makefile | 2 +- makefile.rdma | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 makefile.rdma diff --git a/makefile b/makefile index 4fb4ec0..b445c94 100644 --- a/makefile +++ b/makefile @@ -10,5 +10,5 @@ all: $(PROGS) clean: rm -f $(PROGS) -nsdperf: nsdperf.C +$(PROGS): nsdperf.C $(CXX) $(CFLAGS) -o $@ nsdperf.C $(LIBS) diff --git a/makefile.rdma b/makefile.rdma new file mode 100644 index 0000000..3339fb1 --- /dev/null +++ b/makefile.rdma @@ -0,0 +1,14 @@ +# Makefile for net programs +CXX = g++ +LIBS = -lpthread -lrt -libverbs -lrdmacm +CFLAGS = -O2 -DRDMA + +PROGS = nsdperf_rdma + +all: $(PROGS) + +clean: + rm -f $(PROGS) + +$(PROGS): nsdperf.C + $(CXX) $(CFLAGS) -o $@ nsdperf.C $(LIBS) From baabe3ce791c9b588438aa032fee70f95f923b86 Mon Sep 17 00:00:00 2001 From: Bogdan Eugen Cristea Date: Wed, 24 Mar 2021 16:23:35 +0200 Subject: [PATCH 3/5] iterate over all available port interfaces --- nsdperf.C | 103 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 59 insertions(+), 44 deletions(-) diff --git a/nsdperf.C b/nsdperf.C index 7b2dbc1..327c305 100755 --- a/nsdperf.C +++ b/nsdperf.C @@ -2937,7 +2937,7 @@ static string rdmaStart() for (rdi = rdmaDevTab.begin(); rdi != rdmaDevTab.end(); ++rdi) { int j; - UInt64 portIf; + set portIfSet; IpAddr portAddr; string ifName; RdmaDevice *rdevP = *rdi; @@ -2963,7 +2963,8 @@ static string rdmaStart() } // Fetch the interface ID portion of the GID for the port - portIf = 0; + portIfSet.clear(); + UInt64 portIf = 0; for (j = 0; j < portAttr.gid_tbl_len; j++) { union ibv_gid gid; @@ -2976,6 +2977,8 @@ static string rdmaStart() portIf = gid.global.interface_id; if (portIf != 0) + portIfSet.insert(portIf); + else break; } @@ -2988,56 +2991,68 @@ static string rdmaStart() ifName.clear(); if (useCM) { - for (ni = netInterfaces.begin(); ni != netInterfaces.end(); ++ni) + bool stop = false; + for (auto it = portIfSet.begin(); it != portIfSet.end(); ++j) { - if (!ni->addr.isLinkLocal()) - continue; - if (rdevP->ibContext->device->transport_type == IBV_TRANSPORT_IWARP) + portIf = *it; + for (ni = netInterfaces.begin(); ni != netInterfaces.end(); ++ni) { - struct ifreq mac_req; - - if (ioctl_sock < 0) - ioctl_sock = socket(AF_INET, SOCK_DGRAM, 0); - if (ioctl_sock < 0) - break; - - memset(&mac_req, 0, sizeof mac_req); - strcpy(mac_req.ifr_name, ni->ifName.c_str()); - - if (ioctl(ioctl_sock, SIOCGIFHWADDR, &mac_req) < 0) + if (!ni->addr.isLinkLocal()) + continue; + if (rdevP->ibContext->device->transport_type == IBV_TRANSPORT_IWARP) { - Logm("Cannot get HW address"); - continue; + struct ifreq mac_req; + + if (ioctl_sock < 0) + ioctl_sock = socket(AF_INET, SOCK_DGRAM, 0); + if (ioctl_sock < 0) + { + stop = true; + break; + } + + memset(&mac_req, 0, sizeof mac_req); + strcpy(mac_req.ifr_name, ni->ifName.c_str()); + + if (ioctl(ioctl_sock, SIOCGIFHWADDR, &mac_req) < 0) + { + Logm("Cannot get HW address"); + continue; + } + + if (!(memcmp(&portIf, mac_req.ifr_hwaddr.sa_data, 6))) + { + ifName = ni->ifName; + Logt(3, "Found iWarp RDMA port " << portIfToString(portIf) << " " << ni->ifName); + stop = true; + break; + } } - - if (!(memcmp(&portIf, mac_req.ifr_hwaddr.sa_data, 6))) + else { - ifName = ni->ifName; - Logt(3, "Found iWarp RDMA port " << portIfToString(portIf) << " " << ni->ifName); - break; - } - } - else - { -#ifdef IPV6_SUPPORT - struct sockaddr_storage saddr; - struct sockaddr_in6 *sip6 = - reinterpret_cast(ni->addr.toSockaddr(0, &saddr)); - - // 7 bytes of the port interface ID will match 7 bytes in the - // IPv6 address. The first byte won't match because the - // universal/local bit is flipped in the IPv6 address (see - // RFC 4291). - char *a = reinterpret_cast(&portIf); - char *b = reinterpret_cast(&sip6->sin6_addr.s6_addr); - if (!memcmp(a+1, b+9, 7)) - { - ifName = ni->ifName; - break; + #ifdef IPV6_SUPPORT + struct sockaddr_storage saddr; + struct sockaddr_in6 *sip6 = + reinterpret_cast(ni->addr.toSockaddr(0, &saddr)); + + // 7 bytes of the port interface ID will match 7 bytes in the + // IPv6 address. The first byte won't match because the + // universal/local bit is flipped in the IPv6 address (see + // RFC 4291). + char *a = reinterpret_cast(&portIf); + char *b = reinterpret_cast(&sip6->sin6_addr.s6_addr); + if (!memcmp(a+1, b+9, 7)) + { + ifName = ni->ifName; + stop = true; + break; + } + #endif } -#endif } + if (stop) + break; } // Skip this port if no link-local address was found From 45bcadb9768323db0829ddf97c086127f12a5fe7 Mon Sep 17 00:00:00 2001 From: Bogdan Eugen Cristea Date: Wed, 24 Mar 2021 18:23:53 +0200 Subject: [PATCH 4/5] correction --- nsdperf.C | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nsdperf.C b/nsdperf.C index 327c305..47d0ec6 100755 --- a/nsdperf.C +++ b/nsdperf.C @@ -2992,7 +2992,7 @@ static string rdmaStart() if (useCM) { bool stop = false; - for (auto it = portIfSet.begin(); it != portIfSet.end(); ++j) + for (auto it = portIfSet.begin(); it != portIfSet.end(); ++it) { portIf = *it; for (ni = netInterfaces.begin(); ni != netInterfaces.end(); ++ni) From d10fb3dba6b169f6bbc0dc7552f7bd34a646444d Mon Sep 17 00:00:00 2001 From: Bogdan Eugen Cristea Date: Wed, 24 Mar 2021 18:44:52 +0200 Subject: [PATCH 5/5] rm debug line --- koet.py | 1 - 1 file changed, 1 deletion(-) diff --git a/koet.py b/koet.py index af0fa12..c6eed94 100755 --- a/koet.py +++ b/koet.py @@ -1312,7 +1312,6 @@ def load_multiple_fping(logdir, hosts_dictionary): latencies_list = latencies.split(' ') # our mean calculation expect strings. Need to change this when # optimizing - print(latencies_list) mean_all.append(str(mean_list(latencies_list))) max_all.append(max(latencies_list)) min_all.append(min(latencies_list))