rread may return EAGAIN when socket configured blocking

[Chris Hegarty <chris.hegarty@xxxxxxxxxx>]

Hi,

There is ongoing work to support RDMA-based sockets in the JDK [1] (Java
Development Kit), which builds on top of rsocket and the underlying
platform support. The Java API allows configuration of both listening
and connecting sockets, as well as blocking and non-blocking. We've
encountered an issue where `rread`, on a socket configured to be
blocking, returns EAGAIN. This is an unexpected error code for such a
blocking socket and the Java implementation currently does not handle it
in a way that conforms to the higher-level Java API specification. We
either need to resolve the issue or workaround it somehow.

To demonstrate the issue I put together a native test ( includes no Java
 code ) [*]. The output os as follows:

$ ./testNonBlocking_raccept

listening on port 45361
connecting to 10.0.2.15:45361

accepted rs2: 8
reading ...
rread: Resource temporarily unavailable
rread returned: -1, errno: 11

This clearly demonstrates that EAGAIN is being returned from `rread`,
when the socket is configured blocking.

---

System configuration and details:
 $ cat /etc/os-release
 NAME="Ubuntu"
 VERSION="18.10 (Cosmic Cuttlefish)"
 ID=ubuntu
 ID_LIKE=debian
 PRETTY_NAME="Ubuntu 18.10"
 VERSION_ID="18.10"
 HOME_URL="https://www.ubuntu.com/";
 SUPPORT_URL="https://help.ubuntu.com/";
 BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/";

PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy";
 VERSION_CODENAME=cosmic
 UBUNTU_CODENAME=cosmic

 $ uname -a
 Linux chhegar-VirtualBox 4.18.0-11-generic #12-Ubuntu SMP Tue Oct 23 
19:22:37 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux

 $ apt list | grep rdma
 librdmacm-dev/cosmic,now 19.0-1 amd64 [installed]
 librdmacm1/cosmic,now 19.0-1 amd64 [installed,automatic]
 rdma-core/cosmic,now 19.0-1 amd64 [installed]
 rdmacm-utils/cosmic,now 19.0-1 amd64 [installed]

 - No special hardware installed, using Soft-RoCE
 $ lsmod | grep rdma
 rdma_rxe              118784  0
 rpcrdma               221184  0
 sunrpc                364544  1 rpcrdma
 ip6_udp_tunnel         16384  1 rdma_rxe
 udp_tunnel             16384  1 rdma_rxe
 rdma_ucm               28672  0
 rdma_cm                61440  3 rpcrdma,ib_iser,rdma_ucm
 iw_cm                  45056  1 rdma_cm
 ib_cm                  53248  1 rdma_cm
 ib_uverbs              94208  1 rdma_ucm
 ib_core               233472  8 
rdma_cm,rdma_rxe,rpcrdma,iw_cm,ib_iser,rdma_ucm,ib_uverbs,ib_cm

 $ rxe_cfg status
   Name    Link  Driver  Speed  NMTU  IPv4_addr  RDEV  RMTU
   enp0s3  yes   e1000                           rxe0  1024  (3)

Regards,
-Chris.

[1] http://openjdk.java.net/jeps/337

[*]
// gcc testNonBlocking_raccept.c -lrdmacm -lpthread -o 
testNonBlocking_raccept

#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <arpa/inet.h>
#include <rdma/rsocket.h>
#include <errno.h>
#include <unistd.h>
#include <pthread.h>

static const char* hostname = "10.0.2.15";  // << INSERT YOUR IP HERE
static int port;

void *connecting_side_function(void *ptr) {
    int rs, ret;
    struct sockaddr_in addr;

    rs = rsocket(AF_INET, SOCK_STREAM, 0);
    if (rs < 0)
        perror("rsocket connecting-side");

    memset(&addr, 0, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_port = port;

    ret = inet_aton(hostname, &addr.sin_addr);
    if (ret < 0)
        perror("inet_aton");

    fprintf(stdout, "\nconnecting to %s:%d \n",
            inet_ntoa(addr.sin_addr), ntohs(addr.sin_port));
    ret = rconnect(rs, (struct sockaddr *)&addr, sizeof(struct sockaddr));
    if (ret < 0) {
        perror("rconnect");
        exit(3);
    }
    fprintf(stdout, "\nconnected");

    usleep(100000);  // just wait
}

int main() {
    int rs1, rs2, ret;
    pthread_t thread;
    char buffer[256];
    struct sockaddr_in addr, addr2;
    socklen_t addr_len = sizeof(addr);

    rs1 = rsocket(AF_INET, SOCK_STREAM, 0);
    if (rs1 < 0)
        perror("rsocket");

    memset(&addr, 0, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_addr.s_addr = htonl(INADDR_ANY);
    addr.sin_port = htons(0);
    ret = rbind(rs1, (struct sockaddr *) &addr, addr_len);
    if (ret < 0)
        perror("rbind");

    ret = rlisten(rs1, 10);
    if (ret < 0)
        perror("rlisten");

    memset(&addr, 0, sizeof(addr));
    if (rgetsockname(rs1, (struct sockaddr *) &addr, &addr_len) < 0)
        perror("rgetsockname");
    port = addr.sin_port;
    fprintf(stdout, "\nlistening on port %d", ntohs(port));

    // Configure the listening socket to be non-blocking
    rfcntl(rs1, F_SETFL, (rfcntl(rs2, F_GETFL) | O_NONBLOCK));

    // Start the connecting thread
    pthread_create(&thread, NULL, connecting_side_function, (void*) NULL);

    while (1) {
        rs2 = raccept(rs1, (struct sockaddr *)&addr2, &addr_len);
        if (rs2 >= 0) {
           fprintf(stdout, "\naccepted rs2: %d ", rs2);
           break;
        } else if (rs2 < 0 && !(errno == EAGAIN || errno == EWOULDBLOCK)) {
            perror("raccept");
            exit(1);
        }
        usleep(50*100);
    }

    // Configure the accepted socket to be blocking
    rfcntl(rs1, F_SETFL, (rfcntl(rs2, F_GETFL) & ~O_NONBLOCK));

    //usleep(10*100);  // if you wait a little all is ok!!!

    fprintf(stdout, "\nreading ...");
    fflush(stdout);
    ret = rread(rs2, buffer, 255);
    if (ret < 0) {
        fprintf(stdout, "\nrread returned: %d, errno: %d ", ret, errno);
        perror("rread");
    } else {
        fprintf(stdout, "\nrread returned: %d ", ret);
    }
}