RDMA Basic Read Example

This example demonstrates the essential control flow for performing a one-sided RDMA Read from a client into a memory region hosted on the server, using librdmacm and libibverbs.

On the server side, the program:

• Initializes an RDMA endpoint, listens on a TCP port, and accepts an incoming RDMA connection.
• Creates a Queue Pair and allocates a 4 KB buffer in host memory.
• Fills this buffer with an application-defined message (e.g., "Hello RDMA READ (from server)").
• Registers this buffer as a memory region with IBV_ACCESS_REMOTE_READ, and sends its (addr, rkey, len) to the client via the RDMA connection manager’s private data field.
• Does not issue any RDMA work requests itself; it simply waits while the client pulls the data and then prints the buffer content.

On the client side, the program:

• Resolves the server’s address and route, creates a Queue Pair, and establishes the RDMA connection.
• Receives the server’s memory metadata (addr, rkey, len) through private data.
• Allocates and registers a local buffer as the destination of the RDMA Read.
• Posts a single IBV_WR_RDMA_READ work request that pulls data from the server’s buffer into its local memory and waits for completion by polling the send completion queue.
• After completion, prints the data that was fetched from the server.

The following are the actual implementations:

#include <rdma/rdma_cma.h>
#include <infiniband/verbs.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <arpa/inet.h>
#include <netdb.h>

struct Info {
  uint64_t addr;
  uint32_t rkey, len;
} __attribute__((packed));

static void die(const char *m) {
  perror(m);
  exit(1);
}

int main(int argc, char **argv) {
  if (argc < 3) {
    fprintf(stderr, "Usage: %s <server_ip> <port>\n", argv[0]);
    return 1;
  }
  const char *ip = argv[1];
  int port = atoi(argv[2]);

  struct rdma_event_channel *ec = rdma_create_event_channel();
  struct rdma_cm_id *id;
  struct rdma_cm_event *e;

  if (rdma_create_id(ec, &id, NULL, RDMA_PS_TCP))
    die("create_id");

  struct addrinfo *res;
  char ps[16];
  snprintf(ps, sizeof(ps), "%d", port);
  if (getaddrinfo(ip, ps, NULL, &res))
    die("getaddrinfo");

  if (rdma_resolve_addr(id, NULL, res->ai_addr, 2000))
    die("resolve_addr");
  if (rdma_get_cm_event(ec, &e))
    die("event1");
  rdma_ack_cm_event(e);

  if (rdma_resolve_route(id, 2000))
    die("resolve_route");
  if (rdma_get_cm_event(ec, &e))
    die("event2");
  rdma_ack_cm_event(e);

  struct ibv_qp_init_attr qa = {0};
  qa.qp_type = IBV_QPT_RC;
  qa.cap.max_send_wr = qa.cap.max_recv_wr = 8;
  qa.cap.max_send_sge = qa.cap.max_recv_sge = 1;
  qa.sq_sig_all = 1;
  if (rdma_create_qp(id, id->pd, &qa))
    die("create_qp");

  struct rdma_conn_param p = {0};
  if (rdma_connect(id, &p))
    die("connect");

  if (rdma_get_cm_event(ec, &e))
    die("event3");
  struct Info info;
  memcpy(&info, e->param.conn.private_data, sizeof(info));
  rdma_ack_cm_event(e);

  size_t len = info.len;
  char *buf = aligned_alloc(4096, len);
  if (!buf)
    die("aligned_alloc");
  memset(buf, 0, len);

  struct ibv_mr *mr = ibv_reg_mr(
      id->pd, buf, len,
      IBV_ACCESS_LOCAL_WRITE);
  if (!mr)
    die("reg_mr");

  struct ibv_sge s = {
      .addr = (uintptr_t)buf,
      .length = (uint32_t)len,
      .lkey = mr->lkey};

  struct ibv_send_wr wr = {0}, *bad = NULL;
  wr.opcode = IBV_WR_RDMA_READ;
  wr.sg_list = &s;
  wr.num_sge = 1;
  wr.wr.rdma.remote_addr = info.addr;
  wr.wr.rdma.rkey = info.rkey;

  if (ibv_post_send(id->qp, &wr, &bad))
    die("post_send");

  struct ibv_wc wc;
  while (ibv_poll_cq(id->qp->send_cq, 1, &wc) == 0) {
  }
  if (wc.status)
    die("wc");

  printf("[client] read: '%.*s'\n", 64, buf);

  rdma_disconnect(id);
  ibv_dereg_mr(mr);
  free(buf);
  rdma_destroy_qp(id);
  rdma_destroy_id(id);
  rdma_destroy_event_channel(ec);
  freeaddrinfo(res);
  return 0;
}

#include <arpa/inet.h>
#include <infiniband/verbs.h>
#include <rdma/rdma_cma.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

struct Info {
  uint64_t addr;
  uint32_t rkey, len;
} __attribute__((packed));

static void die(const char *m) {
  perror(m);
  exit(1);
}

int main(int argc, char **argv) {
  if (argc < 2) {
    fprintf(stderr, "Usage: %s <port>\n", argv[0]);
    return 1;
  }
  int port = atoi(argv[1]);

  struct rdma_event_channel *ec = rdma_create_event_channel();
  struct rdma_cm_id *lid, *id;
  struct rdma_cm_event *e;
  struct sockaddr_in a = {0};
  a.sin_family = AF_INET;
  a.sin_port = htons(port);

  if (rdma_create_id(ec, &lid, NULL, RDMA_PS_TCP))
    die("create_id");
  if (rdma_bind_addr(lid, (struct sockaddr *)&a))
    die("bind");
  if (rdma_listen(lid, 1))
    die("listen");
  printf("[server] listening on %d ...\n", port);

  if (rdma_get_cm_event(ec, &e))
    die("get_event");
  id = e->id;
  rdma_ack_cm_event(e);

  struct ibv_qp_init_attr qa = {0};
  qa.qp_type = IBV_QPT_RC;
  qa.cap.max_send_wr = qa.cap.max_recv_wr = 8;
  qa.cap.max_send_sge = qa.cap.max_recv_sge = 1;
  qa.sq_sig_all = 1;
  if (rdma_create_qp(id, id->pd, &qa))
    die("create_qp");

  size_t len = 4096;
  char *buf = aligned_alloc(4096, len);
  if (!buf)
    die("aligned_alloc");
  memset(buf, 0, len);
  snprintf(buf, len, "Hello RDMA READ (from server)");

  struct ibv_mr *mr = ibv_reg_mr(
      id->pd, buf, len,
      IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ);
  if (!mr)
    die("reg_mr");

  struct Info info = {(uint64_t)buf, mr->rkey, (uint32_t)len};
  struct rdma_conn_param p = {0};
  p.private_data = &info;
  p.private_data_len = sizeof(info);
  if (rdma_accept(id, &p))
    die("accept");

  if (rdma_get_cm_event(ec, &e))
    die("event2");
  rdma_ack_cm_event(e);

  sleep(2);
  printf("[server] buffer still contains: '%.*s'\n", 64, buf);

  rdma_disconnect(id);
  ibv_dereg_mr(mr);
  free(buf);
  rdma_destroy_qp(id);
  rdma_destroy_id(id);
  rdma_destroy_id(lid);
  rdma_destroy_event_channel(ec);
  return 0;
}