struct ibv_sge {uint64_t        addr;uint32_t        length;uint32_t        lkey;
};

#include <infiniband/verbs.h>int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr,struct ibv_send_wr **bad_wr);

struct ibv_send_wr {uint64_t                wr_id;                  /* User defined WR ID */struct ibv_send_wr     *next;                   /* Pointer to next WR in list, NULL if last WR */struct ibv_sge         *sg_list;                /* Pointer to the s/g array */int                     num_sge;                /* Size of the s/g array */enum ibv_wr_opcode      opcode;                 /* Operation type */int                     send_flags;             /* Flags of the WR properties */uint32_t                imm_data;               /* Immediate data (in network byte order) */union {struct {uint64_t        remote_addr;    /* Start address of remote memory buffer */uint32_t        rkey;           /* Key of the remote Memory Region */} rdma;struct {uint64_t        remote_addr;    /* Start address of remote memory buffer */uint64_t        compare_add;    /* Compare operand */uint64_t        swap;           /* Swap operand */uint32_t        rkey;           /* Key of the remote Memory Region */} atomic;struct {struct ibv_ah  *ah;             /* Address handle (AH) for the remote node address */uint32_t        remote_qpn;     /* QP number of the destination QP */uint32_t        remote_qkey;    /* Q_Key number of the destination QP */} ud;} wr;
};struct ibv_sge {uint64_t        addr;   /* Start address of the local memory buffer */uint32_t        length; /* Length of the buffer */uint32_t        lkey;   /* Key of the local Memory Region */
};

 1 #define BUFFER_SIZE     1024
 2
 3 struct connection {
 4         struct rdma_cm_id       *id;
 5         struct ibv_qp           *qp;
 6
 7         struct ibv_mr           *recv_mr;
 8         struct ibv_mr           *send_mr;
 9
10         char                    recv_region[BUFFER_SIZE];
11         char                    send_region[BUFFER_SIZE];
12
13         int                     num_completions;
14 };
15
16 void foo_send(void *context)
17 {
18         struct connection *conn = (struct connection *)context;
19
20         /* 1. Fill the array SGL having only one element */
21         struct ibv_sge sge;
22
23         memset(&sge, 0, sizeof(sge));
24         sge.addr        = (uintptr_t)conn->send_region;
25         sge.length      = BUFFER_SIZE;
26         sge.lkey        = conn->send_mr->lkey;
27
28         /* 2. Fill the singly-linked list WR having only one node */
29         struct ibv_send_wr wr;
30         struct ibv_send_wr *bad_wr = NULL;
31
32         memset(&wr, 0, sizeof(wr));
33         wr.wr_id        = (uintptr_t)conn;
34         wr.opcode       = IBV_WR_SEND;
35         wr.sg_list      = &sge;
36         wr.num_sge      = 1;
37         wr.send_flags   = IBV_SEND_SIGNALED;
38
39         /* 3. Now send ... */
40         ibv_post_send(conn->qp, &wr, &bad_wr);
41
42         ...<snip>...
43 }

A great ship asks deep waters. | 是大船就得走深水，是蛟龙就得去大海里畅游。