WQWork Queue‌存储工作请求的队列包含多个WQE。软件通过WQ向硬件下发任务硬件从WQ中取出WQE执行‌WQEWork Queue Element‌工作队列元素是软件提交给硬件的具体任务描述如发送/接收数据的地址、长度等‌QPQueue Pair‌由发送队列SQ和接收队列RQ组成是RDMA通信的基本单位用于双向数据传输‌SQSend Queue‌存放发送任务的队列包含发送操作的WQE如RDMA_Write、Send等‌RQReceive Queue‌存放接收任务的队列包含预分配的接收缓冲区地址RQ WQE‌CQCompletion Queue‌存储硬件完成任务的通知CQE用于软件轮询或中断确认操作状态‌SRQShared Receive Queue‌多个QP共享的接收队列减少内存和硬件资源占用‌CQECompletion Queue Element‌完成队列元素记录操作状态成功/失败和类型MRMemory Region内存区域‌发送流程‌应用程序提交WQE到QP的SQ - 触发Doorbell通知硬件 - 硬件执行操作 - 生成CQE到CQ‌接收流程‌应用程序预分配接收缓冲区并提交WQE到RQ或SRQ - 硬件将远端数据写入指定缓冲区 - 生成CQE到CQ一、应用层收发流程write_server.cpp#include iostream #include cstring #include cstdlib #include unistd.h #include netdb.h #include arpa/inet.h #include rdma/rdma_cma.h #define PORT 51216 #define BUFFER_SIZE 1024 // 定义数据结构用于交换内存信息 struct mem_info { uint64_t addr; // 内存地址 uint32_t rkey; // 远程键 }; struct connection { struct ibv_qp *qp; struct ibv_mr *mr; char *buffer; struct ibv_cq *send_cq; struct ibv_cq *recv_cq; }; int main(int argc, char **argv) { struct rdma_event_channel *ec NULL; struct rdma_cm_id *listener NULL; struct rdma_cm_event *event NULL; struct ibv_pd *pd NULL; struct connection conn; int err; // 解析设备参数 const char *dev_name mlx5_0; // 默认设备 if (argc 2 strcmp(argv[1], -d) 0) { dev_name argv[2]; } std::cout Server starting with device: dev_name std::endl; // 1. 创建事件通道 ec rdma_create_event_channel(); if (!ec) { std::cerr Failed to create event channel std::endl; return -1; } // 2. 创建RDMA CM ID err rdma_create_id(ec, listener, NULL, RDMA_PS_TCP); if (err) { std::cerr Failed to create RDMA CM ID std::endl; return -1; } // 3. 绑定到端口 struct sockaddr_in addr; memset(addr, 0, sizeof(addr)); addr.sin_family AF_INET; addr.sin_port htons(atoi(PORT)); addr.sin_addr.s_addr INADDR_ANY; err rdma_bind_addr(listener, (struct sockaddr*)addr); if (err) { std::cerr Failed to bind address std::endl; return -1; } // 4. 监听连接 err rdma_listen(listener, 1); if (err) { std::cerr Failed to listen std::endl; return -1; } std::cout Server listening on port PORT std::endl; // 5. 等待连接请求事件 err rdma_get_cm_event(ec, event); if (err) { std::cerr Failed to get CM event std::endl; return -1; } if (event-event ! RDMA_CM_EVENT_CONNECT_REQUEST) { std::cerr Expected CONNECT_REQUEST, got event-event std::endl; rdma_ack_cm_event(event); return -1; } // 6. 接受连接 struct rdma_cm_id *conn_id event-id; rdma_ack_cm_event(event); // 7. 获取保护域 pd ibv_alloc_pd(conn_id-verbs); if (!pd) { std::cerr Failed to allocate protection domain std::endl; return -1; } // 8. 分配并注册内存 conn.buffer (char*)malloc(BUFFER_SIZE); if (!conn.buffer) { std::cerr Failed to allocate buffer std::endl; ibv_dealloc_pd(pd); return -1; } memset(conn.buffer, 0, BUFFER_SIZE); conn.mr ibv_reg_mr(pd, conn.buffer, BUFFER_SIZE, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); if (!conn.mr) { std::cerr Failed to register memory region std::endl; free(conn.buffer); ibv_dealloc_pd(pd); return -1; } std::cout Registered memory at address: (void*)conn.buffer , rkey: conn.mr-rkey std::endl; // 9. 创建队列对 struct ibv_qp_init_attr qp_attr; memset(qp_attr, 0, sizeof(qp_attr)); qp_attr.cap.max_send_wr 1; qp_attr.cap.max_recv_wr 1; qp_attr.cap.max_send_sge 1; qp_attr.cap.max_recv_sge 1; qp_attr.qp_type IBV_QPT_RC; conn.send_cq ibv_create_cq(conn_id-verbs, 10, NULL, NULL, 0); conn.recv_cq ibv_create_cq(conn_id-verbs, 10, NULL, NULL, 0); qp_attr.send_cq conn.send_cq; qp_attr.recv_cq conn.recv_cq; if (!conn.send_cq || !conn.recv_cq) { std::cerr Failed to create completion queue std::endl; ibv_dereg_mr(conn.mr); free(conn.buffer); ibv_dealloc_pd(pd); return -1; } err rdma_create_qp(conn_id, pd, qp_attr); if (err) { std::cerr Failed to create QP std::endl; ibv_destroy_cq(conn.send_cq); ibv_destroy_cq(conn.recv_cq); ibv_dereg_mr(conn.mr); free(conn.buffer); ibv_dealloc_pd(pd); return -1; } conn.qp conn_id-qp; // 10. 发送连接信息给客户端 struct rdma_conn_param conn_param; memset(conn_param, 0, sizeof(conn_param)); conn_param.responder_resources 1; conn_param.initiator_depth 1; conn_param.rnr_retry_count 7; // 创建内存信息结构体 struct mem_info minfo; minfo.addr (uint64_t)conn.buffer; minfo.rkey conn.mr-rkey; // 发送服务器MR的地址和rkey给客户端 conn_param.private_data minfo; conn_param.private_data_len sizeof(minfo); std::cout Sending memory info to client - addr: (void*)minfo.addr , rkey: minfo.rkey std::endl; err rdma_accept(conn_id, conn_param); if (err) { std::cerr Failed to accept connection std::endl; return -1; } std::cout Connection established. Waiting for data... std::endl; // 11. 等待连接建立完成 err rdma_get_cm_event(ec, event); if (err) { std::cerr Failed to get CM event std::endl; return -1; } if (event-event ! RDMA_CM_EVENT_ESTABLISHED) { std::cerr Expected ESTABLISHED, got event-event std::endl; rdma_ack_cm_event(event); return -1; } // 可以在这里读取客户端发送的信息 if (event-param.conn.private_data event-param.conn.private_data_len sizeof(uint32_t)) { uint32_t client_rkey; memcpy(client_rkey, event-param.conn.private_data, sizeof(uint32_t)); std::cout Got client rkey: client_rkey std::endl; } rdma_ack_cm_event(event); // 12. 等待数据写入 std::cout Waiting for RDMA write... std::endl; // 等待一段时间让客户端写入数据 for (int i 0; i 10; i) { if (strlen(conn.buffer) 0) { break; } sleep(1); } if (strlen(conn.buffer) 0) { std::cout Received data: conn.buffer std::endl; } else { std::cout No data received or data was empty std::endl; } // 清理 std::cout Disconnecting... std::endl; rdma_disconnect(conn_id); ibv_destroy_cq(conn.send_cq); ibv_destroy_cq(conn.recv_cq); ibv_dereg_mr(conn.mr); free(conn.buffer); rdma_destroy_qp(conn_id); ibv_dealloc_pd(pd); rdma_destroy_id(conn_id); rdma_destroy_id(listener); rdma_destroy_event_channel(ec); return 0; }write_client.cpp#include iostream #include cstring #include cstdlib #include unistd.h #include netdb.h #include arpa/inet.h #include rdma/rdma_cma.h #define PORT 51216 #define BUFFER_SIZE 1024 // 定义数据结构用于交换内存信息 struct mem_info { uint64_t addr; // 内存地址 uint32_t rkey; // 远程键 }; struct connection { struct ibv_qp *qp; struct ibv_mr *mr; char *buffer; uint32_t remote_rkey; uint64_t remote_addr; struct ibv_cq *send_cq; struct ibv_cq *recv_cq; }; int main(int argc, char **argv) { struct rdma_event_channel *ec NULL; struct rdma_cm_id *cm_id NULL; struct rdma_cm_event *event NULL; struct ibv_pd *pd NULL; struct connection conn; int err; // 解析参数 const char *dev_name mlx5_0; // 默认设备 const char *server_addr 127.0.0.1; // 默认地址 for (int i 1; i argc; i) { if (strcmp(argv[i], -d) 0 i 1 argc) { dev_name argv[i]; } else if (i argc - 1) { server_addr argv[i]; } } std::cout Connecting to server: server_addr using device: dev_name std::endl; // 1. 创建事件通道 ec rdma_create_event_channel(); if (!ec) { std::cerr Failed to create event channel std::endl; return -1; } // 2. 创建RDMA CM ID err rdma_create_id(ec, cm_id, NULL, RDMA_PS_TCP); if (err) { std::cerr Failed to create RDMA CM ID std::endl; return -1; } // 3. 解析服务器地址 struct addrinfo *res; struct addrinfo hints; memset(hints, 0, sizeof(hints)); hints.ai_family AF_INET; hints.ai_socktype SOCK_STREAM; err getaddrinfo(server_addr, PORT, hints, res); if (err) { std::cerr Failed to resolve address: gai_strerror(err) std::endl; return -1; } // 4. 连接到服务器 err rdma_resolve_addr(cm_id, NULL, res-ai_addr, 2000); if (err) { std::cerr Failed to resolve address std::endl; freeaddrinfo(res); return -1; } freeaddrinfo(res); // 5. 等待地址解析完成 err rdma_get_cm_event(ec, event); if (err) { std::cerr Failed to get CM event std::endl; return -1; } if (event-event ! RDMA_CM_EVENT_ADDR_RESOLVED) { std::cerr Expected ADDR_RESOLVED, got event-event std::endl; rdma_ack_cm_event(event); return -1; } rdma_ack_cm_event(event); // 6. 解析路由 err rdma_resolve_route(cm_id, 2000); if (err) { std::cerr Failed to resolve route std::endl; return -1; } // 7. 等待路由解析完成 err rdma_get_cm_event(ec, event); if (err) { std::cerr Failed to get CM event std::endl; return -1; } if (event-event ! RDMA_CM_EVENT_ROUTE_RESOLVED) { std::cerr Expected ROUTE_RESOLVED, got event-event std::endl; rdma_ack_cm_event(event); return -1; } rdma_ack_cm_event(event); // 8. 获取保护域 pd ibv_alloc_pd(cm_id-verbs); if (!pd) { std::cerr Failed to allocate protection domain std::endl; return -1; } // 9. 分配并注册内存 conn.buffer (char*)malloc(BUFFER_SIZE); if (!conn.buffer) { std::cerr Failed to allocate buffer std::endl; ibv_dealloc_pd(pd); return -1; } // 准备要发送的数据 const char *message Hello from RDMA client!; strncpy(conn.buffer, message, BUFFER_SIZE - 1); conn.buffer[BUFFER_SIZE - 1] \0; conn.mr ibv_reg_mr(pd, conn.buffer, BUFFER_SIZE, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); if (!conn.mr) { std::cerr Failed to register memory region std::endl; free(conn.buffer); ibv_dealloc_pd(pd); return -1; } std::cout Registered memory at address: (void*)conn.buffer , rkey: conn.mr-rkey std::endl; // 10. 创建队列对 struct ibv_qp_init_attr qp_attr; memset(qp_attr, 0, sizeof(qp_attr)); qp_attr.cap.max_send_wr 1; qp_attr.cap.max_recv_wr 1; qp_attr.cap.max_send_sge 1; qp_attr.cap.max_recv_sge 1; qp_attr.qp_type IBV_QPT_RC; conn.send_cq ibv_create_cq(cm_id-verbs, 10, NULL, NULL, 0); conn.recv_cq ibv_create_cq(cm_id-verbs, 10, NULL, NULL, 0); qp_attr.send_cq conn.send_cq; qp_attr.recv_cq conn.recv_cq; if (!conn.send_cq || !conn.recv_cq) { std::cerr Failed to create completion queue std::endl; ibv_dereg_mr(conn.mr); free(conn.buffer); ibv_dealloc_pd(pd); return -1; } err rdma_create_qp(cm_id, pd, qp_attr); if (err) { std::cerr Failed to create QP std::endl; ibv_destroy_cq(conn.send_cq); ibv_destroy_cq(conn.recv_cq); ibv_dereg_mr(conn.mr); free(conn.buffer); ibv_dealloc_pd(pd); return -1; } conn.qp cm_id-qp; // 11. 连接服务器 struct rdma_conn_param conn_param; memset(conn_param, 0, sizeof(conn_param)); conn_param.initiator_depth 1; conn_param.responder_resources 1; conn_param.retry_count 7; // 发送本地rkey给服务器 conn_param.private_data conn.mr-rkey; conn_param.private_data_len sizeof(conn.mr-rkey); err rdma_connect(cm_id, conn_param); if (err) { std::cerr Failed to connect std::endl; return -1; } // 12. 等待连接建立完成 err rdma_get_cm_event(ec, event); if (err) { std::cerr Failed to get CM event std::endl; return -1; } if (event-event ! RDMA_CM_EVENT_ESTABLISHED) { std::cerr Expected ESTABLISHED, got event-event std::endl; rdma_ack_cm_event(event); return -1; } // 从服务器接收远程内存信息 if (event-param.conn.private_data event-param.conn.private_data_len sizeof(struct mem_info)) { struct mem_info minfo; memcpy(minfo, event-param.conn.private_data, sizeof(minfo)); conn.remote_addr minfo.addr; conn.remote_rkey minfo.rkey; std::cout Got remote memory info - addr: (void*)conn.remote_addr , rkey: conn.remote_rkey std::endl; } else { std::cerr No valid private data received from server std::endl; if (event-param.conn.private_data) { std::cerr Data length: event-param.conn.private_data_len std::endl; } rdma_ack_cm_event(event); return -1; } rdma_ack_cm_event(event); std::cout Connection established. Sending data... std::endl; // 13. 执行RDMA Write操作 struct ibv_sge sge; sge.addr (uint64_t)conn.buffer; sge.length strlen(message) 1; sge.lkey conn.mr-lkey; struct ibv_send_wr wr; memset(wr, 0, sizeof(wr)); wr.wr_id 0; wr.sg_list sge; wr.num_sge 1; wr.opcode IBV_WR_RDMA_WRITE; wr.send_flags IBV_SEND_SIGNALED; // 设置远程内存信息现在有正确的地址了 wr.wr.rdma.remote_addr conn.remote_addr; wr.wr.rdma.rkey conn.remote_rkey; std::cout Posting RDMA write to remote addr: (void*)conn.remote_addr with rkey: conn.remote_rkey std::endl; struct ibv_send_wr *bad_wr NULL; err ibv_post_send(conn.qp, wr, bad_wr); if (err) { std::cerr Failed to post send: strerror(errno) std::endl; return -1; } // 14. 等待发送完成 struct ibv_wc wc; int num_completed; int retry_count 0; const int max_retries 100; std::cout Waiting for completion... std::endl; do { num_completed ibv_poll_cq(conn.send_cq, 1, wc); if (num_completed 0) { std::cerr Error polling CQ std::endl; break; } retry_count; if (retry_count max_retries) { std::cerr Timeout waiting for completion std::endl; break; } usleep(1000); // 睡眠1ms } while (num_completed 0); if (num_completed 0) { if (wc.status IBV_WC_SUCCESS) { std::cout RDMA Write completed successfully! std::endl; std::cout Sent message: message std::endl; } else { std::cerr RDMA Write failed with status: ibv_wc_status_str(wc.status) ( wc.status ) std::endl; } } sleep(1); // 确保服务器收到数据 // 清理 std::cout Disconnecting... std::endl; rdma_disconnect(cm_id); ibv_destroy_cq(conn.send_cq); ibv_destroy_cq(conn.recv_cq); ibv_dereg_mr(conn.mr); free(conn.buffer); rdma_destroy_qp(cm_id); ibv_dealloc_pd(pd); rdma_destroy_id(cm_id); rdma_destroy_event_channel(ec); return 0; }ibv_post_send和ibv_post_recv都是在QPQueue Pair上投递 WQE 的函数但是方向不一样这里server端代码并没有使用ibv_post_recv()函数是因为发送端使用的是wr.opcode IBV_WR_RDMA_WRITE接收端是否需要ibv_post_recv()取决于发送端send的wr类型如下所示二、内核层收发流程内核有完整的RDMA驱动框架主要实现以下回调函数static int urdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, struct ib_udata *udata) { return 0; } static int urdma_query_port(struct ib_device *ibdev, u32 port_num, struct ib_port_attr *attr) { return 0; } static int urdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) { return 0; } static int urdma_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) { return 0; } static int urdma_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { return 0; } static int urdma_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { return 0; } static int urdma_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) { return 0; } static int urdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { return 0; } static int urdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { return 0; } static int urdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata) { return 0; } static int urdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { return 0; } static int urdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) { return 0; } static int urdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { return 0; } static struct ib_mr *urdma_get_dma_mr(struct ib_pd *ibpd, int access) { struct ib_mr *mr kzalloc(sizeof(*mr), GFP_KERNEL); return mr ? mr : ERR_PTR(-ENOMEM); } static struct ib_mr *urdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_udata *udata) { struct ib_mr *mr kzalloc(sizeof(*mr), GFP_KERNEL); return mr ? mr : ERR_PTR(-ENOMEM); } static int urdma_dereg_mr(struct ib_mr *mr, struct ib_udata *udata) { kfree(mr); return 0; } static int urdma_get_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { return 0; } static int urdma_alloc_ucontext(struct ib_ucontext *ibuc, struct ib_udata *udata) { return 0; } static void urdma_dealloc_ucontext(struct ib_ucontext *ibuc) { return; } static int urdma_query_gid(struct ib_device *ibdev, u32 port_num, int index, union ib_gid *gid) { return 0; } static const struct ib_device_ops urdma_device_ops { .owner THIS_MODULE, .driver_id RDMA_DRIVER_UNKNOWN, .uverbs_abi_ver 1, .query_device urdma_query_device, .query_port urdma_query_port, .alloc_pd urdma_alloc_pd, .dealloc_pd urdma_dealloc_pd, .create_qp urdma_create_qp, .modify_qp urdma_modify_qp, .destroy_qp urdma_destroy_qp, .post_send urdma_post_send, .post_recv urdma_post_recv, .create_cq urdma_create_cq, .destroy_cq urdma_destroy_cq, .poll_cq urdma_poll_cq, .req_notify_cq urdma_req_notify_cq, .get_dma_mr urdma_get_dma_mr, .reg_user_mr urdma_reg_user_mr, .dereg_mr urdma_dereg_mr, .get_port_immutable urdma_get_port_immutable, .alloc_ucontext urdma_alloc_ucontext, .dealloc_ucontext urdma_dealloc_ucontext, .query_gid urdma_query_gid, };三、硬件层收发流程四、应用层与内核层交互13. RDMA之用户态与内核态交互五、RDMA CM接口协议详解librdmacm库QP0UD类型SMISubnet Management Interfaceinfiniband网络专用QP1UD类型GSIGeneral Services InterfaceRoCE网络专用Q_key0x800_1000023. RDMA之基于CM API的QP间建链六、RoCEv2协议详解RoCE V2 协议RDMA-InfiniBand基本传输头BTH分析三RoCE V2协议技术解析七、infiniband协议原文《Infiniband Specification Vol 1-Release-1.4-2020-04-07.pdf》八、UD/RC、send/receive/read/write、MR tableUD不可靠连接一对多没有ACK/NACK支持send/receiveRC可靠连接一对一有ACK/NACK支持send/receive/read/write某厂商仅支持send/receive/writeUD和RC的send/receive原理是一样的只是UD是一对多RC是一对一send发送数据到对端不携带VA/key接收端从CQE获取PAreceive向CQ里插入一个CQECQE里有PAread发起读操作携带VA和key对端在MR_table里查找VA对应的PAwrite发起写操作携带VA和key对端在MR_table里查找VA对应的PA。UD是不可靠连接仅支持send/receive可在ibv接口指定为UD模式UD模式是双边操作receive端在CQ挂载一个CQE里面指定了存储地址该地址需要在MR_table管理范围内send端在SQ上挂载一个WQE传输不需要指定目的地址数据到receive端时receive端从CQ上获取一个CQE读取里面的地址将数据存储到指定地址RC是可靠连接支持send/receive/read/write可在ibv接口指定为RC模式RC模式是单边操作对端不使用CQsend/receive/read/write传输到对端时携带虚拟地址使用虚拟地址在MR_table里查找物理地址比对key、查看权限根据物理地址进行数据传输。MR_table保存在ddribv注册mr时添加一个表项每个表项里面保存有PD/VA/PA/key等信息如下图所示九、问题1. PD号怎么生成和传递的PD值由内核生成本地同步不用传递用于管理本地QP和MR2. QPN怎么生成和传递的QPN在ibv_alloc_qp()时内核选择一个未使用的值rdma_connet()/rdma_accept()时交换另外该值隐藏在rdma_cm_id里3. r_key怎么生成和传递的rdma_connet()/rdma_accept()时交换4. ibv_post_recv数据保存在哪里的直接传入到参数的recv_wr.addr里了RoCE V2 协议rdma-2【RDMA】RDMA read和write编程实例RDMA CM0. 《RDMA杂谈》专栏索引22. RDMA之基于Socket API的QP间建链23. RDMA之基于CM API的QP间建链