Commit ef6d4ccd authored by Yuval Shaia's avatar Yuval Shaia Committed by Marcel Apfelbaum

hw/rdma: Implementation of generic rdma device layers

This layer is composed of two sub-modules, backend and resource manager.
Backend sub-module is responsible for all the interaction with IB layers
such as ibverbs and umad (external libraries).
Resource manager is a collection of functions and structures to manage
RDMA resources such as QPs, CQs and MRs.
Reviewed-by: 's avatarDotan Barak <>
Reviewed-by: 's avatarZhu Yanjun <>
Signed-off-by: 's avatarYuval Shaia <>
Signed-off-by: 's avatarMarcel Apfelbaum <>
parent b3a92277
......@@ -130,6 +130,7 @@ trace-events-subdirs += hw/block/dataplane
trace-events-subdirs += hw/char
trace-events-subdirs += hw/intc
trace-events-subdirs += hw/net
trace-events-subdirs += hw/rdma
trace-events-subdirs += hw/virtio
trace-events-subdirs += hw/audio
trace-events-subdirs += hw/misc
......@@ -1572,7 +1572,7 @@ disabled with --disable-FEATURE, default is enabled if available:
hax HAX acceleration support
hvf Hypervisor.framework acceleration support
whpx Windows Hypervisor Platform acceleration support
rdma RDMA-based migration support
rdma Enable RDMA-based migration and PVRDMA support
vde support for vde network
netmap support for netmap network
linux-aio Linux AIO support
......@@ -2923,15 +2923,16 @@ if test "$rdma" != "no" ; then
#include <rdma/rdma_cma.h>
int main(void) { return 0; }
rdma_libs="-lrdmacm -libverbs"
rdma_libs="-lrdmacm -libverbs -libumad"
if compile_prog "" "$rdma_libs" ; then
libs_softmmu="$libs_softmmu $rdma_libs"
if test "$rdma" = "yes" ; then
error_exit \
" OpenFabrics librdmacm/libibverbs not present." \
" OpenFabrics librdmacm/libibverbs/libibumad not present." \
" Your options:" \
" (1) Fast: Install infiniband packages from your distro." \
" (1) Fast: Install infiniband packages (devel) from your distro." \
" (2) Cleanest: Install libraries from" \
" (3) Also: Install softiwarp if you don't have RDMA hardware"
ifeq ($(CONFIG_RDMA),y)
obj-$(CONFIG_PCI) += rdma_utils.o
obj-$(CONFIG_PCI) += rdma_utils.o rdma_backend.o rdma_rm.o
This diff is collapsed.
* RDMA device: Definitions of Backend Device functions
* Copyright (C) 2018 Oracle
* Copyright (C) 2018 Red Hat Inc
* Authors:
* Yuval Shaia <>
* Marcel Apfelbaum <>
* This work is licensed under the terms of the GNU GPL, version 2 or later.
* See the COPYING file in the top-level directory.
#include <qapi/error.h>
#include "rdma_rm_defs.h"
#include "rdma_backend_defs.h"
/* Add definition for QP0 and QP1 as there is no userspace enums for them */
enum ibv_special_qp_type {
static inline union ibv_gid *rdma_backend_gid(RdmaBackendDev *dev)
return &dev->gid;
static inline uint32_t rdma_backend_qpn(const RdmaBackendQP *qp)
return qp->ibqp ? qp->ibqp->qp_num : 0;
static inline uint32_t rdma_backend_mr_lkey(const RdmaBackendMR *mr)
return mr->ibmr ? mr->ibmr->lkey : 0;
static inline uint32_t rdma_backend_mr_rkey(const RdmaBackendMR *mr)
return mr->ibmr ? mr->ibmr->rkey : 0;
int rdma_backend_init(RdmaBackendDev *backend_dev,
RdmaDeviceResources *rdma_dev_res,
const char *backend_device_name, uint8_t port_num,
uint8_t backend_gid_idx, struct ibv_device_attr *dev_attr,
Error **errp);
void rdma_backend_fini(RdmaBackendDev *backend_dev);
void rdma_backend_register_comp_handler(void (*handler)(int status,
unsigned int vendor_err, void *ctx));
void rdma_backend_unregister_comp_handler(void);
int rdma_backend_query_port(RdmaBackendDev *backend_dev,
struct ibv_port_attr *port_attr);
int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd);
void rdma_backend_destroy_pd(RdmaBackendPD *pd);
int rdma_backend_create_mr(RdmaBackendMR *mr, RdmaBackendPD *pd, uint64_t addr,
size_t length, int access);
void rdma_backend_destroy_mr(RdmaBackendMR *mr);
int rdma_backend_create_cq(RdmaBackendDev *backend_dev, RdmaBackendCQ *cq,
int cqe);
void rdma_backend_destroy_cq(RdmaBackendCQ *cq);
void rdma_backend_poll_cq(RdmaDeviceResources *rdma_dev_res, RdmaBackendCQ *cq);
int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
RdmaBackendPD *pd, RdmaBackendCQ *scq,
RdmaBackendCQ *rcq, uint32_t max_send_wr,
uint32_t max_recv_wr, uint32_t max_send_sge,
uint32_t max_recv_sge);
int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
uint8_t qp_type, uint32_t qkey);
int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
uint8_t qp_type, union ibv_gid *dgid,
uint32_t dqpn, uint32_t rq_psn, uint32_t qkey,
bool use_qkey);
int rdma_backend_qp_state_rts(RdmaBackendQP *qp, uint8_t qp_type,
uint32_t sq_psn, uint32_t qkey, bool use_qkey);
void rdma_backend_destroy_qp(RdmaBackendQP *qp);
void rdma_backend_post_send(RdmaBackendDev *backend_dev,
RdmaBackendQP *qp, uint8_t qp_type,
struct ibv_sge *sge, uint32_t num_sge,
union ibv_gid *dgid, uint32_t dqpn, uint32_t dqkey,
void *ctx);
void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
RdmaDeviceResources *rdma_dev_res,
RdmaBackendQP *qp, uint8_t qp_type,
struct ibv_sge *sge, uint32_t num_sge, void *ctx);
This diff is collapsed.
* RDMA device: Definitions of Resource Manager functions
* Copyright (C) 2018 Oracle
* Copyright (C) 2018 Red Hat Inc
* Authors:
* Yuval Shaia <>
* Marcel Apfelbaum <>
* This work is licensed under the terms of the GNU GPL, version 2 or later.
* See the COPYING file in the top-level directory.
#ifndef RDMA_RM_H
#define RDMA_RM_H
#include <qapi/error.h>
#include "rdma_backend_defs.h"
#include "rdma_rm_defs.h"
int rdma_rm_init(RdmaDeviceResources *dev_res, struct ibv_device_attr *dev_attr,
Error **errp);
void rdma_rm_fini(RdmaDeviceResources *dev_res);
int rdma_rm_alloc_pd(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev,
uint32_t *pd_handle, uint32_t ctx_handle);
RdmaRmPD *rdma_rm_get_pd(RdmaDeviceResources *dev_res, uint32_t pd_handle);
void rdma_rm_dealloc_pd(RdmaDeviceResources *dev_res, uint32_t pd_handle);
int rdma_rm_alloc_mr(RdmaDeviceResources *dev_res, uint32_t pd_handle,
uint64_t guest_start, size_t guest_length, void *host_virt,
int access_flags, uint32_t *mr_handle, uint32_t *lkey,
uint32_t *rkey);
RdmaRmMR *rdma_rm_get_mr(RdmaDeviceResources *dev_res, uint32_t mr_handle);
void rdma_rm_dealloc_mr(RdmaDeviceResources *dev_res, uint32_t mr_handle);
int rdma_rm_alloc_uc(RdmaDeviceResources *dev_res, uint32_t pfn,
uint32_t *uc_handle);
RdmaRmUC *rdma_rm_get_uc(RdmaDeviceResources *dev_res, uint32_t uc_handle);
void rdma_rm_dealloc_uc(RdmaDeviceResources *dev_res, uint32_t uc_handle);
int rdma_rm_alloc_cq(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev,
uint32_t cqe, uint32_t *cq_handle, void *opaque);
RdmaRmCQ *rdma_rm_get_cq(RdmaDeviceResources *dev_res, uint32_t cq_handle);
void rdma_rm_req_notify_cq(RdmaDeviceResources *dev_res, uint32_t cq_handle,
bool notify);
void rdma_rm_dealloc_cq(RdmaDeviceResources *dev_res, uint32_t cq_handle);
int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res, uint32_t pd_handle,
uint8_t qp_type, uint32_t max_send_wr,
uint32_t max_send_sge, uint32_t send_cq_handle,
uint32_t max_recv_wr, uint32_t max_recv_sge,
uint32_t recv_cq_handle, void *opaque, uint32_t *qpn);
RdmaRmQP *rdma_rm_get_qp(RdmaDeviceResources *dev_res, uint32_t qpn);
int rdma_rm_modify_qp(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev,
uint32_t qp_handle, uint32_t attr_mask,
union ibv_gid *dgid, uint32_t dqpn,
enum ibv_qp_state qp_state, uint32_t qkey,
uint32_t rq_psn, uint32_t sq_psn);
void rdma_rm_dealloc_qp(RdmaDeviceResources *dev_res, uint32_t qp_handle);
int rdma_rm_alloc_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t *cqe_ctx_id,
void *ctx);
void *rdma_rm_get_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id);
void rdma_rm_dealloc_cqe_ctx(RdmaDeviceResources *dev_res, uint32_t cqe_ctx_id);
# See docs/tracing.txt for syntax documentation.
create_ah_cache_hit(uint64_t subnet, uint64_t net_id) "subnet = 0x%"PRIx64" net_id = 0x%"PRIx64
create_ah_cache_miss(uint64_t subnet, uint64_t net_id) "subnet = 0x%"PRIx64" net_id = 0x%"PRIx64
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment