libmooshika
Data Structures | Macros | Typedefs | Functions
mooshika.h File Reference

rdma helper include file More...

#include <rdma/rdma_cma.h>

Go to the source code of this file.

Data Structures

struct  msk_data
 data size and content to send/just received More...
 
union  sockaddr_union
 
struct  msk_stats
 
struct  msk_pd
 
struct  msk_trans
 RDMA transport instance. More...
 
struct  msk_trans_attr
 
struct  msk_rloc
 stores one remote address to write/read at More...
 

Macros

#define MOOSHIKA_API_VERSION   5
 
#define PD_GUARD   ((void*)-1)
 
#define MSK_CLIENT   0
 
#define MSK_SERVER_CHILD   -1
 
#define MSK_DEBUG_EVENT   0x0001
 
#define MSK_DEBUG_SETUP   0x0002
 
#define MSK_DEBUG_SEND   0x0004
 
#define MSK_DEBUG_RECV   0x0008
 
#define MSK_DEBUG_WORKERS   (MSK_DEBUG_SEND | MSK_DEBUG_RECV)
 
#define MSK_DEBUG_CM_LOCKS   0x0010
 
#define MSK_DEBUG_CTX   0x0020
 
#define MSK_DEBUG_SPEED   0x8000
 

Typedefs

typedef struct msk_trans msk_trans_t
 
typedef struct msk_trans_attr msk_trans_attr_t
 
typedef struct msk_data msk_data_t
 
typedef union sockaddr_union sockaddr_union_t
 
typedef void(* disconnect_callback_t) (msk_trans_t *trans)
 
typedef void(* ctx_callback_t) (msk_trans_t *trans, msk_data_t *data, void *arg)
 
typedef struct msk_rloc msk_rloc_t
 

Functions

int msk_post_n_recv (msk_trans_t *trans, msk_data_t *data, int num_sge, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg)
 msk_post_n_recv: Post a receive buffer. More...
 
int msk_post_n_send (msk_trans_t *trans, msk_data_t *data, int num_sge, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg)
 Post a send buffer. More...
 
int msk_wait_n_recv (msk_trans_t *trans, msk_data_t *data, int num_sge)
 Post a receive buffer and waits for that one and not any other to be filled. More...
 
int msk_wait_n_send (msk_trans_t *trans, msk_data_t *data, int num_sge)
 Post a send buffer and waits for that one to be completely sent. More...
 
int msk_post_n_read (msk_trans_t *trans, msk_data_t *data, int num_sge, msk_rloc_t *rloc, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg)
 
int msk_post_n_write (msk_trans_t *trans, msk_data_t *data, int num_sge, msk_rloc_t *rloc, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg)
 
int msk_wait_n_read (msk_trans_t *trans, msk_data_t *data, int num_sge, msk_rloc_t *rloc)
 
int msk_wait_n_write (msk_trans_t *trans, msk_data_t *data, int num_sge, msk_rloc_t *rloc)
 
int msk_init (msk_trans_t **ptrans, msk_trans_attr_t *attr)
 msk_init: part of the init that's the same for client and server More...
 
int msk_bind_server (msk_trans_t *trans)
 msk_bind_server More...
 
msk_trans_tmsk_accept_one_wait (msk_trans_t *trans, int msleep)
 
msk_trans_tmsk_accept_one_timedwait (msk_trans_t *trans, struct timespec *abstime)
 msk_accept_one: given a listening trans, waits till one connection is requested and accepts it More...
 
int msk_finalize_accept (msk_trans_t *trans)
 msk_finalize_accept: does the real connection acceptance and wait for other side to be ready More...
 
void msk_destroy_trans (msk_trans_t **ptrans)
 msk_destroy_trans: disconnects and free trans data More...
 
int msk_connect (msk_trans_t *trans)
 msk_connect: connects a client to a server More...
 
int msk_finalize_connect (msk_trans_t *trans)
 msk_finalize_connect: tells the other side we're ready to receive stuff (does the actual rdma_connect) and waits for its ack More...
 
struct ibv_mr * msk_reg_mr (msk_trans_t *trans, void *memaddr, size_t size, int access)
 msk_reg_mr: registers memory for rdma use (almost the same as ibv_reg_mr) More...
 
int msk_dereg_mr (struct ibv_mr *mr)
 msk_reg_mr: deregisters memory for rdma use (exactly ibv_dereg_mr) More...
 
msk_rloc_tmsk_make_rloc (struct ibv_mr *mr, uint64_t addr, uint32_t size)
 msk_make_rloc: makes a rkey to send it for remote host use More...
 
void msk_print_devinfo (msk_trans_t *trans)
 
struct sockaddr * msk_get_dst_addr (msk_trans_t *trans)
 
struct sockaddr * msk_get_src_addr (msk_trans_t *trans)
 
uint16_t msk_get_src_port (msk_trans_t *trans)
 
uint16_t msk_get_dst_port (msk_trans_t *trans)
 
struct msk_pdmsk_getpd (msk_trans_t *trans)
 msk_getpd: helper function to get the right pd for a given trans More...
 
const char * msk_wc_status_str (enum ibv_wc_status status)
 

Detailed Description

rdma helper include file

This is (very) loosely based on a mix of diod, rping (librdmacm/examples) and kernel's net/9p/trans_rdma.c

Macro Definition Documentation

#define MOOSHIKA_API_VERSION   5
#define MSK_CLIENT   0
#define MSK_DEBUG_CM_LOCKS   0x0010
#define MSK_DEBUG_CTX   0x0020
#define MSK_DEBUG_EVENT   0x0001
#define MSK_DEBUG_RECV   0x0008

Referenced by main().

#define MSK_DEBUG_SEND   0x0004

Referenced by main().

#define MSK_DEBUG_SETUP   0x0002

Referenced by msk_destroy_trans().

#define MSK_DEBUG_SPEED   0x8000

Referenced by main().

#define MSK_DEBUG_WORKERS   (MSK_DEBUG_SEND | MSK_DEBUG_RECV)
#define MSK_SERVER_CHILD   -1
#define PD_GUARD   ((void*)-1)

Referenced by msk_getpd().

Typedef Documentation

typedef void(* ctx_callback_t) (msk_trans_t *trans, msk_data_t *data, void *arg)
typedef void(* disconnect_callback_t) (msk_trans_t *trans)
typedef struct msk_data msk_data_t
typedef struct msk_rloc msk_rloc_t
typedef struct msk_trans msk_trans_t

Function Documentation

msk_trans_t* msk_accept_one_timedwait ( struct msk_trans trans,
struct timespec *  abstime 
)

msk_accept_one: given a listening trans, waits till one connection is requested and accepts it

Parameters
trans[IN] the parent trans
Returns
a new trans for the child on success, NULL on failure
Todo:
make it return an int an' use trans as argument
Todo:
timeout?

References msk_trans::debug, INFO_LOG, and MSK_DEBUG_EVENT.

msk_trans_t* msk_accept_one_wait ( msk_trans_t trans,
int  msleep 
)

References NSEC_IN_SEC.

int msk_bind_server ( struct msk_trans trans)

msk_bind_server

Parameters
trans[INOUT]
Returns
0 on success, errno value on failure

Referenced by main().

int msk_connect ( struct msk_trans trans)

msk_connect: connects a client to a server

Parameters
trans[INOUT] trans must be init first
Returns
0 on success, the value of errno on error

Referenced by main().

int msk_dereg_mr ( struct ibv_mr *  mr)

msk_reg_mr: deregisters memory for rdma use (exactly ibv_dereg_mr)

Parameters
mr[INOUT] the mr to deregister
Returns
0 on success, errno value on failure

Referenced by handle_trans(), and main().

void msk_destroy_trans ( struct msk_trans **  ptrans)

msk_destroy_trans: disconnects and free trans data

Parameters
ptrans[INOUT] pointer to the trans to destroy
Todo:
check if it is init. if not should just return EINVAL but.. lock.__lock, cond.__lock might work.

References msk_trans::cm_cond, msk_trans::cm_id, msk_trans::cm_lock, msk_trans::debug, INFO_LOG, msk_trans::MSK_CLOSED, msk_trans::MSK_CLOSING, MSK_DEBUG_CM_LOCKS, MSK_DEBUG_SETUP, msk_trans::MSK_ERROR, msk_trans::MSK_LISTENING, and msk_trans::state.

Referenced by handle_trans(), and main().

int msk_finalize_accept ( struct msk_trans trans)

msk_finalize_accept: does the real connection acceptance and wait for other side to be ready

Parameters
trans[IN]
Returns
0 on success, the value of errno on error

Referenced by handle_trans(), and main().

int msk_finalize_connect ( struct msk_trans trans)

msk_finalize_connect: tells the other side we're ready to receive stuff (does the actual rdma_connect) and waits for its ack

Parameters
trans[IN]
Returns
0 on success, errno value on failure

Referenced by handle_trans(), and main().

struct sockaddr* msk_get_dst_addr ( msk_trans_t trans)
uint16_t msk_get_dst_port ( msk_trans_t trans)
struct sockaddr* msk_get_src_addr ( msk_trans_t trans)
uint16_t msk_get_src_port ( msk_trans_t trans)
struct msk_pd* msk_getpd ( struct msk_trans trans)

msk_getpd: helper function to get the right pd for a given trans

Parameters
trans[IN] the connection handle
Returns
NULL if nothing is available, next free pd if none fit, correct one if any match

References atomic_dec, atomic_postinc, msk_trans::cm_id, msk_pd::context, msk_trans::pd, PD_GUARD, and msk_pd::used.

Referenced by msk_reg_mr(), and setup_recv().

int msk_init ( struct msk_trans **  ptrans,
struct msk_trans_attr attr 
)

msk_init: part of the init that's the same for client and server

Parameters
ptrans[INOUT]
attr[IN] attributes to set parameters in ptrans. attr->addr must be set, others can be either 0 or sane values.
Returns
0 on success, errno value on failure

References INFO_LOG, and MSK_DEBUG_EVENT.

Referenced by main().

msk_rloc_t* msk_make_rloc ( struct ibv_mr *  mr,
uint64_t  addr,
uint32_t  size 
)

msk_make_rloc: makes a rkey to send it for remote host use

Parameters
mr[IN] the mr in which the addr belongs
addr[IN] the addr to give
size[IN] the size to allow (hint)
Returns
a pointer to the rkey on success, NULL on failure.

References INFO_LOG, MSK_DEBUG_EVENT, msk_rloc::raddr, msk_rloc::rkey, and msk_rloc::size.

Referenced by main().

int msk_post_n_read ( msk_trans_t trans,
msk_data_t data,
int  num_sge,
msk_rloc_t rloc,
ctx_callback_t  callback,
ctx_callback_t  err_callback,
void *  callback_arg 
)
int msk_post_n_recv ( struct msk_trans trans,
msk_data_t data,
int  num_sge,
ctx_callback_t  callback,
ctx_callback_t  err_callback,
void *  callback_arg 
)

msk_post_n_recv: Post a receive buffer.

Need to post recv buffers before the opposite side tries to send anything!

Parameters
trans[IN]
data[OUT] the data buffer to be filled with received data
num_sge[IN] the number of elements in data to register
callback[IN] function that'll be called when done
err_callback[IN] function that'll be called on error
callback_arg[IN] argument to give to the callback
Returns
0 on success, the value of errno on error
Todo:
np_uerror(ret)

Referenced by main().

int msk_post_n_send ( struct msk_trans trans,
msk_data_t data,
int  num_sge,
ctx_callback_t  callback,
ctx_callback_t  err_callback,
void *  callback_arg 
)

Post a send buffer.

Parameters
trans[IN]
data[IN] the data buffer to be sent
num_sge[IN] the number of elements in data to send
callback[IN] function that'll be called when done
err_callback[IN] function that'll be called on error
callback_arg[IN] argument to give to the callback
Returns
0 on success, the value of errno on error

Referenced by main().

int msk_post_n_write ( msk_trans_t trans,
msk_data_t data,
int  num_sge,
msk_rloc_t rloc,
ctx_callback_t  callback,
ctx_callback_t  err_callback,
void *  callback_arg 
)
void msk_print_devinfo ( msk_trans_t trans)

References msk_trans::cm_id.

struct ibv_mr* msk_reg_mr ( struct msk_trans trans,
void *  memaddr,
size_t  size,
int  access 
)

msk_reg_mr: registers memory for rdma use (almost the same as ibv_reg_mr)

Parameters
trans[IN]
memaddr[IN] the address to register
size[IN] the size of the area to register
access[IN] the access to grants to the mr (e.g. IBV_ACCESS_LOCAL_WRITE)
Returns
a pointer to the mr if registered correctly or NULL on failure

References msk_getpd(), msk_pd::pd, and msk_pd::used.

Referenced by handle_trans(), main(), and post_recvs().

int msk_wait_n_read ( msk_trans_t trans,
msk_data_t data,
int  num_sge,
msk_rloc_t rloc 
)
int msk_wait_n_recv ( struct msk_trans trans,
msk_data_t data,
int  num_sge 
)

Post a receive buffer and waits for that one and not any other to be filled.

Generally a bad idea to use that one unless only that one is used.

Parameters
trans[IN]
data[OUT] the data buffer to be filled with the received data
num_sge[IN] the number of elements in data to register
Returns
0 on success, the value of errno on error
int msk_wait_n_send ( struct msk_trans trans,
msk_data_t data,
int  num_sge 
)

Post a send buffer and waits for that one to be completely sent.

Parameters
trans[IN]
data[IN] the data to send
num_sge[IN] the number of elements in data to send
Returns
0 on success, the value of errno on error
int msk_wait_n_write ( msk_trans_t trans,
msk_data_t data,
int  num_sge,
msk_rloc_t rloc 
)
const char* msk_wc_status_str ( enum ibv_wc_status  status)