libmooshika
Macros | Functions
trans_rdma.c File Reference

rdma helper More...

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <limits.h>
#include <inttypes.h>
#include <endian.h>
#include <errno.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <pthread.h>
#include <semaphore.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/epoll.h>
#include <sys/eventfd.h>
#include <rdma/rdma_cma.h>
#include <netdb.h>
#include "utils.h"
#include "mooshika.h"

Macros

#define EPOLL_MAX_EVENTS   16
 
#define NUM_WQ_PER_POLL   16
 
#define VALGRIND_MAKE_MEM_DEFINED(addr, len)
 
#define THREAD_STACK_SIZE   2116488
 msk_create_thread: Simple wrapper around pthread_create More...
 

Functions

void __attribute__ ((constructor))
 
void __attribute__ ((destructor))
 
struct msk_pdmsk_getpd (struct msk_trans *trans)
 msk_getpd: helper function to get the right pd for a given trans More...
 
struct ibv_mr * msk_reg_mr (struct msk_trans *trans, void *memaddr, size_t size, int access)
 msk_reg_mr: registers memory for rdma use (almost the same as ibv_reg_mr) More...
 
int msk_dereg_mr (struct ibv_mr *mr)
 msk_reg_mr: deregisters memory for rdma use (exactly ibv_dereg_mr) More...
 
msk_rloc_tmsk_make_rloc (struct ibv_mr *mr, uint64_t addr, uint32_t size)
 msk_make_rloc: makes a rkey to send it for remote host use More...
 
void msk_print_devinfo (struct msk_trans *trans)
 
void * msk_stats_thread (void *arg)
 msk_stats_thread: unix socket thread More...
 
void msk_destroy_trans (struct msk_trans **ptrans)
 msk_destroy_trans: disconnects and free trans data More...
 
int msk_init (struct msk_trans **ptrans, struct msk_trans_attr *attr)
 msk_init: part of the init that's the same for client and server More...
 
int msk_bind_server (struct msk_trans *trans)
 msk_bind_server More...
 
int msk_finalize_accept (struct msk_trans *trans)
 msk_finalize_accept: does the real connection acceptance and wait for other side to be ready More...
 
struct msk_transmsk_accept_one_timedwait (struct msk_trans *trans, struct timespec *abstime)
 msk_accept_one: given a listening trans, waits till one connection is requested and accepts it More...
 
struct msk_transmsk_accept_one_wait (struct msk_trans *trans, int msleep)
 
int msk_finalize_connect (struct msk_trans *trans)
 msk_finalize_connect: tells the other side we're ready to receive stuff (does the actual rdma_connect) and waits for its ack More...
 
int msk_connect (struct msk_trans *trans)
 msk_connect: connects a client to a server More...
 
int msk_post_n_recv (struct msk_trans *trans, msk_data_t *data, int num_sge, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg)
 msk_post_n_recv: Post a receive buffer. More...
 
int msk_post_n_send (struct msk_trans *trans, msk_data_t *data, int num_sge, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg)
 Post a send buffer. More...
 
int msk_wait_n_recv (struct msk_trans *trans, msk_data_t *data, int num_sge)
 Post a receive buffer and waits for that one and not any other to be filled. More...
 
int msk_wait_n_send (struct msk_trans *trans, msk_data_t *data, int num_sge)
 Post a send buffer and waits for that one to be completely sent. More...
 
int msk_post_n_read (struct msk_trans *trans, msk_data_t *data, int num_sge, msk_rloc_t *rloc, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg)
 
int msk_post_n_write (struct msk_trans *trans, msk_data_t *data, int num_sge, msk_rloc_t *rloc, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg)
 
int msk_wait_n_read (struct msk_trans *trans, msk_data_t *data, int num_sge, msk_rloc_t *rloc)
 
int msk_wait_n_write (struct msk_trans *trans, msk_data_t *data, int num_sge, msk_rloc_t *rloc)
 
struct sockaddr * msk_get_dst_addr (struct msk_trans *trans)
 
struct sockaddr * msk_get_src_addr (struct msk_trans *trans)
 
uint16_t msk_get_src_port (struct msk_trans *trans)
 
uint16_t msk_get_dst_port (struct msk_trans *trans)
 
const char * msk_wc_status_str (enum ibv_wc_status status)
 

Detailed Description

rdma helper

This is (very) loosely based on a mix of diod, rping (librdmacm/examples) and kernel's net/9p/trans_rdma.c

Macro Definition Documentation

#define EPOLL_MAX_EVENTS   16
#define NUM_WQ_PER_POLL   16
#define THREAD_STACK_SIZE   2116488

msk_create_thread: Simple wrapper around pthread_create

#define VALGRIND_MAKE_MEM_DEFINED (   addr,
  len 
)

Function Documentation

void __attribute__ ( (constructor)  )

References ERROR_LOG.

void __attribute__ ( (destructor)  )
struct msk_trans* msk_accept_one_timedwait ( struct msk_trans trans,
struct timespec *  abstime 
)

msk_accept_one: given a listening trans, waits till one connection is requested and accepts it

Parameters
trans[IN] the parent trans
Returns
a new trans for the child on success, NULL on failure
Todo:
make it return an int an' use trans as argument
Todo:
timeout?

References msk_trans::debug, INFO_LOG, and MSK_DEBUG_EVENT.

struct msk_trans* msk_accept_one_wait ( struct msk_trans trans,
int  msleep 
)

References NSEC_IN_SEC.

int msk_bind_server ( struct msk_trans trans)

msk_bind_server

Parameters
trans[INOUT]
Returns
0 on success, errno value on failure

Referenced by main().

int msk_connect ( struct msk_trans trans)

msk_connect: connects a client to a server

Parameters
trans[INOUT] trans must be init first
Returns
0 on success, the value of errno on error

Referenced by main().

int msk_dereg_mr ( struct ibv_mr *  mr)

msk_reg_mr: deregisters memory for rdma use (exactly ibv_dereg_mr)

Parameters
mr[INOUT] the mr to deregister
Returns
0 on success, errno value on failure

Referenced by handle_trans(), and main().

void msk_destroy_trans ( struct msk_trans **  ptrans)

msk_destroy_trans: disconnects and free trans data

Parameters
ptrans[INOUT] pointer to the trans to destroy
Todo:
check if it is init. if not should just return EINVAL but.. lock.__lock, cond.__lock might work.

References msk_trans::cm_cond, msk_trans::cm_id, msk_trans::cm_lock, msk_trans::debug, INFO_LOG, msk_trans::MSK_CLOSED, msk_trans::MSK_CLOSING, MSK_DEBUG_CM_LOCKS, MSK_DEBUG_SETUP, msk_trans::MSK_ERROR, msk_trans::MSK_LISTENING, and msk_trans::state.

Referenced by handle_trans(), and main().

int msk_finalize_accept ( struct msk_trans trans)

msk_finalize_accept: does the real connection acceptance and wait for other side to be ready

Parameters
trans[IN]
Returns
0 on success, the value of errno on error

Referenced by handle_trans(), and main().

int msk_finalize_connect ( struct msk_trans trans)

msk_finalize_connect: tells the other side we're ready to receive stuff (does the actual rdma_connect) and waits for its ack

Parameters
trans[IN]
Returns
0 on success, errno value on failure

Referenced by handle_trans(), and main().

struct sockaddr* msk_get_dst_addr ( struct msk_trans trans)
uint16_t msk_get_dst_port ( struct msk_trans trans)
struct sockaddr* msk_get_src_addr ( struct msk_trans trans)
uint16_t msk_get_src_port ( struct msk_trans trans)
struct msk_pd* msk_getpd ( struct msk_trans trans)

msk_getpd: helper function to get the right pd for a given trans

Parameters
trans[IN] the connection handle
Returns
NULL if nothing is available, next free pd if none fit, correct one if any match

References atomic_dec, atomic_postinc, msk_trans::cm_id, msk_pd::context, msk_trans::pd, PD_GUARD, and msk_pd::used.

Referenced by msk_reg_mr(), and setup_recv().

int msk_init ( struct msk_trans **  ptrans,
struct msk_trans_attr attr 
)

msk_init: part of the init that's the same for client and server

Parameters
ptrans[INOUT]
attr[IN] attributes to set parameters in ptrans. attr->addr must be set, others can be either 0 or sane values.
Returns
0 on success, errno value on failure

References INFO_LOG, and MSK_DEBUG_EVENT.

Referenced by main().

msk_rloc_t* msk_make_rloc ( struct ibv_mr *  mr,
uint64_t  addr,
uint32_t  size 
)

msk_make_rloc: makes a rkey to send it for remote host use

Parameters
mr[IN] the mr in which the addr belongs
addr[IN] the addr to give
size[IN] the size to allow (hint)
Returns
a pointer to the rkey on success, NULL on failure.

References INFO_LOG, MSK_DEBUG_EVENT, msk_rloc::raddr, msk_rloc::rkey, and msk_rloc::size.

Referenced by main().

int msk_post_n_read ( struct msk_trans trans,
msk_data_t data,
int  num_sge,
msk_rloc_t rloc,
ctx_callback_t  callback,
ctx_callback_t  err_callback,
void *  callback_arg 
)
int msk_post_n_recv ( struct msk_trans trans,
msk_data_t data,
int  num_sge,
ctx_callback_t  callback,
ctx_callback_t  err_callback,
void *  callback_arg 
)

msk_post_n_recv: Post a receive buffer.

Need to post recv buffers before the opposite side tries to send anything!

Parameters
trans[IN]
data[OUT] the data buffer to be filled with received data
num_sge[IN] the number of elements in data to register
callback[IN] function that'll be called when done
err_callback[IN] function that'll be called on error
callback_arg[IN] argument to give to the callback
Returns
0 on success, the value of errno on error
Todo:
np_uerror(ret)

Referenced by main().

int msk_post_n_send ( struct msk_trans trans,
msk_data_t data,
int  num_sge,
ctx_callback_t  callback,
ctx_callback_t  err_callback,
void *  callback_arg 
)

Post a send buffer.

Parameters
trans[IN]
data[IN] the data buffer to be sent
num_sge[IN] the number of elements in data to send
callback[IN] function that'll be called when done
err_callback[IN] function that'll be called on error
callback_arg[IN] argument to give to the callback
Returns
0 on success, the value of errno on error

Referenced by main().

int msk_post_n_write ( struct msk_trans trans,
msk_data_t data,
int  num_sge,
msk_rloc_t rloc,
ctx_callback_t  callback,
ctx_callback_t  err_callback,
void *  callback_arg 
)
void msk_print_devinfo ( struct msk_trans trans)

References msk_trans::cm_id.

struct ibv_mr* msk_reg_mr ( struct msk_trans trans,
void *  memaddr,
size_t  size,
int  access 
)

msk_reg_mr: registers memory for rdma use (almost the same as ibv_reg_mr)

Parameters
trans[IN]
memaddr[IN] the address to register
size[IN] the size of the area to register
access[IN] the access to grants to the mr (e.g. IBV_ACCESS_LOCAL_WRITE)
Returns
a pointer to the mr if registered correctly or NULL on failure

References msk_getpd(), msk_pd::pd, and msk_pd::used.

Referenced by handle_trans(), main(), and post_recvs().

void* msk_stats_thread ( void *  arg)
int msk_wait_n_read ( struct msk_trans trans,
msk_data_t data,
int  num_sge,
msk_rloc_t rloc 
)
int msk_wait_n_recv ( struct msk_trans trans,
msk_data_t data,
int  num_sge 
)

Post a receive buffer and waits for that one and not any other to be filled.

Generally a bad idea to use that one unless only that one is used.

Parameters
trans[IN]
data[OUT] the data buffer to be filled with the received data
num_sge[IN] the number of elements in data to register
Returns
0 on success, the value of errno on error
int msk_wait_n_send ( struct msk_trans trans,
msk_data_t data,
int  num_sge 
)

Post a send buffer and waits for that one to be completely sent.

Parameters
trans[IN]
data[IN] the data to send
num_sge[IN] the number of elements in data to send
Returns
0 on success, the value of errno on error
int msk_wait_n_write ( struct msk_trans trans,
msk_data_t data,
int  num_sge,
msk_rloc_t rloc 
)
const char* msk_wc_status_str ( enum ibv_wc_status  status)