libmooshika
|
rdma helper More...
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <limits.h>
#include <inttypes.h>
#include <endian.h>
#include <errno.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <pthread.h>
#include <semaphore.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/epoll.h>
#include <sys/eventfd.h>
#include <rdma/rdma_cma.h>
#include <netdb.h>
#include "utils.h"
#include "mooshika.h"
Macros | |
#define | EPOLL_MAX_EVENTS 16 |
#define | NUM_WQ_PER_POLL 16 |
#define | VALGRIND_MAKE_MEM_DEFINED(addr, len) |
#define | THREAD_STACK_SIZE 2116488 |
msk_create_thread: Simple wrapper around pthread_create More... | |
Functions | |
void | __attribute__ ((constructor)) |
void | __attribute__ ((destructor)) |
struct msk_pd * | msk_getpd (struct msk_trans *trans) |
msk_getpd: helper function to get the right pd for a given trans More... | |
struct ibv_mr * | msk_reg_mr (struct msk_trans *trans, void *memaddr, size_t size, int access) |
msk_reg_mr: registers memory for rdma use (almost the same as ibv_reg_mr) More... | |
int | msk_dereg_mr (struct ibv_mr *mr) |
msk_reg_mr: deregisters memory for rdma use (exactly ibv_dereg_mr) More... | |
msk_rloc_t * | msk_make_rloc (struct ibv_mr *mr, uint64_t addr, uint32_t size) |
msk_make_rloc: makes a rkey to send it for remote host use More... | |
void | msk_print_devinfo (struct msk_trans *trans) |
void * | msk_stats_thread (void *arg) |
msk_stats_thread: unix socket thread More... | |
void | msk_destroy_trans (struct msk_trans **ptrans) |
msk_destroy_trans: disconnects and free trans data More... | |
int | msk_init (struct msk_trans **ptrans, struct msk_trans_attr *attr) |
msk_init: part of the init that's the same for client and server More... | |
int | msk_bind_server (struct msk_trans *trans) |
msk_bind_server More... | |
int | msk_finalize_accept (struct msk_trans *trans) |
msk_finalize_accept: does the real connection acceptance and wait for other side to be ready More... | |
struct msk_trans * | msk_accept_one_timedwait (struct msk_trans *trans, struct timespec *abstime) |
msk_accept_one: given a listening trans, waits till one connection is requested and accepts it More... | |
struct msk_trans * | msk_accept_one_wait (struct msk_trans *trans, int msleep) |
int | msk_finalize_connect (struct msk_trans *trans) |
msk_finalize_connect: tells the other side we're ready to receive stuff (does the actual rdma_connect) and waits for its ack More... | |
int | msk_connect (struct msk_trans *trans) |
msk_connect: connects a client to a server More... | |
int | msk_post_n_recv (struct msk_trans *trans, msk_data_t *data, int num_sge, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg) |
msk_post_n_recv: Post a receive buffer. More... | |
int | msk_post_n_send (struct msk_trans *trans, msk_data_t *data, int num_sge, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg) |
Post a send buffer. More... | |
int | msk_wait_n_recv (struct msk_trans *trans, msk_data_t *data, int num_sge) |
Post a receive buffer and waits for that one and not any other to be filled. More... | |
int | msk_wait_n_send (struct msk_trans *trans, msk_data_t *data, int num_sge) |
Post a send buffer and waits for that one to be completely sent. More... | |
int | msk_post_n_read (struct msk_trans *trans, msk_data_t *data, int num_sge, msk_rloc_t *rloc, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg) |
int | msk_post_n_write (struct msk_trans *trans, msk_data_t *data, int num_sge, msk_rloc_t *rloc, ctx_callback_t callback, ctx_callback_t err_callback, void *callback_arg) |
int | msk_wait_n_read (struct msk_trans *trans, msk_data_t *data, int num_sge, msk_rloc_t *rloc) |
int | msk_wait_n_write (struct msk_trans *trans, msk_data_t *data, int num_sge, msk_rloc_t *rloc) |
struct sockaddr * | msk_get_dst_addr (struct msk_trans *trans) |
struct sockaddr * | msk_get_src_addr (struct msk_trans *trans) |
uint16_t | msk_get_src_port (struct msk_trans *trans) |
uint16_t | msk_get_dst_port (struct msk_trans *trans) |
const char * | msk_wc_status_str (enum ibv_wc_status status) |
rdma helper
This is (very) loosely based on a mix of diod, rping (librdmacm/examples) and kernel's net/9p/trans_rdma.c
#define EPOLL_MAX_EVENTS 16 |
#define NUM_WQ_PER_POLL 16 |
#define THREAD_STACK_SIZE 2116488 |
msk_create_thread: Simple wrapper around pthread_create
#define VALGRIND_MAKE_MEM_DEFINED | ( | addr, | |
len | |||
) |
void __attribute__ | ( | (constructor) | ) |
References ERROR_LOG.
void __attribute__ | ( | (destructor) | ) |
msk_accept_one: given a listening trans, waits till one connection is requested and accepts it
trans | [IN] the parent trans |
References msk_trans::debug, INFO_LOG, and MSK_DEBUG_EVENT.
References NSEC_IN_SEC.
int msk_bind_server | ( | struct msk_trans * | trans | ) |
msk_bind_server
trans | [INOUT] |
Referenced by main().
int msk_connect | ( | struct msk_trans * | trans | ) |
msk_connect: connects a client to a server
trans | [INOUT] trans must be init first |
Referenced by main().
int msk_dereg_mr | ( | struct ibv_mr * | mr | ) |
msk_reg_mr: deregisters memory for rdma use (exactly ibv_dereg_mr)
mr | [INOUT] the mr to deregister |
Referenced by handle_trans(), and main().
void msk_destroy_trans | ( | struct msk_trans ** | ptrans | ) |
msk_destroy_trans: disconnects and free trans data
ptrans | [INOUT] pointer to the trans to destroy |
References msk_trans::cm_cond, msk_trans::cm_id, msk_trans::cm_lock, msk_trans::debug, INFO_LOG, msk_trans::MSK_CLOSED, msk_trans::MSK_CLOSING, MSK_DEBUG_CM_LOCKS, MSK_DEBUG_SETUP, msk_trans::MSK_ERROR, msk_trans::MSK_LISTENING, and msk_trans::state.
Referenced by handle_trans(), and main().
int msk_finalize_accept | ( | struct msk_trans * | trans | ) |
msk_finalize_accept: does the real connection acceptance and wait for other side to be ready
trans | [IN] |
Referenced by handle_trans(), and main().
int msk_finalize_connect | ( | struct msk_trans * | trans | ) |
msk_finalize_connect: tells the other side we're ready to receive stuff (does the actual rdma_connect) and waits for its ack
trans | [IN] |
Referenced by handle_trans(), and main().
struct sockaddr* msk_get_dst_addr | ( | struct msk_trans * | trans | ) |
uint16_t msk_get_dst_port | ( | struct msk_trans * | trans | ) |
struct sockaddr* msk_get_src_addr | ( | struct msk_trans * | trans | ) |
uint16_t msk_get_src_port | ( | struct msk_trans * | trans | ) |
msk_getpd: helper function to get the right pd for a given trans
trans | [IN] the connection handle |
References atomic_dec, atomic_postinc, msk_trans::cm_id, msk_pd::context, msk_trans::pd, PD_GUARD, and msk_pd::used.
Referenced by msk_reg_mr(), and setup_recv().
int msk_init | ( | struct msk_trans ** | ptrans, |
struct msk_trans_attr * | attr | ||
) |
msk_init: part of the init that's the same for client and server
ptrans | [INOUT] |
attr | [IN] attributes to set parameters in ptrans. attr->addr must be set, others can be either 0 or sane values. |
References INFO_LOG, and MSK_DEBUG_EVENT.
Referenced by main().
msk_rloc_t* msk_make_rloc | ( | struct ibv_mr * | mr, |
uint64_t | addr, | ||
uint32_t | size | ||
) |
msk_make_rloc: makes a rkey to send it for remote host use
mr | [IN] the mr in which the addr belongs |
addr | [IN] the addr to give |
size | [IN] the size to allow (hint) |
References INFO_LOG, MSK_DEBUG_EVENT, msk_rloc::raddr, msk_rloc::rkey, and msk_rloc::size.
Referenced by main().
int msk_post_n_read | ( | struct msk_trans * | trans, |
msk_data_t * | data, | ||
int | num_sge, | ||
msk_rloc_t * | rloc, | ||
ctx_callback_t | callback, | ||
ctx_callback_t | err_callback, | ||
void * | callback_arg | ||
) |
int msk_post_n_recv | ( | struct msk_trans * | trans, |
msk_data_t * | data, | ||
int | num_sge, | ||
ctx_callback_t | callback, | ||
ctx_callback_t | err_callback, | ||
void * | callback_arg | ||
) |
msk_post_n_recv: Post a receive buffer.
Need to post recv buffers before the opposite side tries to send anything!
trans | [IN] |
data | [OUT] the data buffer to be filled with received data |
num_sge | [IN] the number of elements in data to register |
callback | [IN] function that'll be called when done |
err_callback | [IN] function that'll be called on error |
callback_arg | [IN] argument to give to the callback |
Referenced by main().
int msk_post_n_send | ( | struct msk_trans * | trans, |
msk_data_t * | data, | ||
int | num_sge, | ||
ctx_callback_t | callback, | ||
ctx_callback_t | err_callback, | ||
void * | callback_arg | ||
) |
Post a send buffer.
trans | [IN] |
data | [IN] the data buffer to be sent |
num_sge | [IN] the number of elements in data to send |
callback | [IN] function that'll be called when done |
err_callback | [IN] function that'll be called on error |
callback_arg | [IN] argument to give to the callback |
Referenced by main().
int msk_post_n_write | ( | struct msk_trans * | trans, |
msk_data_t * | data, | ||
int | num_sge, | ||
msk_rloc_t * | rloc, | ||
ctx_callback_t | callback, | ||
ctx_callback_t | err_callback, | ||
void * | callback_arg | ||
) |
References msk_trans::debug, and MSK_DEBUG_CM_LOCKS.
void msk_print_devinfo | ( | struct msk_trans * | trans | ) |
References msk_trans::cm_id.
struct ibv_mr* msk_reg_mr | ( | struct msk_trans * | trans, |
void * | memaddr, | ||
size_t | size, | ||
int | access | ||
) |
msk_reg_mr: registers memory for rdma use (almost the same as ibv_reg_mr)
trans | [IN] |
memaddr | [IN] the address to register |
size | [IN] the size of the area to register |
access | [IN] the access to grants to the mr (e.g. IBV_ACCESS_LOCAL_WRITE) |
References msk_getpd(), msk_pd::pd, and msk_pd::used.
Referenced by handle_trans(), main(), and post_recvs().
void* msk_stats_thread | ( | void * | arg | ) |
msk_stats_thread: unix socket thread
Well, a thread. arg = trans
References msk_trans::debug, INFO_LOG, MSK_DEBUG_EVENT, msk_stats::nsec_callback, msk_stats::nsec_compevent, NSEC_IN_SEC, msk_stats::rx_bytes, msk_stats::rx_err, msk_stats::rx_pkt, msk_trans::stats, msk_trans::stats_sock, msk_stats::tx_bytes, msk_stats::tx_err, and msk_stats::tx_pkt.
int msk_wait_n_read | ( | struct msk_trans * | trans, |
msk_data_t * | data, | ||
int | num_sge, | ||
msk_rloc_t * | rloc | ||
) |
int msk_wait_n_recv | ( | struct msk_trans * | trans, |
msk_data_t * | data, | ||
int | num_sge | ||
) |
Post a receive buffer and waits for that one and not any other to be filled.
Generally a bad idea to use that one unless only that one is used.
trans | [IN] |
data | [OUT] the data buffer to be filled with the received data |
num_sge | [IN] the number of elements in data to register |
int msk_wait_n_send | ( | struct msk_trans * | trans, |
msk_data_t * | data, | ||
int | num_sge | ||
) |
Post a send buffer and waits for that one to be completely sent.
trans | [IN] |
data | [IN] the data to send |
num_sge | [IN] the number of elements in data to send |
int msk_wait_n_write | ( | struct msk_trans * | trans, |
msk_data_t * | data, | ||
int | num_sge, | ||
msk_rloc_t * | rloc | ||
) |
const char* msk_wc_status_str | ( | enum ibv_wc_status | status | ) |