corosync  3.1.9
totemknet.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2022 Red Hat, Inc.
3  *
4  * All rights reserved.
5  *
6  * Author: Christine Caulfield (ccaulfie@redhat.com)
7 
8  * This software licensed under BSD license, the text of which follows:
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions are met:
12  *
13  * - Redistributions of source code must retain the above copyright notice,
14  * this list of conditions and the following disclaimer.
15  * - Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  * - Neither the name of the MontaVista Software, Inc. nor the names of its
19  * contributors may be used to endorse or promote products derived from this
20  * software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32  * THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #include <config.h>
36 
37 #include <assert.h>
38 #include <sys/mman.h>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <sys/socket.h>
42 #include <netdb.h>
43 #include <sys/un.h>
44 #include <sys/ioctl.h>
45 #include <sys/param.h>
46 #include <netinet/in.h>
47 #include <net/ethernet.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <pthread.h>
55 #include <sched.h>
56 #include <time.h>
57 #include <sys/time.h>
58 #include <sys/poll.h>
59 #include <sys/uio.h>
60 #include <limits.h>
61 
62 #include <qb/qbdefs.h>
63 #include <qb/qbloop.h>
64 #ifdef HAVE_LIBNOZZLE
65 #include <libgen.h>
66 #include <libnozzle.h>
67 #endif
68 
69 #include <corosync/sq.h>
70 #include <corosync/swab.h>
71 #include <corosync/logsys.h>
72 #include <corosync/icmap.h>
73 #include <corosync/totem/totemip.h>
74 #include "totemknet.h"
75 
76 #include "main.h"
77 #include "util.h"
78 
79 #include <libknet.h>
81 
82 #ifndef MSG_NOSIGNAL
83 #define MSG_NOSIGNAL 0
84 #endif
85 
86 #ifdef HAVE_LIBNOZZLE
87 static int setup_nozzle(void *knet_context);
88 #endif
89 
90 /* Should match that used by cfg */
91 #define CFG_INTERFACE_STATUS_MAX_LEN 512
92 
94  struct crypto_instance *crypto_inst;
95 
96  struct knet_handle_crypto_cfg last_good_crypto_cfg;
97 
98  qb_loop_t *poll_handle;
99 
100  knet_handle_t knet_handle;
101 
103 
104  void *context;
105 
107  void *context,
108  const void *msg,
109  unsigned int msg_len,
110  const struct sockaddr_storage *system_from);
111 
113  void *context,
114  const struct totem_ip_address *iface_address,
115  unsigned int link_no);
116 
118  void *context,
119  int net_mtu);
120 
122 
123  /*
124  * Function and data used to log messages
125  */
127 
129 
131 
133 
135 
137 
139 
141  int level,
142  int subsys,
143  const char *function,
144  const char *file,
145  int line,
146  const char *format,
147  ...)__attribute__((format(printf, 6, 7)));
148 
149  void *knet_context;
150 
151  char iov_buffer[KNET_MAX_PACKET_SIZE];
152 
154 
156 
158 
160 
162 
164 
166 
167  qb_loop_timer_handle timer_netif_check_timeout;
168 
169  qb_loop_timer_handle timer_merge_detect_timeout;
170 
172 
174 
175  int logpipes[2];
176  int knet_fd;
177 
178  pthread_mutex_t log_mutex;
179 #ifdef HAVE_LIBNOZZLE
180  char *nozzle_name;
184  nozzle_t nozzle_handle;
185 #endif
186 };
187 
188 /* Awkward. But needed to get stats from knet */
190 
191 struct work_item {
192  const void *msg;
193  unsigned int msg_len;
195 };
196 
198  void *knet_context);
199 
200 
201 static int totemknet_configure_compression (
202  struct totemknet_instance *instance,
203  struct totem_config *totem_config);
204 
205 static void totemknet_start_merge_detect_timeout(
206  void *knet_context);
207 
208 static void totemknet_stop_merge_detect_timeout(
209  void *knet_context);
210 
211 static void log_flush_messages (
212  void *knet_context);
213 
214 static void totemknet_instance_initialize (struct totemknet_instance *instance)
215 {
216  int res;
217 
218  memset (instance, 0, sizeof (struct totemknet_instance));
219  res = pthread_mutex_init(&instance->log_mutex, NULL);
220  /*
221  * There is not too much else what can be done.
222  */
223  assert(res == 0);
224 }
225 
226 #define knet_log_printf_lock(level, subsys, function, file, line, format, args...) \
227 do { \
228  (void)pthread_mutex_lock(&instance->log_mutex); \
229  instance->totemknet_log_printf ( \
230  level, subsys, function, file, line, \
231  (const char *)format, ##args); \
232  (void)pthread_mutex_unlock(&instance->log_mutex); \
233 } while (0);
234 
235 #define knet_log_printf(level, format, args...) \
236 do { \
237  knet_log_printf_lock ( \
238  level, instance->totemknet_subsys_id, \
239  __FUNCTION__, __FILE__, __LINE__, \
240  (const char *)format, ##args); \
241 } while (0);
242 
243 #define libknet_log_printf(level, format, args...) \
244 do { \
245  knet_log_printf_lock ( \
246  level, instance->knet_subsys_id, \
247  __FUNCTION__, "libknet.h", __LINE__, \
248  (const char *)format, ##args); \
249 } while (0);
250 
251 #define KNET_LOGSYS_PERROR(err_num, level, fmt, args...) \
252 do { \
253  char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
254  const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
255  instance->totemknet_log_printf ( \
256  level, instance->totemknet_subsys_id, \
257  __FUNCTION__, __FILE__, __LINE__, \
258  fmt ": %s (%d)", ##args, _error_ptr, err_num); \
259  } while(0)
260 
261 
262 #ifdef HAVE_LIBNOZZLE
263 static inline int is_ether_addr_multicast(const uint8_t *addr)
264 {
265  return (addr[0] & 0x01);
266 }
267 static inline int is_ether_addr_zero(const uint8_t *addr)
268 {
269  return (!addr[0] && !addr[1] && !addr[2] && !addr[3] && !addr[4] && !addr[5]);
270 }
271 
272 static int ether_host_filter_fn(void *private_data,
273  const unsigned char *outdata,
274  ssize_t outdata_len,
275  uint8_t tx_rx,
276  knet_node_id_t this_host_id,
277  knet_node_id_t src_host_id,
278  int8_t *channel,
279  knet_node_id_t *dst_host_ids,
280  size_t *dst_host_ids_entries)
281 {
282  struct ether_header *eth_h = (struct ether_header *)outdata;
283  uint8_t *dst_mac = (uint8_t *)eth_h->ether_dhost;
284  uint16_t dst_host_id;
285 
286  if (is_ether_addr_zero(dst_mac))
287  return -1;
288 
289  if (is_ether_addr_multicast(dst_mac)) {
290  return 1;
291  }
292 
293  memmove(&dst_host_id, &dst_mac[4], 2);
294 
295  dst_host_ids[0] = ntohs(dst_host_id);
296  *dst_host_ids_entries = 1;
297 
298  return 0;
299 }
300 #endif
301 
302 static int dst_host_filter_callback_fn(void *private_data,
303  const unsigned char *outdata,
304  ssize_t outdata_len,
305  uint8_t tx_rx,
306  knet_node_id_t this_host_id,
307  knet_node_id_t src_host_id,
308  int8_t *channel,
309  knet_node_id_t *dst_host_ids,
310  size_t *dst_host_ids_entries)
311 {
312  struct totem_message_header *header = (struct totem_message_header *)outdata;
313  int res;
314 
315 #ifdef HAVE_LIBNOZZLE
316  if (*channel != 0) {
317  return ether_host_filter_fn(private_data,
318  outdata, outdata_len,
319  tx_rx,
320  this_host_id, src_host_id,
321  channel,
322  dst_host_ids,
323  dst_host_ids_entries);
324  }
325 #endif
326  if (header->target_nodeid) {
327  dst_host_ids[0] = header->target_nodeid;
328  *dst_host_ids_entries = 1;
329  res = 0; /* unicast message */
330  }
331  else {
332  *dst_host_ids_entries = 0;
333  res = 1; /* multicast message */
334  }
335  return res;
336 }
337 
338 static void socket_error_callback_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)
339 {
340  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
341 
342  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet socket ERROR notification called: txrx=%d, error=%d, errorno=%d", tx_rx, error, errorno);
343  if ((error == -1 && errorno != EAGAIN) || (error == 0)) {
344  knet_handle_remove_datafd(instance->knet_handle, datafd);
345  }
346 }
347 
348 static void host_change_callback_fn(void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)
349 {
350  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
351 
352  // TODO: what? if anything.
353  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet host change callback. nodeid: " CS_PRI_NODE_ID " reachable: %d", host_id, reachable);
354 }
355 
356 static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
357 {
358  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
359  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
360 
361  /* We don't need to tell corosync the actual knet MTU */
362 // instance->totemknet_mtu_changed(instance->context, data_mtu);
363 }
364 
366  void *knet_context,
367  const char *cipher_type,
368  const char *hash_type)
369 {
370  return (0);
371 }
372 
373 
374 static inline void ucast_sendmsg (
375  struct totemknet_instance *instance,
376  struct totem_ip_address *system_to,
377  const void *msg,
378  unsigned int msg_len)
379 {
380  int res = 0;
381  struct totem_message_header *header = (struct totem_message_header *)msg;
382  struct msghdr msg_ucast;
383  struct iovec iovec;
384 
385  header->target_nodeid = system_to->nodeid;
386 
387  iovec.iov_base = (void *)msg;
388  iovec.iov_len = msg_len;
389 
390  /*
391  * Build unicast message
392  */
393  memset(&msg_ucast, 0, sizeof(msg_ucast));
394  msg_ucast.msg_iov = (void *)&iovec;
395  msg_ucast.msg_iovlen = 1;
396 #ifdef HAVE_MSGHDR_CONTROL
397  msg_ucast.msg_control = 0;
398 #endif
399 #ifdef HAVE_MSGHDR_CONTROLLEN
400  msg_ucast.msg_controllen = 0;
401 #endif
402 #ifdef HAVE_MSGHDR_FLAGS
403  msg_ucast.msg_flags = 0;
404 #endif
405 #ifdef HAVE_MSGHDR_ACCRIGHTS
406  msg_ucast.msg_accrights = NULL;
407 #endif
408 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
409  msg_ucast.msg_accrightslen = 0;
410 #endif
411 
412  /*
413  * Transmit unicast message
414  * An error here is recovered by totemsrp
415  */
416 
417  res = sendmsg (instance->knet_fd, &msg_ucast, MSG_NOSIGNAL);
418  if (res < 0) {
420  "sendmsg(ucast) failed (non-critical)");
421  }
422 }
423 
424 static inline void mcast_sendmsg (
425  struct totemknet_instance *instance,
426  const void *msg,
427  unsigned int msg_len,
428  int only_active)
429 {
430  int res;
431  struct totem_message_header *header = (struct totem_message_header *)msg;
432  struct msghdr msg_mcast;
433  struct iovec iovec;
434 
435  iovec.iov_base = (void *)msg;
436  iovec.iov_len = msg_len;
437 
438  header->target_nodeid = 0;
439 
440  /*
441  * Build multicast message
442  */
443  memset(&msg_mcast, 0, sizeof(msg_mcast));
444  msg_mcast.msg_iov = (void *)&iovec;
445  msg_mcast.msg_iovlen = 1;
446 #ifdef HAVE_MSGHDR_CONTROL
447  msg_mcast.msg_control = 0;
448 #endif
449 #ifdef HAVE_MSGHDR_CONTROLLEN
450  msg_mcast.msg_controllen = 0;
451 #endif
452 #ifdef HAVE_MSGHDR_FLAGS
453  msg_mcast.msg_flags = 0;
454 #endif
455 #ifdef HAVE_MSGHDR_ACCRIGHTS
456  msg_mcast.msg_accrights = NULL;
457 #endif
458 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
459  msg_mcast.msg_accrightslen = 0;
460 #endif
461 
462 
463 // log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_sendmsg. only_active=%d, len=%d", only_active, msg_len);
464 
465  res = sendmsg (instance->knet_fd, &msg_mcast, MSG_NOSIGNAL);
466  if (res < msg_len) {
467  knet_log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_send sendmsg returned %d", res);
468  }
469 
470  if (!only_active || instance->send_merge_detect_message) {
471  /*
472  * Current message was sent to all nodes
473  */
475  instance->send_merge_detect_message = 0;
476  }
477 }
478 
479 static int node_compare(const void *aptr, const void *bptr)
480 {
481  uint16_t a,b;
482 
483  a = *(uint16_t *)aptr;
484  b = *(uint16_t *)bptr;
485 
486  return a > b;
487 }
488 
489 #ifndef OWN_INDEX_NONE
490 #define OWN_INDEX_NONE -1
491 #endif
492 
494  void *knet_context,
495  unsigned int nodeid,
496  struct totem_node_status *node_status)
497 {
498  int i;
499  int res = 0;
500  struct knet_link_status link_status;
501  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
502  struct knet_host_status knet_host_status;
503  uint8_t link_list[KNET_MAX_LINK];
504  size_t num_links;
505 
506  if (!instance->knet_handle) {
507  return CS_ERR_NOT_EXIST; /* Not using knet */
508  }
509 
510  if (!node_status) {
511  return CS_ERR_INVALID_PARAM;
512  }
513 
514  res = knet_host_get_status(instance->knet_handle,
515  nodeid,
516  &knet_host_status);
517  if (res) {
518  knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_handle_get_host_status(%d) failed: %d", nodeid, res);
519  return (-1);
520  }
521  node_status->nodeid = nodeid;
522  node_status->reachable = knet_host_status.reachable;
523  node_status->remote = knet_host_status.remote;
524  node_status->external = knet_host_status.external;
525 
526 #ifdef HAVE_KNET_ONWIRE_VER
527  res = knet_handle_get_onwire_ver(instance->knet_handle,
528  nodeid,
529  &node_status->onwire_min,
530  &node_status->onwire_max,
531  &node_status->onwire_ver);
532  if (res) {
533  knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_handle_get_onwire_ver(%d) failed: %d", nodeid, res);
534  return (-1);
535  }
536 #endif
537  /* Get link info */
538  res = knet_link_get_link_list(instance->knet_handle,
539  nodeid, link_list, &num_links);
540  if (res) {
541  knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_link_get_link_list(%d) failed: %d", nodeid, res);
542  return (-1);
543  }
544 
545  /* node_status[] has been zeroed for us in totempg.c */
546  for (i=0; i < num_links; i++) {
547  if (!instance->totem_config->interfaces[link_list[i]].configured) {
548  continue;
549  }
550  res = knet_link_get_status(instance->knet_handle,
551  nodeid,
552  link_list[i],
553  &link_status,
554  sizeof(link_status));
555  if (res == 0) {
556  node_status->link_status[link_list[i]].enabled = link_status.enabled;
557  node_status->link_status[link_list[i]].connected = link_status.connected;
558  node_status->link_status[link_list[i]].dynconnected = link_status.dynconnected;
559  node_status->link_status[link_list[i]].mtu = link_status.mtu;
560  memcpy(node_status->link_status[link_list[i]].src_ipaddr, link_status.src_ipaddr, KNET_MAX_HOST_LEN);
561  memcpy(node_status->link_status[link_list[i]].dst_ipaddr, link_status.dst_ipaddr, KNET_MAX_HOST_LEN);
562  } else {
563  knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_link_get_link_status(%d, %d) failed: %d", nodeid, link_list[i], res);
564  }
565  }
566  return res;
567 }
568 
569 
570 
571 int totemknet_ifaces_get (void *knet_context,
572  char ***status,
573  unsigned int *iface_count)
574 {
575  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
576  struct knet_link_status link_status;
577  knet_node_id_t host_list[KNET_MAX_HOST];
578  uint8_t link_list[KNET_MAX_LINK];
579  size_t num_hosts;
580  size_t num_links;
581  size_t link_idx;
582  int i,j;
583  char *ptr;
584  int res = 0;
585 
586  /*
587  * Don't do the whole 'link_info' bit if the caller just wants
588  * a count of interfaces.
589  */
590  if (status) {
591  int own_idx = OWN_INDEX_NONE;
592 
593  res = knet_host_get_host_list(instance->knet_handle,
594  host_list, &num_hosts);
595  if (res) {
596  return (-1);
597  }
598  qsort(host_list, num_hosts, sizeof(uint16_t), node_compare);
599 
600  for (j=0; j<num_hosts; j++) {
601  if (host_list[j] == instance->our_nodeid) {
602  own_idx = j;
603  break;
604  }
605  }
606 
607  for (i=0; i<INTERFACE_MAX; i++) {
608  memset(instance->link_status[i], 'd', CFG_INTERFACE_STATUS_MAX_LEN-1);
609  if (own_idx != OWN_INDEX_NONE) {
610  instance->link_status[i][own_idx] = 'n';
611  }
612  instance->link_status[i][num_hosts] = '\0';
613  }
614 
615  /* This is all a bit "inside-out" because "status" is a set of strings per link
616  * and knet orders things by host
617  */
618  for (j=0; j<num_hosts; j++) {
619  if (own_idx != OWN_INDEX_NONE && j == own_idx) {
620  continue ;
621  }
622 
623  res = knet_link_get_link_list(instance->knet_handle,
624  host_list[j], link_list, &num_links);
625  if (res) {
626  return (-1);
627  }
628 
629  link_idx = 0;
630  for (i=0; i < num_links; i++) {
631  /*
632  * Skip over links that are unconfigured to corosync. This is basically
633  * link0 if corosync isn't using it for comms, as we will still
634  * have it set up for loopback.
635  */
636  if (!instance->totem_config->interfaces[link_list[i]].configured) {
637  continue;
638  }
639  ptr = instance->link_status[link_idx++];
640 
641  res = knet_link_get_status(instance->knet_handle,
642  host_list[j],
643  link_list[i],
644  &link_status,
645  sizeof(link_status));
646  if (res == 0) {
647  ptr[j] = '0' + (link_status.enabled |
648  link_status.connected<<1 |
649  link_status.dynconnected<<2);
650  }
651  else {
653  "totemknet_ifaces_get: Cannot get link status: %s", strerror(errno));
654  ptr[j] = '?';
655  }
656  }
657  }
658  *status = instance->link_status;
659  }
660 
661  *iface_count = INTERFACE_MAX;
662 
663  return (res);
664 }
665 
667  void *knet_context)
668 {
669  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
670  int res = 0;
671  int i,j;
672  static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
673  uint8_t links[KNET_MAX_LINK];
674  size_t num_nodes;
675  size_t num_links;
676 
677  knet_log_printf(LOG_DEBUG, "totemknet: finalize");
678 
679  qb_loop_poll_del (instance->poll_handle, instance->logpipes[0]);
680  qb_loop_poll_del (instance->poll_handle, instance->knet_fd);
681 
682  /*
683  * Disable forwarding to make knet flush send queue. This ensures that the LEAVE message will be sent.
684  */
685  res = knet_handle_setfwd(instance->knet_handle, 0);
686  if (res) {
687  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_setfwd failed: %s", strerror(errno));
688  }
689 
690  res = knet_host_get_host_list(instance->knet_handle, nodes, &num_nodes);
691  if (res) {
692  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet node list for shutdown: %s", strerror(errno));
693  /* Crash out anyway */
694  goto finalise_error;
695  }
696 
697  /* Tidily shut down all nodes & links. */
698  for (i=0; i<num_nodes; i++) {
699 
700  res = knet_link_get_link_list(instance->knet_handle, nodes[i], links, &num_links);
701  if (res) {
702  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet link list for node " CS_PRI_NODE_ID ": %s", nodes[i], strerror(errno));
703  goto finalise_error;
704  }
705  for (j=0; j<num_links; j++) {
706  res = knet_link_set_enable(instance->knet_handle, nodes[i], links[j], 0);
707  if (res) {
708  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_set_enable(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
709  }
710  res = knet_link_clear_config(instance->knet_handle, nodes[i], links[j]);
711  if (res) {
712  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_clear_config(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
713  }
714  }
715  res = knet_host_remove(instance->knet_handle, nodes[i]);
716  if (res) {
717  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_host_remove(node " CS_PRI_NODE_ID ") failed: %s", nodes[i], strerror(errno));
718  }
719  }
720 
721 finalise_error:
722  res = knet_handle_free(instance->knet_handle);
723  if (res) {
724  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_free failed: %s", strerror(errno));
725  }
726 
727  totemknet_stop_merge_detect_timeout(instance);
728 
729  log_flush_messages(instance);
730 
731  /*
732  * Error is deliberately ignored
733  */
734  (void)pthread_mutex_destroy(&instance->log_mutex);
735 
736  return (res);
737 }
738 
739 static int log_deliver_fn (
740  int fd,
741  int revents,
742  void *data)
743 {
744  struct totemknet_instance *instance = (struct totemknet_instance *)data;
745  char buffer[sizeof(struct knet_log_msg)*4];
746  char *bufptr = buffer;
747  int done = 0;
748  int len;
749 
750  len = read(fd, buffer, sizeof(buffer));
751  while (done < len) {
752  struct knet_log_msg *msg = (struct knet_log_msg *)bufptr;
753  switch (msg->msglevel) {
754  case KNET_LOG_ERR:
756  knet_log_get_subsystem_name(msg->subsystem),
757  msg->msg);
758  break;
759  case KNET_LOG_WARN:
761  knet_log_get_subsystem_name(msg->subsystem),
762  msg->msg);
763  break;
764  case KNET_LOG_INFO:
766  knet_log_get_subsystem_name(msg->subsystem),
767  msg->msg);
768  break;
769  case KNET_LOG_DEBUG:
771  knet_log_get_subsystem_name(msg->subsystem),
772  msg->msg);
773  break;
774 #ifdef KNET_LOG_TRACE
775  case KNET_LOG_TRACE:
777  knet_log_get_subsystem_name(msg->subsystem),
778  msg->msg);
779  break;
780 #endif
781  }
782  bufptr += sizeof(struct knet_log_msg);
783  done += sizeof(struct knet_log_msg);
784  }
785  return 0;
786 }
787 
788 static int data_deliver_fn (
789  int fd,
790  int revents,
791  void *data)
792 {
793  struct totemknet_instance *instance = (struct totemknet_instance *)data;
794  struct msghdr msg_hdr;
795  struct iovec iov_recv;
796  struct sockaddr_storage system_from;
797  ssize_t msg_len;
798  int truncated_packet;
799 
800  iov_recv.iov_base = instance->iov_buffer;
801  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
802 
803  msg_hdr.msg_name = &system_from;
804  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
805  msg_hdr.msg_iov = &iov_recv;
806  msg_hdr.msg_iovlen = 1;
807 #ifdef HAVE_MSGHDR_CONTROL
808  msg_hdr.msg_control = 0;
809 #endif
810 #ifdef HAVE_MSGHDR_CONTROLLEN
811  msg_hdr.msg_controllen = 0;
812 #endif
813 #ifdef HAVE_MSGHDR_FLAGS
814  msg_hdr.msg_flags = 0;
815 #endif
816 #ifdef HAVE_MSGHDR_ACCRIGHTS
817  msg_hdr.msg_accrights = NULL;
818 #endif
819 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
820  msg_hdr.msg_accrightslen = 0;
821 #endif
822 
823  msg_len = recvmsg (fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
824  if (msg_len <= 0) {
825  return (0);
826  }
827 
828  truncated_packet = 0;
829 
830 #ifdef HAVE_MSGHDR_FLAGS
831  if (msg_hdr.msg_flags & MSG_TRUNC) {
832  truncated_packet = 1;
833  }
834 #else
835  /*
836  * We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
837  * if bytes_received == KNET_MAX_PACKET_SIZE then packet is truncated
838  */
839  if (bytes_received == KNET_MAX_PACKET_SIZE) {
840  truncated_packet = 1;
841  }
842 #endif
843 
844  if (truncated_packet) {
846  "Received too big message. This may be because something bad is happening"
847  "on the network (attack?), or you tried join more nodes than corosync is"
848  "compiled with (%u) or bug in the code (bad estimation of "
849  "the KNET_MAX_PACKET_SIZE). Dropping packet.", PROCESSOR_COUNT_MAX);
850  return (0);
851  }
852 
853  /*
854  * Handle incoming message
855  */
856  instance->totemknet_deliver_fn (
857  instance->context,
858  instance->iov_buffer,
859  msg_len,
860  &system_from);
861 
862  return (0);
863 }
864 
865 static void timer_function_netif_check_timeout (
866  void *data)
867 {
868  struct totemknet_instance *instance = (struct totemknet_instance *)data;
869  int i;
870  int res = 0;
871 
872  for (i=0; i < INTERFACE_MAX; i++) {
873  if (!instance->totem_config->interfaces[i].configured) {
874  continue;
875  }
876  res = instance->totemknet_iface_change_fn (instance->context,
877  &instance->my_ids[i],
878  i);
879  }
880  if (res != 0) {
881  /* This is only called at startup, so we can quit here.
882  Refresh takes a different path */
884  }
885 }
886 
887 static void knet_set_access_list_config(struct totemknet_instance *instance)
888 {
889 #ifdef HAVE_KNET_ACCESS_LIST
890  uint32_t value;
891  cs_error_t err;
892 
893  value = instance->totem_config->block_unlisted_ips;
894  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_enable access list: %d", value);
895 
896  err = knet_handle_enable_access_lists(instance->knet_handle, value);
897  if (err) {
898  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_access_lists failed");
899  }
900 #endif
901 }
902 
904 {
905  int logsys_log_mode;
906  int knet_log_mode = KNET_LOG_INFO;
907  uint8_t s;
908  int err;
909 
910  if (!global_instance || !global_instance->knet_handle) {
911  return;
912  }
913 
914  /* Reconfigure logging level */
915  logsys_log_mode = logsys_config_debug_get("KNET");
916 
917  switch (logsys_log_mode) {
918  case LOGSYS_DEBUG_OFF:
919  knet_log_mode = KNET_LOG_INFO;
920  break;
921  case LOGSYS_DEBUG_ON:
922  knet_log_mode = KNET_LOG_DEBUG;
923  break;
924  case LOGSYS_DEBUG_TRACE:
925 #ifdef KNET_LOG_TRACE
926  knet_log_mode = KNET_LOG_TRACE;
927 #else
928  knet_log_mode = KNET_LOG_DEBUG;
929 #endif
930  break;
931  }
932  log_printf (LOGSYS_LEVEL_DEBUG, "totemknet setting log level %s", knet_log_get_loglevel_name(knet_log_mode));
933  err = 0;
934  for (s = 0; s<KNET_MAX_SUBSYSTEMS; s++) {
935  err = knet_log_set_loglevel(global_instance->knet_handle, s, knet_log_mode);
936  }
937 
938  /* If one fails, they all fail. no point in issuing KNET_MAX_SUBSYSTEMS errors */
939  if (err) {
940  log_printf (LOGSYS_LEVEL_ERROR, "totemknet failed to set log level: %s", strerror(errno));
941  }
942 }
943 
944 
945 /* NOTE: this relies on the fact that totem_reload_notify() is called first */
946 static void totemknet_refresh_config(
947  int32_t event,
948  const char *key_name,
949  struct icmap_notify_value new_val,
950  struct icmap_notify_value old_val,
951  void *user_data)
952 {
953  uint8_t reloading;
954  int after_reload;
955  uint32_t link_no;
956  size_t num_nodes;
957  knet_node_id_t host_ids[KNET_MAX_HOST];
958  int i;
959  int err;
960  struct totemknet_instance *instance = (struct totemknet_instance *)user_data;
961 
962  ENTER();
963 
964  /*
965  * If a full reload is in progress then don't do anything until it's done and
966  * can reconfigure it all atomically
967  */
968  if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
969  return;
970  }
971 
972  after_reload = (strcmp(key_name, "config.totemconfig_reload_in_progress") == 0);
973 
974  knet_set_access_list_config(instance);
975 
976  if (strcmp(key_name, "totem.knet_pmtud_interval") == 0 || after_reload) {
977  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %u",
978  instance->totem_config->knet_pmtud_interval);
979  err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
980  if (err) {
981  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
982  }
983  }
984 
985  if (strcmp(key_name, "totem.knet_mtu") == 0 || after_reload) {
986  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_mtu now %u", instance->totem_config->knet_mtu);
987  err = knet_handle_pmtud_set(instance->knet_handle, instance->totem_config->knet_mtu);
988  if (err) {
989  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud failed");
990  }
991  }
992 
993  /* Configure link parameters for each node */
994  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
995  if (err != 0) {
996  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list failed");
997  }
998 
999  for (i=0; i<num_nodes; i++) {
1000  int linkerr = 0;
1001  for (link_no = 0; link_no < INTERFACE_MAX; link_no++) {
1002  if (host_ids[i] == instance->our_nodeid || !instance->totem_config->interfaces[link_no].configured) {
1003  continue;
1004  }
1005 
1006  err = knet_link_set_ping_timers(instance->knet_handle, host_ids[i], link_no,
1007  instance->totem_config->interfaces[link_no].knet_ping_interval,
1008  instance->totem_config->interfaces[link_no].knet_ping_timeout,
1009  instance->totem_config->interfaces[link_no].knet_ping_precision);
1010  if (err) {
1011  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
1012  linkerr = err;
1013  }
1014  err = knet_link_set_pong_count(instance->knet_handle, host_ids[i], link_no,
1015  instance->totem_config->interfaces[link_no].knet_pong_count);
1016  if (err) {
1017  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for node " CS_PRI_NODE_ID " link %d failed",host_ids[i], link_no);
1018  linkerr = err;
1019  }
1020  err = knet_link_set_priority(instance->knet_handle, host_ids[i], link_no,
1021  instance->totem_config->interfaces[link_no].knet_link_priority);
1022  if (err) {
1023  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
1024  linkerr = err;
1025  }
1026 
1027  }
1028  if (linkerr) {
1029  icmap_set_string("config.reload_error_message", "Failed to set knet ping timers(2)");
1030  }
1031  }
1032 
1033  /* Log levels get reconfigured from logconfig.c as that happens last in the reload */
1034  LEAVE();
1035 }
1036 
1037 static void totemknet_add_config_notifications(struct totemknet_instance *instance)
1038 {
1039  icmap_track_t icmap_track_totem = NULL;
1040  icmap_track_t icmap_track_reload = NULL;
1041 
1042  ENTER();
1043 
1044  icmap_track_add("totem.",
1046  totemknet_refresh_config,
1047  instance,
1048  &icmap_track_totem);
1049 
1050  icmap_track_add("config.totemconfig_reload_in_progress",
1052  totemknet_refresh_config,
1053  instance,
1054  &icmap_track_reload);
1055 
1056  LEAVE();
1057 }
1058 
1059 static int totemknet_is_crypto_enabled(const struct totemknet_instance *instance)
1060 {
1061 
1062  return (!(strcmp(instance->totem_config->crypto_cipher_type, "none") == 0 &&
1063  strcmp(instance->totem_config->crypto_hash_type, "none") == 0));
1064 
1065 }
1066 
1067 static int totemknet_set_knet_crypto(struct totemknet_instance *instance)
1068 {
1069  struct knet_handle_crypto_cfg crypto_cfg;
1070  int res;
1071 
1072  /* These have already been validated */
1073  memcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model, sizeof(crypto_cfg.crypto_model));
1074  memcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type, sizeof(crypto_cfg.crypto_model));
1075  memcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type, sizeof(crypto_cfg.crypto_model));
1076  memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
1077  crypto_cfg.private_key_len = instance->totem_config->private_key_len;
1078 
1079 #ifdef HAVE_KNET_CRYPTO_RECONF
1080 
1081  knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s on index %d",
1082  crypto_cfg.crypto_model,
1083  crypto_cfg.crypto_cipher_type,
1084  crypto_cfg.crypto_hash_type,
1085  instance->totem_config->crypto_index
1086  );
1087 
1088  /* If crypto is being disabled we need to explicitly allow cleartext traffic in knet */
1089  if (!totemknet_is_crypto_enabled(instance)) {
1090  res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
1091  if (res) {
1092  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(ALLOW) failed %s", strerror(errno));
1093  }
1094  }
1095 
1096  /* use_config will be called later when all nodes are synced */
1097  res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, instance->totem_config->crypto_index);
1098  if (res == 0) {
1099  /* Keep a copy in case it fails in future */
1100  memcpy(&instance->last_good_crypto_cfg, &crypto_cfg, sizeof(crypto_cfg));
1101  }
1102  if (res == -1) {
1103  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: %s", instance->totem_config->crypto_index, strerror(errno));
1104  goto exit_error;
1105  }
1106  if (res == -2) {
1107  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: -2", instance->totem_config->crypto_index);
1108  goto exit_error;
1109  }
1110 #else
1111  knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s",
1112  crypto_cfg.crypto_model,
1113  crypto_cfg.crypto_cipher_type,
1114  crypto_cfg.crypto_hash_type
1115  );
1116 
1117  res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
1118  if (res == -1) {
1119  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
1120  goto exit_error;
1121  }
1122  if (res == -2) {
1123  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
1124  goto exit_error;
1125  }
1126 #endif
1127 
1128 exit_error:
1129 #ifdef HAVE_KNET_CRYPTO_RECONF
1130  if (res) {
1131  icmap_set_string("config.reload_error_message", "Failed to set crypto parameters");
1132 
1133  /* Restore the old values in cmap & totem_config */
1134  icmap_set_string("totem.crypto_cipher", instance->last_good_crypto_cfg.crypto_cipher_type);
1135  icmap_set_string("totem.crypto_hash", instance->last_good_crypto_cfg.crypto_hash_type);
1136  icmap_set_string("totem.crypto_model", instance->last_good_crypto_cfg.crypto_model);
1137 
1138  memcpy(instance->totem_config->crypto_hash_type, instance->last_good_crypto_cfg.crypto_hash_type,
1139  sizeof(instance->last_good_crypto_cfg.crypto_hash_type));
1140  memcpy(instance->totem_config->crypto_cipher_type, instance->last_good_crypto_cfg.crypto_cipher_type,
1141  sizeof(instance->last_good_crypto_cfg.crypto_cipher_type));
1142  memcpy(instance->totem_config->crypto_model, instance->last_good_crypto_cfg.crypto_model,
1143  sizeof(instance->last_good_crypto_cfg.crypto_model));
1144  }
1145 #endif
1146  return res;
1147 }
1148 
1149 /*
1150  * Create an instance
1151  */
1153  qb_loop_t *poll_handle,
1154  void **knet_context,
1155  struct totem_config *totem_config,
1156  totemsrp_stats_t *stats,
1157  void *context,
1158 
1159  int (*deliver_fn) (
1160  void *context,
1161  const void *msg,
1162  unsigned int msg_len,
1163  const struct sockaddr_storage *system_from),
1164 
1165  int (*iface_change_fn) (
1166  void *context,
1167  const struct totem_ip_address *iface_address,
1168  unsigned int link_no),
1169 
1170  void (*mtu_changed) (
1171  void *context,
1172  int net_mtu),
1173 
1174  void (*target_set_completed) (
1175  void *context))
1176 {
1177  struct totemknet_instance *instance;
1178  char *tmp_str;
1179  int8_t channel=0;
1180  int allow_knet_handle_fallback=0;
1181  int res;
1182  int i;
1183 
1184  instance = malloc (sizeof (struct totemknet_instance));
1185  if (instance == NULL) {
1186  return (-1);
1187  }
1188 
1189  totemknet_instance_initialize (instance);
1190 
1191  instance->totem_config = totem_config;
1192 
1193  /*
1194  * Configure logging
1195  */
1196  instance->totemknet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
1203 
1204  instance->knet_subsys_id = _logsys_subsys_create("KNET", "libknet.h");
1205 
1206  /*
1207  * Initialize local variables for totemknet
1208  */
1209 
1210  instance->our_nodeid = instance->totem_config->node_id;
1211 
1212  for (i=0; i< INTERFACE_MAX; i++) {
1213  totemip_copy(&instance->my_ids[i], &totem_config->interfaces[i].bindnet);
1214  instance->my_ids[i].nodeid = instance->our_nodeid;
1215  instance->ip_port[i] = totem_config->interfaces[i].ip_port;
1216 
1217  /* Needed for totemsrp */
1218  totem_config->interfaces[i].boundto.nodeid = instance->our_nodeid;
1219  }
1220 
1221  instance->poll_handle = poll_handle;
1222 
1223  instance->context = context;
1224  instance->totemknet_deliver_fn = deliver_fn;
1225 
1226  instance->totemknet_iface_change_fn = iface_change_fn;
1227 
1228  instance->totemknet_mtu_changed = mtu_changed;
1229 
1230  instance->totemknet_target_set_completed = target_set_completed;
1231 
1232  instance->loopback_link = 0;
1233 
1234  res = pipe(instance->logpipes);
1235  if (res == -1) {
1236  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
1237  goto exit_error;
1238  }
1239  if (fcntl(instance->logpipes[0], F_SETFL, O_NONBLOCK) == -1 ||
1240  fcntl(instance->logpipes[1], F_SETFL, O_NONBLOCK) == -1) {
1241  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to set O_NONBLOCK flag for instance->logpipes");
1242  goto exit_error;
1243  }
1244 
1245  if (icmap_get_string("system.allow_knet_handle_fallback", &tmp_str) == CS_OK) {
1246  if (strcmp(tmp_str, "yes") == 0) {
1247  allow_knet_handle_fallback = 1;
1248  }
1249  free(tmp_str);
1250  }
1251 
1252 #if defined(KNET_API_VER) && (KNET_API_VER == 2)
1253  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, KNET_HANDLE_FLAG_PRIVILEGED);
1254 #else
1255  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG);
1256 #endif
1257 
1258  if (allow_knet_handle_fallback && !instance->knet_handle && errno == ENAMETOOLONG) {
1259  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_new failed, trying unprivileged");
1260 #if defined(KNET_API_VER) && (KNET_API_VER == 2)
1261  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
1262 #else
1263  instance->knet_handle = knet_handle_new_ex(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
1264 #endif
1265  }
1266 
1267  if (!instance->knet_handle) {
1268  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "knet_handle_new failed");
1269  goto exit_error;
1270  }
1271 
1272  knet_set_access_list_config(instance);
1273 
1274  res = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
1275  if (res) {
1276  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
1277  }
1278  res = knet_handle_pmtud_set(instance->knet_handle, instance->totem_config->knet_mtu);
1279  if (res) {
1280  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_set failed");
1281  }
1282  res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
1283  if (res) {
1284  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");
1285  }
1286  res = knet_handle_enable_sock_notify(instance->knet_handle, instance, socket_error_callback_fn);
1287  if (res) {
1288  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_sock_notify failed");
1289  }
1290  res = knet_host_enable_status_change_notify(instance->knet_handle, instance, host_change_callback_fn);
1291  if (res) {
1292  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_host_enable_status_change_notify failed");
1293  }
1294  res = knet_handle_enable_pmtud_notify(instance->knet_handle, instance, pmtu_change_callback_fn);
1295  if (res) {
1296  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
1297  }
1298  global_instance = instance;
1299 
1300  /* Setup knet logging level */
1302 
1303  /* Get an fd into knet */
1304  instance->knet_fd = 0;
1305  res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel);
1306  if (res) {
1307  knet_log_printf(LOG_DEBUG, "knet_handle_add_datafd failed: %s", strerror(errno));
1308  goto exit_error;
1309  }
1310 
1311  /* Enable crypto if requested */
1312 #ifdef HAVE_KNET_CRYPTO_RECONF
1313  if (totemknet_is_crypto_enabled(instance)) {
1314  res = totemknet_set_knet_crypto(instance);
1315  if (res == 0) {
1316  res = knet_handle_crypto_use_config(instance->knet_handle, totem_config->crypto_index);
1317  if (res) {
1318  knet_log_printf(LOG_DEBUG, "knet_handle_crypto_use_config failed: %s", strerror(errno));
1319  goto exit_error;
1320  }
1321  } else {
1322  knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
1323  goto exit_error;
1324  }
1325  res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
1326  if (res) {
1327  knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (DISALLOW) failed: %s", strerror(errno));
1328  goto exit_error;
1329  }
1330 
1331  } else {
1332  res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
1333  if (res) {
1334  knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (ALLOW) failed: %s", strerror(errno));
1335  goto exit_error;
1336  }
1337  }
1338 #else
1339  if (totemknet_is_crypto_enabled(instance)) {
1340  res = totemknet_set_knet_crypto(instance);
1341  if (res) {
1342  knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
1343  goto exit_error;
1344  }
1345  }
1346 #endif
1347 
1348  /* Set up compression */
1349  if (strcmp(totem_config->knet_compression_model, "none") != 0) {
1350  /* Not fatal, but will log */
1351  (void)totemknet_configure_compression(instance, totem_config);
1352  }
1353 
1354  knet_handle_setfwd(instance->knet_handle, 1);
1355 
1356  instance->link_mode = KNET_LINK_POLICY_PASSIVE;
1357  if (strcmp(instance->totem_config->link_mode, "active")==0) {
1358  instance->link_mode = KNET_LINK_POLICY_ACTIVE;
1359  }
1360  if (strcmp(instance->totem_config->link_mode, "rr")==0) {
1361  instance->link_mode = KNET_LINK_POLICY_RR;
1362  }
1363 
1364  for (i=0; i<INTERFACE_MAX; i++) {
1365  instance->link_status[i] = malloc(CFG_INTERFACE_STATUS_MAX_LEN);
1366  if (!instance->link_status[i]) {
1367  goto exit_error;
1368  }
1369  }
1370 
1371  qb_loop_poll_add (instance->poll_handle,
1372  QB_LOOP_MED,
1373  instance->logpipes[0],
1374  POLLIN, instance, log_deliver_fn);
1375 
1376  qb_loop_poll_add (instance->poll_handle,
1377  QB_LOOP_HIGH,
1378  instance->knet_fd,
1379  POLLIN, instance, data_deliver_fn);
1380 
1381  /*
1382  * Upper layer isn't ready to receive message because it hasn't
1383  * initialized yet. Add short timer to check the interfaces.
1384  */
1385  qb_loop_timer_add (instance->poll_handle,
1386  QB_LOOP_MED,
1387  100*QB_TIME_NS_IN_MSEC,
1388  (void *)instance,
1389  timer_function_netif_check_timeout,
1390  &instance->timer_netif_check_timeout);
1391 
1392  totemknet_start_merge_detect_timeout(instance);
1393 
1394  /* Start listening for config changes */
1395  totemknet_add_config_notifications(instance);
1396 
1397  /* Add stats keys to icmap */
1399 
1400  knet_log_printf (LOGSYS_LEVEL_INFO, "totemknet initialized");
1401  *knet_context = instance;
1402 
1403  return (0);
1404 
1405 exit_error:
1406  log_flush_messages(instance);
1407  free(instance);
1408  return (-1);
1409 }
1410 
1412 {
1413  /* Need to have space for a message AND a struct mcast in case of encapsulated messages */
1414  return malloc(KNET_MAX_PACKET_SIZE + 512);
1415 }
1416 
1418 {
1419  return free (ptr);
1420 }
1421 
1423  void *knet_context,
1424  int processor_count)
1425 {
1426  return (0);
1427 }
1428 
1429 int totemknet_recv_flush (void *knet_context)
1430 {
1431  return (0);
1432 }
1433 
1434 int totemknet_send_flush (void *knet_context)
1435 {
1436  return (0);
1437 }
1438 
1440  void *knet_context,
1441  const void *msg,
1442  unsigned int msg_len)
1443 {
1444  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1445  int res = 0;
1446 
1447  ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
1448 
1449  return (res);
1450 }
1452  void *knet_context,
1453  const void *msg,
1454  unsigned int msg_len)
1455 {
1456  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1457  int res = 0;
1458 
1459  mcast_sendmsg (instance, msg, msg_len, 0);
1460 
1461  return (res);
1462 }
1463 
1465  void *knet_context,
1466  const void *msg,
1467  unsigned int msg_len)
1468 {
1469  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1470  int res = 0;
1471 
1472  mcast_sendmsg (instance, msg, msg_len, 1);
1473 
1474  return (res);
1475 }
1476 
1477 
1478 extern int totemknet_iface_check (void *knet_context)
1479 {
1480  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1481  int res = 0;
1482 
1483  knet_log_printf(LOG_DEBUG, "totemknet: iface_check");
1484 
1485  return (res);
1486 }
1487 
1488 extern void totemknet_net_mtu_adjust (void *knet_context, struct totem_config *totem_config)
1489 {
1490  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1491 
1492  knet_log_printf(LOG_DEBUG, "totemknet: Returning MTU of %d", totem_config->net_mtu);
1493 }
1494 
1496  void *knet_context,
1497  unsigned int nodeid)
1498 {
1499  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1500  int res = 0;
1501 
1502  instance->token_target.nodeid = nodeid;
1503 
1504  instance->totemknet_target_set_completed (instance->context);
1505 
1506  return (res);
1507 }
1508 
1510  void *knet_context)
1511 {
1512  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1513  unsigned int res;
1514  struct sockaddr_storage system_from;
1515  struct msghdr msg_hdr;
1516  struct iovec iov_recv;
1517  struct pollfd ufd;
1518  int nfds;
1519  int msg_processed = 0;
1520 
1521  iov_recv.iov_base = instance->iov_buffer;
1522  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
1523 
1524  msg_hdr.msg_name = &system_from;
1525  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
1526  msg_hdr.msg_iov = &iov_recv;
1527  msg_hdr.msg_iovlen = 1;
1528 #ifdef HAVE_MSGHDR_CONTROL
1529  msg_hdr.msg_control = 0;
1530 #endif
1531 #ifdef HAVE_MSGHDR_CONTROLLEN
1532  msg_hdr.msg_controllen = 0;
1533 #endif
1534 #ifdef HAVE_MSGHDR_FLAGS
1535  msg_hdr.msg_flags = 0;
1536 #endif
1537 #ifdef HAVE_MSGHDR_ACCRIGHTS
1538  msg_msg_hdr.msg_accrights = NULL;
1539 #endif
1540 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1541  msg_msg_hdr.msg_accrightslen = 0;
1542 #endif
1543 
1544  do {
1545  ufd.fd = instance->knet_fd;
1546  ufd.events = POLLIN;
1547  nfds = poll (&ufd, 1, 0);
1548  if (nfds == 1 && ufd.revents & POLLIN) {
1549  res = recvmsg (instance->knet_fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
1550  if (res != -1) {
1551  msg_processed = 1;
1552  } else {
1553  msg_processed = -1;
1554  }
1555  }
1556  } while (nfds == 1);
1557 
1558  return (msg_processed);
1559 }
1560 
1561 int totemknet_iface_set (void *knet_context,
1562  const struct totem_ip_address *local_addr,
1563  unsigned short ip_port,
1564  unsigned int iface_no)
1565 {
1566  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1567 
1568  totemip_copy(&instance->my_ids[iface_no], local_addr);
1569 
1570  knet_log_printf(LOG_INFO, "Configured link number %d: local addr: %s, port=%d", iface_no, totemip_print(local_addr), ip_port);
1571 
1572  instance->ip_port[iface_no] = ip_port;
1573 
1574  return 0;
1575 }
1576 
1577 
1579  void *knet_context,
1580  const struct totem_ip_address *local,
1581  const struct totem_ip_address *member,
1582  int link_no)
1583 {
1584  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1585  int err;
1586  int port = instance->ip_port[link_no];
1587  struct sockaddr_storage remote_ss;
1588  struct sockaddr_storage local_ss;
1589  int addrlen;
1590  int i;
1591  int host_found = 0;
1592  knet_node_id_t host_ids[KNET_MAX_HOST];
1593  size_t num_host_ids;
1594 
1595  /* Only create 1 loopback link and use link 0 */
1596  if (member->nodeid == instance->our_nodeid) {
1597  if (!instance->loopback_link) {
1598  link_no = 0;
1599  instance->loopback_link = 1;
1600  } else {
1601  /* Already done */
1602  return 0;
1603  }
1604  }
1605 
1606  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: " CS_PRI_NODE_ID " (%s), link=%d", member->nodeid, totemip_print(member), link_no);
1607  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: " CS_PRI_NODE_ID " (%s)", local->nodeid, totemip_print(local));
1608 
1609 
1610  /* Only add the host if it doesn't already exist in knet */
1611  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_host_ids);
1612  if (err) {
1613  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list");
1614  return -1;
1615  }
1616  for (i=0; i<num_host_ids; i++) {
1617  if (host_ids[i] == member->nodeid) {
1618  host_found = 1;
1619  }
1620  }
1621 
1622  if (!host_found) {
1623  err = knet_host_add(instance->knet_handle, member->nodeid);
1624  if (err != 0 && errno != EEXIST) {
1625  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
1626  return -1;
1627  }
1628  } else {
1629  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nodeid " CS_PRI_NODE_ID " already added", member->nodeid);
1630  }
1631 
1632 
1633  if (err == 0) {
1634  if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
1635  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
1636  return -1;
1637  }
1638  }
1639 
1640  memset(&local_ss, 0, sizeof(local_ss));
1641  memset(&remote_ss, 0, sizeof(remote_ss));
1642  /* Casts to remove const */
1643  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
1644  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
1645 
1646  if (member->nodeid == instance->our_nodeid) {
1647  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
1648 
1649  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1650  KNET_TRANSPORT_LOOPBACK,
1651  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1652  }
1653  else {
1654  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1655  instance->totem_config->interfaces[link_no].knet_transport,
1656  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1657  }
1658  if (err) {
1659  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
1660  return -1;
1661  }
1662 
1663  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: Setting link prio to %d",
1664  instance->totem_config->interfaces[link_no].knet_link_priority);
1665 
1666  err = knet_link_set_priority(instance->knet_handle, member->nodeid, link_no,
1667  instance->totem_config->interfaces[link_no].knet_link_priority);
1668  if (err) {
1669  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1670  }
1671 
1672  /*
1673  * Ping timeouts may be 0 here for a newly added interface (on a reload),
1674  * so we leave this till later, it will get done in totemknet_refresh_config.
1675  * For the initial startup, we are all preset and ready to go from here.
1676  */
1677  if (instance->totem_config->interfaces[link_no].knet_ping_interval != 0) {
1678  err = knet_link_set_ping_timers(instance->knet_handle, member->nodeid, link_no,
1679  instance->totem_config->interfaces[link_no].knet_ping_interval,
1680  instance->totem_config->interfaces[link_no].knet_ping_timeout,
1681  instance->totem_config->interfaces[link_no].knet_ping_precision);
1682  if (err) {
1683  /* Flush logs before reporting this error so that the knet message prints before ours */
1684  int saved_errno = errno;
1685  log_flush_messages(instance);
1686  errno = saved_errno;
1687  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1688 
1689  icmap_set_string("config.reload_error_message", "Failed to set knet ping timers");
1690 
1691  return -1;
1692  }
1693  err = knet_link_set_pong_count(instance->knet_handle, member->nodeid, link_no,
1694  instance->totem_config->interfaces[link_no].knet_pong_count);
1695  if (err) {
1696  /* Flush logs before reporting this error so that the knet message prints before ours */
1697  int saved_errno = errno;
1698  log_flush_messages(instance);
1699  errno = saved_errno;
1700  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1701  icmap_set_string("config.reload_error_message", "Failed to set knet pong count");
1702  return -1;
1703  }
1704  }
1705 
1706  err = knet_link_set_enable(instance->knet_handle, member->nodeid, link_no, 1);
1707  if (err) {
1708  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_enable for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1709  return -1;
1710  }
1711 
1712  /* register stats */
1713  stats_knet_add_member(member->nodeid, link_no);
1714  return (0);
1715 }
1716 
1718  void *knet_context,
1719  const struct totem_ip_address *token_target,
1720  int link_no)
1721 {
1722  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1723  int res;
1724  uint8_t link_list[KNET_MAX_LINK];
1725  size_t num_links;
1726 
1727  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: " CS_PRI_NODE_ID ", link=%d", token_target->nodeid, link_no);
1728 
1729  /* Don't remove the link with the loopback on it until we shut down */
1730  if (token_target->nodeid == instance->our_nodeid) {
1731  return 0;
1732  }
1733 
1734  /* Tidy stats */
1735  stats_knet_del_member(token_target->nodeid, link_no);
1736 
1737  /* Remove the link first */
1738  res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
1739  if (res != 0) {
1740  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set enable(off) for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1741  return res;
1742  }
1743 
1744  res = knet_link_clear_config(instance->knet_handle, token_target->nodeid, link_no);
1745  if (res != 0) {
1746  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_clear_config for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1747  return res;
1748  }
1749 
1750  /* If this is the last link, then remove the node */
1751  res = knet_link_get_link_list(instance->knet_handle,
1752  token_target->nodeid, link_list, &num_links);
1753  if (res) {
1754  return (0); /* not really failure */
1755  }
1756 
1757  if (num_links == 0) {
1758  res = knet_host_remove(instance->knet_handle, token_target->nodeid);
1759  }
1760  return res;
1761 }
1762 
1764  void *knet_context)
1765 {
1766  return (0);
1767 }
1768 
1769 
1770 static int totemknet_configure_compression (
1771  struct totemknet_instance *instance,
1772  struct totem_config *totem_config)
1773 {
1774  struct knet_handle_compress_cfg compress_cfg;
1775  int res = 0;
1776 
1777  assert(strlen(totem_config->knet_compression_model) < sizeof(compress_cfg.compress_model));
1778  strcpy(compress_cfg.compress_model, totem_config->knet_compression_model);
1779 
1780  compress_cfg.compress_threshold = totem_config->knet_compression_threshold;
1781  compress_cfg.compress_level = totem_config->knet_compression_level;
1782 
1783  res = knet_handle_compress(instance->knet_handle, &compress_cfg);
1784  if (res) {
1785  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
1786  }
1787  return res;
1788 }
1789 
1791  void *knet_context,
1792  struct totem_config *totem_config)
1793 {
1794  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1795  int res = 0;
1796 
1797  (void)totemknet_configure_compression(instance, totem_config);
1798 
1799 #ifdef HAVE_LIBNOZZLE
1800  /* Set up nozzle device(s). Return code is ignored, because inability
1801  * configure nozzle is not fatal problem, errors are logged and
1802  * there is not much else we can do */
1803  (void)setup_nozzle(instance);
1804 #endif
1805 
1806  if (totem_config->crypto_changed) {
1807  /* Flip crypto_index */
1808  totem_config->crypto_index = 3-totem_config->crypto_index;
1809  res = totemknet_set_knet_crypto(instance);
1810  if (res == 0) {
1811  knet_log_printf(LOG_INFO, "kronosnet crypto reconfigured on index %d: %s/%s/%s", totem_config->crypto_index,
1812  totem_config->crypto_model,
1813  totem_config->crypto_cipher_type,
1814  totem_config->crypto_hash_type);
1815  } else {
1816  icmap_set_string("config.reload_error_message", "Failed to set knet crypto");
1817  }
1818  }
1819  return (res);
1820 }
1821 
1822 
1824  void *knet_context,
1825  struct totem_config *totem_config,
1827 {
1828 #ifdef HAVE_KNET_CRYPTO_RECONF
1829  int res;
1830  int config_to_use;
1831  int config_to_clear;
1832  struct knet_handle_crypto_cfg crypto_cfg;
1833  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1834 
1835  knet_log_printf(LOGSYS_LEVEL_DEBUG, "totemknet_crypto_reconfigure_phase %d, index=%d\n", phase, totem_config->crypto_index);
1836 
1837  switch (phase) {
1839  config_to_use = totem_config->crypto_index;
1840  if (!totemknet_is_crypto_enabled(instance)) {
1841  config_to_use = 0; /* we are clearing it */
1842  }
1843 
1844  /* Enable the new config on this node */
1845  res = knet_handle_crypto_use_config(instance->knet_handle, config_to_use);
1846  if (res == -1) {
1847  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_use_config %d failed: %s", config_to_use, strerror(errno));
1848  }
1849  break;
1850 
1852  /*
1853  * All nodes should now have the new config. clear the old one out
1854  * OR disable crypto entirely if that's what the new config insists on.
1855  */
1856  config_to_clear = 3-totem_config->crypto_index;
1857  knet_log_printf(LOGSYS_LEVEL_DEBUG, "Clearing old knet crypto config %d\n", config_to_clear);
1858 
1859  strcpy(crypto_cfg.crypto_model, "none");
1860  strcpy(crypto_cfg.crypto_cipher_type, "none");
1861  strcpy(crypto_cfg.crypto_hash_type, "none");
1862  res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, config_to_clear);
1863  if (res == -1) {
1864  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: %s", config_to_clear, strerror(errno));
1865  }
1866  if (res == -2) {
1867  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: -2", config_to_clear);
1868  }
1869 
1870  /* If crypto is enabled then disable all cleartext reception */
1871  if (totemknet_is_crypto_enabled(instance)) {
1872  res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
1873  if (res) {
1874  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(DISALLOW) failed %s", strerror(errno));
1875  }
1876  }
1877  }
1878 #endif
1879  return 0;
1880 }
1881 
1883  void *knet_context)
1884 {
1885  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1886 
1887  (void) knet_handle_clear_stats(instance->knet_handle, KNET_CLEARSTATS_HANDLE_AND_LINK);
1888 }
1889 
1890 /* For the stats module */
1892  knet_node_id_t node, uint8_t link_no,
1893  struct knet_link_status *status)
1894 {
1895  int res;
1896  int ret = CS_OK;
1897 
1898  /* We are probably not using knet */
1899  if (!global_instance) {
1900  return CS_ERR_NOT_EXIST;
1901  }
1902 
1903  if (link_no >= INTERFACE_MAX) {
1904  return CS_ERR_NOT_EXIST; /* Invalid link number */
1905  }
1906 
1907  res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
1908  if (res) {
1909  switch (errno) {
1910  case EINVAL:
1911  ret = CS_ERR_INVALID_PARAM;
1912  break;
1913  case EBUSY:
1914  ret = CS_ERR_BUSY;
1915  break;
1916  case EDEADLK:
1917  ret = CS_ERR_TRY_AGAIN;
1918  break;
1919  default:
1920  ret = CS_ERR_LIBRARY;
1921  break;
1922  }
1923  }
1924 
1925  return (ret);
1926 }
1927 
1929  struct knet_handle_stats *stats)
1930 {
1931  int res;
1932 
1933  /* We are probably not using knet */
1934  if (!global_instance) {
1935  return CS_ERR_NOT_EXIST;
1936  }
1937 
1938  res = knet_handle_get_stats(global_instance->knet_handle, stats, sizeof(struct knet_handle_stats));
1939  if (res != 0) {
1940  return (qb_to_cs_error(-errno));
1941  }
1942 
1943  return CS_OK;
1944 }
1945 
1946 static void timer_function_merge_detect_timeout (
1947  void *data)
1948 {
1949  struct totemknet_instance *instance = (struct totemknet_instance *)data;
1950 
1951  if (instance->merge_detect_messages_sent_before_timeout == 0) {
1952  instance->send_merge_detect_message = 1;
1953  }
1954 
1956 
1957  totemknet_start_merge_detect_timeout(instance);
1958 }
1959 
1960 static void totemknet_start_merge_detect_timeout(
1961  void *knet_context)
1962 {
1963  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1964 
1965  qb_loop_timer_add(instance->poll_handle,
1966  QB_LOOP_MED,
1967  instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1968  (void *)instance,
1969  timer_function_merge_detect_timeout,
1970  &instance->timer_merge_detect_timeout);
1971 
1972 }
1973 
1974 static void totemknet_stop_merge_detect_timeout(
1975  void *knet_context)
1976 {
1977  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1978 
1979  qb_loop_timer_del(instance->poll_handle,
1980  instance->timer_merge_detect_timeout);
1981 }
1982 
1983 static void log_flush_messages (void *knet_context)
1984 {
1985  struct pollfd pfd;
1986  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1987  int cont;
1988 
1989  cont = 1;
1990 
1991  while (cont) {
1992  pfd.fd = instance->logpipes[0];
1993  pfd.events = POLLIN;
1994  pfd.revents = 0;
1995 
1996  if ((poll(&pfd, 1, 0) > 0) &&
1997  (pfd.revents & POLLIN) &&
1998  (log_deliver_fn(instance->logpipes[0], POLLIN, instance) == 0)) {
1999  cont = 1;
2000  } else {
2001  cont = 0;
2002  }
2003  }
2004 }
2005 
2006 
2007 #ifdef HAVE_LIBNOZZLE
2008 #define NOZZLE_NAME "nozzle.name"
2009 #define NOZZLE_IPADDR "nozzle.ipaddr"
2010 #define NOZZLE_PREFIX "nozzle.ipprefix"
2011 #define NOZZLE_MACADDR "nozzle.macaddr"
2012 
2013 #define NOZZLE_CHANNEL 1
2014 
2015 
2016 static char *get_nozzle_script_dir(void *knet_context)
2017 {
2018  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2019  char filename[PATH_MAX + FILENAME_MAX + 1];
2020  static char updown_dirname[PATH_MAX + FILENAME_MAX + 1];
2021  int res;
2022  const char *dirname_res;
2023 
2024  /*
2025  * Build script directory based on corosync.conf file location
2026  */
2027  res = snprintf(filename, sizeof(filename), "%s",
2029  if (res >= sizeof(filename)) {
2030  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
2031  return NULL;
2032  }
2033 
2034  dirname_res = dirname(filename);
2035 
2036  res = snprintf(updown_dirname, sizeof(updown_dirname), "%s/%s",
2037  dirname_res, "updown.d");
2038  if (res >= sizeof(updown_dirname)) {
2039  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
2040  return NULL;
2041  }
2042  return updown_dirname;
2043 }
2044 
2045 /*
2046  * Deliberately doesn't return the status as caller doesn't care.
2047  * The result will be logged though
2048  */
2049 static void run_nozzle_script(struct totemknet_instance *instance, int type, const char *typename)
2050 {
2051  int res;
2052  char *exec_string;
2053 
2054  res = nozzle_run_updown(instance->nozzle_handle, type, &exec_string);
2055  if (res == -1 && errno != ENOENT) {
2056  knet_log_printf (LOGSYS_LEVEL_INFO, "exec nozzle %s script failed: %s", typename, strerror(errno));
2057  } else if (res == -2) {
2058  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle %s script failed", typename);
2059  knet_log_printf (LOGSYS_LEVEL_INFO, "%s", exec_string);
2060  }
2061 }
2062 
2063 /*
2064  * Reparse IP address to add in our node ID
2065  * IPv6 addresses must end in '::'
2066  * IPv4 addresses must just be valid
2067  * '/xx' lengths are optional for IPv6, mandatory for IPv4
2068  *
2069  * Returns the modified IP address as a string to pass into libnozzle
2070  */
2071 static int reparse_nozzle_ip_address(struct totemknet_instance *instance,
2072  const char *input_addr,
2073  const char *prefix, int nodeid,
2074  char *output_addr, size_t output_len)
2075 {
2076  char *coloncolon;
2077  int bits;
2078  int max_prefix = 64;
2079  uint32_t nodeid_mask;
2080  uint32_t addr_mask;
2081  uint32_t masked_nodeid;
2082  struct in_addr *addr;
2083  struct totem_ip_address totemip;
2084 
2085  coloncolon = strstr(input_addr, "::");
2086  if (!coloncolon) {
2087  max_prefix = 30;
2088  }
2089 
2090  bits = atoi(prefix);
2091  if (bits < 8 || bits > max_prefix) {
2092  knet_log_printf(LOGSYS_LEVEL_ERROR, "nozzle IP address prefix must be >= 8 and <= %d (got %d)", max_prefix, bits);
2093  return -1;
2094  }
2095 
2096  /* IPv6 is easy */
2097  if (coloncolon) {
2098  memcpy(output_addr, input_addr, coloncolon-input_addr);
2099  sprintf(output_addr + (coloncolon-input_addr), "::%x", nodeid);
2100  return 0;
2101  }
2102 
2103  /* For IPv4 we need to parse the address into binary, mask off the required bits,
2104  * add in the masked_nodeid and 'print' it out again
2105  */
2106  nodeid_mask = UINT32_MAX & ((1<<(32 - bits)) - 1);
2107  addr_mask = UINT32_MAX ^ nodeid_mask;
2108  masked_nodeid = nodeid & nodeid_mask;
2109 
2110  if (totemip_parse(&totemip, input_addr, AF_INET)) {
2111  knet_log_printf(LOGSYS_LEVEL_ERROR, "Failed to parse IPv4 nozzle IP address");
2112  return -1;
2113  }
2114  addr = (struct in_addr *)&totemip.addr;
2115  addr->s_addr &= htonl(addr_mask);
2116  addr->s_addr |= htonl(masked_nodeid);
2117 
2118  inet_ntop(AF_INET, addr, output_addr, output_len);
2119  return 0;
2120 }
2121 
2122 static int create_nozzle_device(void *knet_context, const char *name,
2123  const char *ipaddr, const char *prefix,
2124  const char *macaddr)
2125 {
2126  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2127  char device_name[IFNAMSIZ+1];
2128  size_t size = IFNAMSIZ;
2129  int8_t channel = NOZZLE_CHANNEL;
2130  nozzle_t nozzle_dev;
2131  int nozzle_fd;
2132  int res;
2133  char *updown_dir;
2134  char parsed_ipaddr[INET6_ADDRSTRLEN];
2135  char mac[19];
2136 
2137  memset(device_name, 0, size);
2138  memset(&mac, 0, sizeof(mac));
2139  strncpy(device_name, name, size);
2140 
2141  updown_dir = get_nozzle_script_dir(knet_context);
2142  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle script dir is %s", updown_dir);
2143 
2144  nozzle_dev = nozzle_open(device_name, size, updown_dir);
2145  if (!nozzle_dev) {
2146  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to init nozzle device %s: %s", device_name, strerror(errno));
2147  return -1;
2148  }
2149  instance->nozzle_handle = nozzle_dev;
2150 
2151  if (nozzle_set_mac(nozzle_dev, macaddr) < 0) {
2152  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle MAC to %s: %s", mac, strerror(errno));
2153  goto out_clean;
2154  }
2155 
2156  if (reparse_nozzle_ip_address(instance, ipaddr, prefix, instance->our_nodeid, parsed_ipaddr, sizeof(parsed_ipaddr))) {
2157  /* Prints its own errors */
2158  goto out_clean;
2159  }
2160  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle IP address is %s / %d", parsed_ipaddr, atoi(prefix));
2161  if (nozzle_add_ip(nozzle_dev, parsed_ipaddr, prefix) < 0) {
2162  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle IP addr to %s/%s: %s", parsed_ipaddr, prefix, strerror(errno));
2163  goto out_clean;
2164  }
2165 
2166  nozzle_fd = nozzle_get_fd(nozzle_dev);
2167  knet_log_printf (LOGSYS_LEVEL_INFO, "Opened '%s' on fd %d", device_name, nozzle_fd);
2168 
2169  res = knet_handle_add_datafd(instance->knet_handle, &nozzle_fd, &channel);
2170  if (res != 0) {
2171  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add nozzle FD to knet: %s", strerror(errno));
2172  goto out_clean;
2173  }
2174 
2175  run_nozzle_script(instance, NOZZLE_PREUP, "pre-up");
2176 
2177  res = nozzle_set_up(nozzle_dev);
2178  if (res != 0) {
2179  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to set nozzle interface UP: %s", strerror(errno));
2180  goto out_clean;
2181  }
2182  run_nozzle_script(instance, NOZZLE_UP, "up");
2183 
2184  return 0;
2185 
2186 out_clean:
2187  nozzle_close(nozzle_dev);
2188  return -1;
2189 }
2190 
2191 static int remove_nozzle_device(void *knet_context)
2192 {
2193  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2194  int res;
2195  int datafd;
2196 
2197  res = knet_handle_get_datafd(instance->knet_handle, NOZZLE_CHANNEL, &datafd);
2198  if (res != 0) {
2199  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't find datafd for channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
2200  return -1;
2201  }
2202 
2203  res = knet_handle_remove_datafd(instance->knet_handle, datafd);
2204  if (res != 0) {
2205  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't remove datafd for nozzle channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
2206  return -1;
2207  }
2208 
2209  run_nozzle_script(instance, NOZZLE_DOWN, "pre-down");
2210  res = nozzle_set_down(instance->nozzle_handle);
2211  if (res != 0) {
2212  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't set nozzle device down: %s", strerror(errno));
2213  return -1;
2214  }
2215  run_nozzle_script(instance, NOZZLE_POSTDOWN, "post-down");
2216 
2217  res = nozzle_close(instance->nozzle_handle);
2218  if (res != 0) {
2219  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't close nozzle device: %s", strerror(errno));
2220  return -1;
2221  }
2222  knet_log_printf (LOGSYS_LEVEL_INFO, "Removed nozzle device");
2223  return 0;
2224 }
2225 
2226 static void free_nozzle(struct totemknet_instance *instance)
2227 {
2228  free(instance->nozzle_name);
2229  free(instance->nozzle_ipaddr);
2230  free(instance->nozzle_prefix);
2231  free(instance->nozzle_macaddr);
2232 
2233  instance->nozzle_name = instance->nozzle_ipaddr = instance->nozzle_prefix =
2234  instance->nozzle_macaddr = NULL;
2235 }
2236 
2237 static int setup_nozzle(void *knet_context)
2238 {
2239  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2240  char *ipaddr_str = NULL;
2241  char *name_str = NULL;
2242  char *prefix_str = NULL;
2243  char *macaddr_str = NULL;
2244  char mac[32];
2245  int name_res;
2246  int macaddr_res;
2247  int res = -1;
2248 
2249  /*
2250  * Return value ignored on purpose. icmap_get_string changes
2251  * ipaddr_str/prefix_str only on success.
2252  */
2253  (void)icmap_get_string(NOZZLE_IPADDR, &ipaddr_str);
2254  (void)icmap_get_string(NOZZLE_PREFIX, &prefix_str);
2255  macaddr_res = icmap_get_string(NOZZLE_MACADDR, &macaddr_str);
2256  name_res = icmap_get_string(NOZZLE_NAME, &name_str);
2257 
2258  /* Is is being removed? */
2259  if (name_res == CS_ERR_NOT_EXIST && instance->nozzle_handle) {
2260  remove_nozzle_device(instance);
2261  free_nozzle(instance);
2262  goto out_free;
2263  }
2264 
2265  if (!name_str) {
2266  /* no nozzle */
2267  goto out_free;
2268  }
2269 
2270  if (!ipaddr_str) {
2271  knet_log_printf (LOGSYS_LEVEL_ERROR, "No IP address supplied for Nozzle device");
2272  goto out_free;
2273  }
2274 
2275  if (!prefix_str) {
2276  knet_log_printf (LOGSYS_LEVEL_ERROR, "No prefix supplied for Nozzle IP address");
2277  goto out_free;
2278  }
2279 
2280  if (macaddr_str && strlen(macaddr_str) != 17) {
2281  knet_log_printf (LOGSYS_LEVEL_ERROR, "macaddr for nozzle device is not in the correct format '%s'", macaddr_str);
2282  goto out_free;
2283  }
2284  if (!macaddr_str) {
2285  macaddr_str = (char*)"54:54:01:00:00:00";
2286  }
2287 
2288  if (instance->nozzle_name &&
2289  (strcmp(name_str, instance->nozzle_name) == 0) &&
2290  (strcmp(ipaddr_str, instance->nozzle_ipaddr) == 0) &&
2291  (strcmp(prefix_str, instance->nozzle_prefix) == 0) &&
2292  (instance->nozzle_macaddr == NULL ||
2293  strcmp(macaddr_str, instance->nozzle_macaddr) == 0)) {
2294  /* Nothing has changed */
2295  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Nozzle device info not changed");
2296  goto out_free;
2297  }
2298 
2299  /* Add nodeid into MAC address */
2300  memcpy(mac, macaddr_str, 12);
2301  snprintf(mac+12, sizeof(mac) - 13, "%02x:%02x",
2302  instance->our_nodeid >> 8,
2303  instance->our_nodeid & 0xFF);
2304  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle MAC address is %s", mac);
2305 
2306  if (name_res == CS_OK && name_str) {
2307  /* Reconfigure */
2308  if (instance->nozzle_name) {
2309  remove_nozzle_device(instance);
2310  free_nozzle(instance);
2311  }
2312 
2313  res = create_nozzle_device(knet_context, name_str, ipaddr_str, prefix_str,
2314  mac);
2315 
2316  instance->nozzle_name = strdup(name_str);
2317  instance->nozzle_ipaddr = strdup(ipaddr_str);
2318  instance->nozzle_prefix = strdup(prefix_str);
2319  instance->nozzle_macaddr = strdup(macaddr_str);
2320  if (!instance->nozzle_name || !instance->nozzle_ipaddr ||
2321  !instance->nozzle_prefix) {
2322  knet_log_printf (LOGSYS_LEVEL_ERROR, "strdup failed in nozzle allocation");
2323  /*
2324  * This 'free' will cause a complete reconfigure of the device next time we reload
2325  * but will also let the the current device keep working until then.
2326  * remove_nozzle() only needs the, statically-allocated, nozzle_handle
2327  */
2328  free_nozzle(instance);
2329  }
2330  }
2331 
2332 out_free:
2333  free(name_str);
2334  free(ipaddr_str);
2335  free(prefix_str);
2336  if (macaddr_res == CS_OK) {
2337  free(macaddr_str);
2338  }
2339 
2340  return res;
2341 }
2342 #endif // HAVE_LIBNOZZLE
#define LOGSYS_DEBUG_TRACE
Definition: logsys.h:94
uint16_t ip_port[INTERFACE_MAX]
Definition: totemknet.c:157
#define NOZZLE_CHANNEL
Definition: totemknet.c:2013
int knet_ping_precision
Definition: totem.h:94
char knet_compression_model[CONFIG_STRING_LEN_MAX]
Definition: totem.h:234
int totemknet_log_level_security
Definition: totemknet.c:126
cfg_message_crypto_reconfig_phase_t
Definition: totem.h:154
qb_loop_t * poll_handle
Definition: totemknet.c:98
char * link_status[INTERFACE_MAX]
Definition: totemknet.c:153
int totemknet_member_remove(void *knet_context, const struct totem_ip_address *token_target, int link_no)
Definition: totemknet.c:1717
struct knet_link_status link_status[KNET_MAX_LINK]
Definition: totem.h:274
int crypto_changed
Definition: totem.h:232
#define MSG_NOSIGNAL
Definition: totemknet.c:83
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:75
int knet_link_priority
Definition: totem.h:91
#define LOGSYS_LEVEL_TRACE
Definition: logsys.h:77
uint32_t value
qb_loop_timer_handle timer_merge_detect_timeout
Definition: totemknet.c:169
struct totem_interface * interfaces
Definition: totem.h:165
void stats_knet_add_handle(void)
Definition: stats.c:753
int totemknet_log_level_error
Definition: totemknet.c:128
unsigned int merge_detect_messages_sent_before_timeout
Definition: totemknet.c:173
#define libknet_log_printf(level, format, args...)
Definition: totemknet.c:243
struct totem_ip_address my_ids[INTERFACE_MAX]
Definition: totemknet.c:155
The totem_ip_address struct.
Definition: coroapi.h:111
#define CFG_INTERFACE_STATUS_MAX_LEN
Definition: totemknet.c:91
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:256
#define knet_log_printf(level, format, args...)
Definition: totemknet.c:235
int send_merge_detect_message
Definition: totemknet.c:171
#define CS_PRI_NODE_ID
Definition: corotypes.h:59
#define corosync_exit_error(err)
Definition: exec/util.h:72
int totemknet_finalize(void *knet_context)
Definition: totemknet.c:666
uint32_t knet_compression_threshold
Definition: totem.h:236
char crypto_hash_type[CONFIG_STRING_LEN_MAX]
Definition: totem.h:228
nozzle_t nozzle_handle
Definition: totemknet.c:184
char * nozzle_macaddr
Definition: totemknet.c:183
char link_mode[TOTEM_LINK_MODE_BYTES]
Definition: totem.h:206
unsigned int knet_pmtud_interval
Definition: totem.h:169
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:77
int totemknet_link_get_status(knet_node_id_t node, uint8_t link_no, struct knet_link_status *status)
Definition: totemknet.c:1891
cs_error_t icmap_set_string(const char *key_name, const char *value)
Definition: icmap.c:631
void totemknet_buffer_release(void *ptr)
Definition: totemknet.c:1417
void totemknet_stats_clear(void *knet_context)
Definition: totemknet.c:1882
#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...)
Definition: totemknet.c:251
unsigned int knet_mtu
Definition: totem.h:170
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:123
void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:740
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition: logsys.c:435
#define NOZZLE_PREFIX
Definition: totemknet.c:2010
#define LOGSYS_DEBUG_ON
Definition: logsys.h:93
unsigned int private_key_len
Definition: totem.h:177
#define log_printf(level, format, args...)
Definition: logsys.h:332
int totemknet_log_level_notice
Definition: totemknet.c:132
uint8_t reachable
Definition: totem.h:268
#define NOZZLE_MACADDR
Definition: totemknet.c:2011
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
#define INTERFACE_MAX
Definition: coroapi.h:88
int totemknet_crypto_set(void *knet_context, const char *cipher_type, const char *hash_type)
Definition: totemknet.c:365
unsigned int block_unlisted_ips
Definition: totem.h:246
uint8_t onwire_ver
Definition: totem.h:273
qb_loop_timer_handle timer_netif_check_timeout
Definition: totemknet.c:167
int totemknet_mcast_flush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1451
int totemknet_iface_check(void *knet_context)
Definition: totemknet.c:1478
void(*) void knet_context)
Definition: totemknet.c:147
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition: icmap.c:872
unsigned int nodeid
Definition: totem.h:267
unsigned int node_id
Definition: totem.h:167
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:73
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
uint8_t configured
Definition: totem.h:89
int totemknet_log_level_warning
Definition: totemknet.c:130
#define OWN_INDEX_NONE
Definition: totemknet.c:490
const char * corosync_get_config_file(void)
Definition: main.c:212
void * user_data
Definition: sam.c:127
int totemknet_token_target_set(void *knet_context, unsigned int nodeid)
Definition: totemknet.c:1495
struct totem_config * totem_config
Definition: totemknet.c:163
char * nozzle_ipaddr
Definition: totemknet.c:181
void totemknet_configure_log_level()
Definition: totemknet.c:903
unsigned int nodeid
Definition: coroapi.h:112
int totemknet_reconfigure(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1790
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
int totemknet_iface_set(void *knet_context, const struct totem_ip_address *local_addr, unsigned short ip_port, unsigned int iface_no)
Definition: totemknet.c:1561
int knet_transport
Definition: totem.h:96
int totemknet_processor_count_set(void *knet_context, int processor_count)
Definition: totemknet.c:1422
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:72
int totemknet_recv_flush(void *knet_context)
Definition: totemknet.c:1429
int totemknet_send_flush(void *knet_context)
Definition: totemknet.c:1434
#define LOGSYS_DEBUG_OFF
Definition: logsys.h:92
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN_MAX]
Definition: totem.h:175
uint8_t onwire_min
Definition: totem.h:271
cs_error_t
The cs_error_t enum.
Definition: corotypes.h:98
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:76
int totemknet_crypto_reconfigure_phase(void *knet_context, struct totem_config *totem_config, cfg_message_crypto_reconfig_phase_t phase)
Definition: totemknet.c:1823
#define NOZZLE_IPADDR
Definition: totemknet.c:2009
struct totem_ip_address boundto
Definition: totem.h:84
#define NOZZLE_NAME
Definition: totemknet.c:2008
char iov_buffer[KNET_MAX_PACKET_SIZE]
Definition: totemknet.c:151
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:101
int totemknet_handle_get_stats(struct knet_handle_stats *stats)
Definition: totemknet.c:1928
struct totemknet_instance * global_instance
Definition: totemknet.c:189
struct totem_message_header header
Definition: totemsrp.c:260
uint16_t ip_port
Definition: totem.h:87
int knet_compression_level
Definition: totem.h:238
int totemip_parse(struct totem_ip_address *totemip, const char *addr, enum totem_ip_version_enum ip_version)
Definition: totemip.c:306
struct crypto_instance * crypto_inst
Definition: totemknet.c:94
#define ENTER
Definition: logsys.h:333
int(* totemknet_iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no)
Definition: totemknet.c:112
unsigned int net_mtu
Definition: totem.h:210
int totemknet_initialize(qb_loop_t *poll_handle, void **knet_context, struct totem_config *totem_config, totemsrp_stats_t *stats, void *context, int(*deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from), int(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no), void(*mtu_changed)(void *context, int net_mtu), void(*target_set_completed)(void *context))
Definition: totemknet.c:1152
void(* totemknet_target_set_completed)(void *context)
Definition: totemknet.c:121
char crypto_cipher_type[CONFIG_STRING_LEN_MAX]
Definition: totem.h:226
struct totem_ip_address token_target
Definition: totemknet.c:165
struct knet_handle_crypto_cfg last_good_crypto_cfg
Definition: totemknet.c:96
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:96
uint8_t external
Definition: totem.h:270
int crypto_index
Definition: totem.h:230
pthread_mutex_t log_mutex
Definition: totemknet.c:178
char crypto_model[CONFIG_STRING_LEN_MAX]
Definition: totem.h:224
int totemknet_member_add(void *knet_context, const struct totem_ip_address *local, const struct totem_ip_address *member, int link_no)
Definition: totemknet.c:1578
uint8_t onwire_max
Definition: totem.h:272
int knet_pong_count
Definition: totem.h:95
struct totemknet_instance * instance
Definition: totemknet.c:194
cs_error_t icmap_get_string(const char *key_name, char **str)
Shortcut for icmap_get for string type.
Definition: icmap.c:860
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:71
int knet_ping_interval
Definition: totem.h:92
const void * msg
Definition: totemknet.c:192
int knet_ping_timeout
Definition: totem.h:93
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition: totemip.c:264
void(* totemknet_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition: totemknet.c:140
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:208
typedef __attribute__
void(* totemknet_mtu_changed)(void *context, int net_mtu)
Definition: totemknet.c:117
void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:730
uint8_t remote
Definition: totem.h:269
struct srp_addr system_from
Definition: totemsrp.c:261
char type
Definition: totem.h:55
void totemknet_net_mtu_adjust(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1488
int totemknet_member_list_rebind_ip(void *knet_context)
Definition: totemknet.c:1763
unsigned int merge_timeout
Definition: totem.h:198
int totemknet_mcast_noflush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1464
int totemknet_log_level_debug
Definition: totemknet.c:134
unsigned int target_nodeid
Definition: totem.h:132
int logsys_config_debug_get(const char *subsys)
Return the debug flag for this subsys.
Definition: logsys.c:806
int(* totemknet_deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from)
Definition: totemknet.c:106
int totemknet_token_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1439
struct totem_ip_address bindnet
Definition: totem.h:83
unsigned int nodeid
Definition: coroapi.h:75
int totemknet_recv_mcast_empty(void *knet_context)
Definition: totemknet.c:1509
cs_error_t qb_to_cs_error(int result)
qb_to_cs_error
char * nozzle_prefix
Definition: totemknet.c:182
unsigned int msg_len
Definition: totemknet.c:193
#define LEAVE
Definition: logsys.h:334
int totemknet_nodestatus_get(void *knet_context, unsigned int nodeid, struct totem_node_status *node_status)
Definition: totemknet.c:493
int totemknet_ifaces_get(void *knet_context, char ***status, unsigned int *iface_count)
Definition: totemknet.c:571
Structure passed as new_value and old_value in change callback.
Definition: icmap.h:91
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition: icmap.c:1163
#define ICMAP_TRACK_PREFIX
Whole prefix is tracked, instead of key only (so "totem." tracking means that "totem.nodeid", "totem.version", ...
Definition: icmap.h:85
void * totemknet_buffer_alloc(void)
Definition: totemknet.c:1411
knet_handle_t knet_handle
Definition: totemknet.c:100