corosync  2.3.2
vsf_quorum.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2008-2012 Red Hat, Inc.
3  *
4  * All rights reserved.
5  *
6  * Author: Christine Caulfield (ccaulfie@redhat.com)
7  *
8  * This software licensed under BSD license, the text of which follows:
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions are met:
12  *
13  * - Redistributions of source code must retain the above copyright notice,
14  * this list of conditions and the following disclaimer.
15  * - Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  * - Neither the name of Red Hat Inc. nor the names of its
19  * contributors may be used to endorse or promote products derived from this
20  * software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32  * THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #include <config.h>
36 
37 #include <pwd.h>
38 #include <grp.h>
39 #include <sys/types.h>
40 #include <sys/poll.h>
41 #include <sys/uio.h>
42 #include <sys/mman.h>
43 #include <sys/socket.h>
44 #include <sys/un.h>
45 #include <sys/time.h>
46 #include <sys/resource.h>
47 #include <netinet/in.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <sched.h>
55 #include <time.h>
56 
57 #include "quorum.h"
58 #include <corosync/corotypes.h>
59 #include <qb/qbipc_common.h>
60 #include <corosync/corodefs.h>
61 #include <corosync/swab.h>
62 #include <corosync/list.h>
63 #include <corosync/mar_gen.h>
64 #include <corosync/ipc_quorum.h>
65 #include <corosync/mar_gen.h>
66 #include <corosync/coroapi.h>
67 #include <corosync/logsys.h>
68 #include <corosync/icmap.h>
69 
70 #include "service.h"
71 #include "votequorum.h"
72 #include "vsf_ykd.h"
73 
74 LOGSYS_DECLARE_SUBSYS ("QUORUM");
75 
76 struct quorum_pd {
77  unsigned char track_flags;
78  int tracking_enabled;
79  struct list_head list;
80  void *conn;
81 };
82 
84  struct list_head list;
86  void *context;
87 };
88 
89 static void message_handler_req_lib_quorum_getquorate (void *conn,
90  const void *msg);
91 static void message_handler_req_lib_quorum_trackstart (void *conn,
92  const void *msg);
93 static void message_handler_req_lib_quorum_trackstop (void *conn,
94  const void *msg);
95 static void message_handler_req_lib_quorum_gettype (void *conn,
96  const void *msg);
97 static void send_library_notification(void *conn);
98 static void send_internal_notification(void);
99 static char *quorum_exec_init_fn (struct corosync_api_v1 *api);
100 static int quorum_lib_init_fn (void *conn);
101 static int quorum_lib_exit_fn (void *conn);
102 
103 static int primary_designated = 0;
104 static int quorum_type = 0;
105 static struct corosync_api_v1 *corosync_api;
106 static struct list_head lib_trackers_list;
107 static struct list_head internal_trackers_list;
108 static struct memb_ring_id quorum_ring_id;
109 static size_t quorum_view_list_entries = 0;
110 static int quorum_view_list[PROCESSOR_COUNT_MAX];
111 struct quorum_services_api_ver1 *quorum_iface = NULL;
112 static char view_buf[64];
113 
114 static void log_view_list(const unsigned int *view_list, size_t view_list_entries)
115 {
116  int total = (int)view_list_entries;
117  int len, pos, ret;
118  int i = 0;
119 
120  while (1) {
121  len = sizeof(view_buf);
122  pos = 0;
123  memset(view_buf, 0, len);
124 
125  for (; i < total; i++) {
126  ret = snprintf(view_buf + pos, len - pos, " %d", view_list[i]);
127  if (ret >= len - pos)
128  break;
129  pos += ret;
130  }
131  log_printf (LOGSYS_LEVEL_NOTICE, "Members[%d]:%s%s",
132  total, view_buf, i < total ? "\\" : "");
133 
134  if (i == total)
135  break;
136  }
137 }
138 
139 /* Internal quorum API function */
140 static void quorum_api_set_quorum(const unsigned int *view_list,
141  size_t view_list_entries,
142  int quorum, struct memb_ring_id *ring_id)
143 {
144  int old_quorum = primary_designated;
145  primary_designated = quorum;
146 
147  if (primary_designated && !old_quorum) {
148  log_printf (LOGSYS_LEVEL_NOTICE, "This node is within the primary component and will provide service.");
149  } else if (!primary_designated && old_quorum) {
150  log_printf (LOGSYS_LEVEL_NOTICE, "This node is within the non-primary component and will NOT provide any services.");
151  }
152 
153  quorum_view_list_entries = view_list_entries;
154  memcpy(&quorum_ring_id, ring_id, sizeof (quorum_ring_id));
155  memcpy(quorum_view_list, view_list, sizeof(unsigned int)*view_list_entries);
156 
157  log_view_list(view_list, view_list_entries);
158 
159  /* Tell internal listeners */
160  send_internal_notification();
161 
162  /* Tell IPC listeners */
163  send_library_notification(NULL);
164 }
165 
166 static struct corosync_lib_handler quorum_lib_service[] =
167 {
168  { /* 0 */
169  .lib_handler_fn = message_handler_req_lib_quorum_getquorate,
170  .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED
171  },
172  { /* 1 */
173  .lib_handler_fn = message_handler_req_lib_quorum_trackstart,
174  .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED
175  },
176  { /* 2 */
177  .lib_handler_fn = message_handler_req_lib_quorum_trackstop,
178  .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED
179  },
180  { /* 3 */
181  .lib_handler_fn = message_handler_req_lib_quorum_gettype,
182  .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED
183  }
184 };
185 
186 static struct corosync_service_engine quorum_service_handler = {
187  .name = "corosync cluster quorum service v0.1",
188  .id = QUORUM_SERVICE,
189  .priority = 1,
190  .private_data_size = sizeof (struct quorum_pd),
191  .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED,
192  .allow_inquorate = CS_LIB_ALLOW_INQUORATE,
193  .lib_init_fn = quorum_lib_init_fn,
194  .lib_exit_fn = quorum_lib_exit_fn,
195  .lib_engine = quorum_lib_service,
196  .exec_init_fn = quorum_exec_init_fn,
197  .lib_engine_count = sizeof (quorum_lib_service) / sizeof (struct corosync_lib_handler)
198 };
199 
201 {
202  return (&quorum_service_handler);
203 }
204 
205 /* -------------------------------------------------- */
206 
207 
208 /*
209  * Internal API functions for corosync
210  */
211 
212 static int quorum_quorate(void)
213 {
214  return primary_designated;
215 }
216 
217 
218 static int quorum_register_callback(quorum_callback_fn_t function, void *context)
219 {
220  struct internal_callback_pd *pd = malloc(sizeof(struct internal_callback_pd));
221  if (!pd)
222  return -1;
223 
224  pd->context = context;
225  pd->callback = function;
226  list_add (&pd->list, &internal_trackers_list);
227 
228  return 0;
229 }
230 
231 static int quorum_unregister_callback(quorum_callback_fn_t function, void *context)
232 {
233  struct internal_callback_pd *pd;
234  struct list_head *tmp;
235 
236  for (tmp = internal_trackers_list.next; tmp != &internal_trackers_list; tmp = tmp->next) {
237 
238  pd = list_entry(tmp, struct internal_callback_pd, list);
239  if (pd->callback == function && pd->context == context) {
240  list_del(&pd->list);
241  return 0;
242  }
243  }
244  return -1;
245 }
246 
247 static struct quorum_callin_functions callins = {
248  .quorate = quorum_quorate,
249  .register_callback = quorum_register_callback,
250  .unregister_callback = quorum_unregister_callback
251 };
252 
253 /* --------------------------------------------------------------------- */
254 
255 static char *quorum_exec_init_fn (struct corosync_api_v1 *api)
256 {
257  char *quorum_module = NULL;
258  char *error;
259 
260  corosync_api = api;
261  list_init (&lib_trackers_list);
262  list_init (&internal_trackers_list);
263 
264  /*
265  * Tell corosync we have a quorum engine.
266  */
267  api->quorum_initialize(&callins);
268 
269  /*
270  * Look for a quorum provider
271  */
272  if (icmap_get_string("quorum.provider", &quorum_module) == CS_OK) {
274  "Using quorum provider %s", quorum_module);
275 
276  error = (char *)"Invalid quorum provider";
277 
278  if (strcmp (quorum_module, "corosync_votequorum") == 0) {
279  error = votequorum_init (api, quorum_api_set_quorum);
280  quorum_type = 1;
281  }
282  if (strcmp (quorum_module, "corosync_ykd") == 0) {
283  error = ykd_init (api, quorum_api_set_quorum);
284  quorum_type = 1;
285  }
286  if (error) {
288  "Quorum provider: %s failed to initialize.",
289  quorum_module);
290  free(quorum_module);
291  return (error);
292  }
293  }
294 
295  if (quorum_module) {
296  free(quorum_module);
297  quorum_module = NULL;
298  }
299 
300  /*
301  * setting quorum_type and primary_designated in the right order is important
302  * always try to lookup/init a quorum module, then revert back to be quorate
303  */
304 
305  if (quorum_type == 0) {
306  primary_designated = 1;
307  }
308 
309  return (NULL);
310 }
311 
312 static int quorum_lib_init_fn (void *conn)
313 {
314  struct quorum_pd *pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
315 
316  log_printf(LOGSYS_LEVEL_DEBUG, "lib_init_fn: conn=%p", conn);
317 
318  list_init (&pd->list);
319  pd->conn = conn;
320 
321  return (0);
322 }
323 
324 static int quorum_lib_exit_fn (void *conn)
325 {
326  struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
327 
328  log_printf(LOGSYS_LEVEL_DEBUG, "lib_exit_fn: conn=%p", conn);
329 
330  if (quorum_pd->tracking_enabled) {
331  list_del (&quorum_pd->list);
332  list_init (&quorum_pd->list);
333  }
334  return (0);
335 }
336 
337 
338 static void send_internal_notification(void)
339 {
340  struct list_head *tmp;
341  struct internal_callback_pd *pd;
342 
343  for (tmp = internal_trackers_list.next; tmp != &internal_trackers_list; tmp = tmp->next) {
344 
345  pd = list_entry(tmp, struct internal_callback_pd, list);
346 
347  pd->callback(primary_designated, pd->context);
348  }
349 }
350 
351 static void send_library_notification(void *conn)
352 {
353  int size = sizeof(struct res_lib_quorum_notification) + sizeof(unsigned int)*quorum_view_list_entries;
354  char buf[size];
355  struct res_lib_quorum_notification *res_lib_quorum_notification = (struct res_lib_quorum_notification *)buf;
356  struct list_head *tmp;
357  int i;
358 
359  log_printf(LOGSYS_LEVEL_DEBUG, "sending quorum notification to %p, length = %d", conn, size);
360 
361  res_lib_quorum_notification->quorate = primary_designated;
362  res_lib_quorum_notification->ring_seq = quorum_ring_id.seq;
363  res_lib_quorum_notification->view_list_entries = quorum_view_list_entries;
364  for (i=0; i<quorum_view_list_entries; i++) {
365  res_lib_quorum_notification->view_list[i] = quorum_view_list[i];
366  }
367 
368  res_lib_quorum_notification->header.id = MESSAGE_RES_QUORUM_NOTIFICATION;
369  res_lib_quorum_notification->header.size = size;
370  res_lib_quorum_notification->header.error = CS_OK;
371 
372  /* Send it to all interested parties */
373  if (conn) {
374  corosync_api->ipc_dispatch_send(conn, res_lib_quorum_notification, size);
375  }
376  else {
377  struct quorum_pd *qpd;
378 
379  for (tmp = lib_trackers_list.next; tmp != &lib_trackers_list; tmp = tmp->next) {
380 
381  qpd = list_entry(tmp, struct quorum_pd, list);
382 
383  corosync_api->ipc_dispatch_send(qpd->conn,
384  res_lib_quorum_notification, size);
385  }
386  }
387  return;
388 }
389 
390 static void message_handler_req_lib_quorum_getquorate (void *conn,
391  const void *msg)
392 {
394 
395  log_printf(LOGSYS_LEVEL_DEBUG, "got quorate request on %p", conn);
396 
397  /* send status */
398  res_lib_quorum_getquorate.quorate = primary_designated;
401  res_lib_quorum_getquorate.header.error = CS_OK;
403 }
404 
405 static void message_handler_req_lib_quorum_trackstart (void *conn,
406  const void *msg)
407 {
409  struct qb_ipc_response_header res;
410  struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
411 
412  log_printf(LOGSYS_LEVEL_DEBUG, "got trackstart request on %p", conn);
413 
414  /*
415  * If an immediate listing of the current cluster membership
416  * is requested, generate membership list
417  */
418  if (req_lib_quorum_trackstart->track_flags & CS_TRACK_CURRENT ||
419  req_lib_quorum_trackstart->track_flags & CS_TRACK_CHANGES) {
420  log_printf(LOGSYS_LEVEL_DEBUG, "sending initial status to %p", conn);
421  send_library_notification(conn);
422  }
423 
424  /*
425  * Record requests for tracking
426  */
427  if (req_lib_quorum_trackstart->track_flags & CS_TRACK_CHANGES ||
428  req_lib_quorum_trackstart->track_flags & CS_TRACK_CHANGES_ONLY) {
429 
430  quorum_pd->track_flags = req_lib_quorum_trackstart->track_flags;
431  quorum_pd->tracking_enabled = 1;
432 
433  list_add (&quorum_pd->list, &lib_trackers_list);
434  }
435 
436  /* send status */
437  res.size = sizeof(res);
439  res.error = CS_OK;
440  corosync_api->ipc_response_send(conn, &res, sizeof(struct qb_ipc_response_header));
441 }
442 
443 static void message_handler_req_lib_quorum_trackstop (void *conn, const void *msg)
444 {
445  struct qb_ipc_response_header res;
446  struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
447 
448  log_printf(LOGSYS_LEVEL_DEBUG, "got trackstop request on %p", conn);
449 
450  if (quorum_pd->tracking_enabled) {
451  res.error = CS_OK;
452  quorum_pd->tracking_enabled = 0;
453  list_del (&quorum_pd->list);
454  list_init (&quorum_pd->list);
455  } else {
456  res.error = CS_ERR_NOT_EXIST;
457  }
458 
459  /* send status */
460  res.size = sizeof(res);
462  res.error = CS_OK;
463  corosync_api->ipc_response_send(conn, &res, sizeof(struct qb_ipc_response_header));
464 }
465 
466 static void message_handler_req_lib_quorum_gettype (void *conn,
467  const void *msg)
468 {
470 
471  log_printf(LOGSYS_LEVEL_DEBUG, "got quorum_type request on %p", conn);
472 
473  /* send status */
475  res_lib_quorum_gettype.header.size = sizeof(res_lib_quorum_gettype);
477  res_lib_quorum_gettype.header.error = CS_OK;
478  corosync_api->ipc_response_send(conn, &res_lib_quorum_gettype, sizeof(res_lib_quorum_gettype));
479 }
480 
int(* quorate)(void)
Definition: coroapi.h:163
const char * name
Definition: coroapi.h:432
struct corosync_service_engine * vsf_quorum_get_service_engine_ver0(void)
Definition: vsf_quorum.c:200
void(* lib_handler_fn)(void *conn, const void *msg)
Definition: coroapi.h:418
struct list_head list
Definition: vsf_quorum.c:84
struct list_head * next
Definition: list.h:47
char * votequorum_init(struct corosync_api_v1 *api, quorum_set_quorate_fn_t q_set_quorate_fn)
char * ykd_init(struct corosync_api_v1 *corosync_api, quorum_set_quorate_fn_t set_primary)
Definition: vsf_ykd.c:511
int tracking_enabled
quorum_callback_fn_t callback
Definition: vsf_quorum.c:85
#define CS_TRACK_CURRENT
Definition: corotypes.h:74
struct quorum_services_api_ver1 * quorum_iface
Definition: vsf_quorum.c:111
Definition: list.h:46
#define log_printf(level, format, args...)
Definition: logsys.h:217
#define CS_TRACK_CHANGES
Definition: corotypes.h:75
mar_uint32_t view_list[]
Definition: ipc_quorum.h:71
int(* quorum_initialize)(struct quorum_callin_functions *fns)
Definition: coroapi.h:345
void *(* ipc_private_data_get)(void *conn)
Definition: coroapi.h:208
struct list_head list
#define LOGSYS_DECLARE_SUBSYS(subsys)
Definition: logsys.h:197
#define CS_TRACK_CHANGES_ONLY
Definition: corotypes.h:76
Linked list API.
unsigned char track_flags
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:74
int(* ipc_dispatch_send)(void *conn, const void *msg, size_t mlen)
Definition: coroapi.h:215
int(* ipc_response_send)(void *conn, const void *msg, size_t mlen)
Definition: coroapi.h:210
unsigned int track_flags
Definition: ipc_quorum.h:57
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:83
mar_uint32_t quorum_type
Definition: ipc_quorum.h:76
cs_error_t icmap_get_string(const char *key_name, char **str)
Definition: icmap.c:896
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:69
#define list_entry(ptr, type, member)
Definition: list.h:84
#define LOGSYS_LEVEL_NOTICE
Definition: logsys.h:72
unsigned long long seq
Definition: coroapi.h:105
void(* quorum_callback_fn_t)(int quorate, void *context)
Definition: coroapi.h:160
struct memb_ring_id ring_id
Definition: totemsrp.c:153