76 #include <sys/types.h>
81 #include <sys/socket.h>
84 #include <sys/resource.h>
86 #include <netinet/in.h>
87 #include <arpa/inet.h>
96 #include <semaphore.h>
98 #include <qb/qbdefs.h>
100 #include <qb/qbloop.h>
101 #include <qb/qbutil.h>
102 #include <qb/qbipcs.h>
124 #ifdef HAVE_SMALL_MEMORY_FOOTPRINT
125 #define IPC_LOGSYS_SIZE 1024*64
127 #define IPC_LOGSYS_SIZE 8192*128
137 #define SERVER_BACKLOG 5
139 static int sched_priority = 0;
141 static unsigned int service_count = 32;
147 static int sync_in_process = 1;
149 static qb_loop_t *corosync_poll_handle;
155 static const char *corosync_lock_file =
LOCALSTATEDIR"/run/corosync.pid";
157 static int ip_version = AF_INET;
161 return (corosync_poll_handle);
169 int (*dispatch_fn) (
int fd,
173 return qb_loop_poll_add(handle, QB_LOOP_MED, fd, events, data,
179 return qb_loop_poll_del(handle, fd);
193 static void corosync_blackbox_write_to_file (
void)
195 char fname[PATH_MAX];
196 char time_str[PATH_MAX];
197 struct tm cur_time_tm;
200 cur_time_t = time(NULL);
201 localtime_r(&cur_time_t, &cur_time_tm);
203 strftime(time_str, PATH_MAX,
"%Y-%m-%dT%H:%M:%S", &cur_time_tm);
204 snprintf(fname, PATH_MAX,
"%s/fdata-%s-%lld",
207 (
long long int)getpid());
209 qb_log_blackbox_write_to_file(fname);
215 static void unlink_all_completed (
void)
218 qb_loop_stop (corosync_poll_handle);
227 static int32_t sig_diag_handler (
int num,
void *data)
233 static int32_t sig_exit_handler (
int num,
void *data)
240 static void sigsegv_handler (
int num)
242 (void)signal (SIGSEGV, SIG_DFL);
243 corosync_blackbox_write_to_file ();
248 static void sigabrt_handler (
int num)
250 (void)signal (SIGABRT, SIG_DFL);
251 corosync_blackbox_write_to_file ();
256 #define LOCALHOST_IP inet_addr("127.0.0.1")
258 static void *corosync_group_handle;
265 static void serialize_lock (
void)
269 static void serialize_unlock (
void)
273 static void corosync_sync_completed (
void)
276 "Completed service synchronization, ready to provide service.");
287 static int corosync_sync_callbacks_retrieve (
295 if (callbacks == NULL) {
310 static void member_object_joined (
unsigned int nodeid)
317 "runtime.totem.pg.mrp.srp.members.%u.ip", nodeid);
319 "runtime.totem.pg.mrp.srp.members.%u.join_count", nodeid);
321 "runtime.totem.pg.mrp.srp.members.%u.status", nodeid);
336 static void member_object_left (
unsigned int nodeid)
341 "runtime.totem.pg.mrp.srp.members.%u.status", nodeid);
348 static void confchg_fn (
350 const unsigned int *member_list,
size_t member_list_entries,
351 const unsigned int *left_list,
size_t left_list_entries,
352 const unsigned int *joined_list,
size_t joined_list_entries,
356 int abort_activate = 0;
358 if (sync_in_process == 1) {
363 memcpy (&corosync_ring_id, ring_id,
sizeof (
struct memb_ring_id));
365 for (i = 0; i < left_list_entries; i++) {
366 member_object_left (left_list[i]);
368 for (i = 0; i < joined_list_entries; i++) {
369 member_object_joined (joined_list[i]);
374 for (i = 0; i < service_count; i++) {
377 member_list, member_list_entries,
378 left_list, left_list_entries,
379 joined_list, joined_list_entries, ring_id);
383 if (abort_activate) {
390 sync_start (member_list, member_list_entries, ring_id);
394 static void priv_drop (
void)
399 static void corosync_tty_detach (
void)
427 devnull = open(
"/dev/null", O_RDWR);
432 if (dup2(devnull, 0) < 0 || dup2(devnull, 1) < 0
433 || dup2(devnull, 2) < 0) {
438 static void corosync_mlockall (
void)
441 struct rlimit rlimit;
443 rlimit.rlim_cur = RLIM_INFINITY;
444 rlimit.rlim_max = RLIM_INFINITY;
446 #ifndef RLIMIT_MEMLOCK
447 #define RLIMIT_MEMLOCK RLIMIT_VMEM
452 res = mlockall (MCL_CURRENT | MCL_FUTURE);
455 "Could not lock memory of service to avoid page faults");
460 static void corosync_totem_stats_updater (
void *data)
463 uint32_t total_mtt_rx_token;
464 uint32_t total_backlog_calc;
465 uint32_t total_token_holdtime;
501 icmap_set_uint8(
"runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure",
507 "Totem is unable to form a cluster because of an "
508 "operating system or network fault. The most common "
509 "cause of this message is that the local firewall is "
510 "configured improperly.");
511 icmap_set_uint8(
"runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure", 1);
513 icmap_set_uint8(
"runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure", 0);
520 total_mtt_rx_token = 0;
521 total_token_holdtime = 0;
522 total_backlog_calc = 0;
543 icmap_set_uint32(
"runtime.totem.pg.mrp.srp.mtt_rx_token", (total_mtt_rx_token / token_count));
544 icmap_set_uint32(
"runtime.totem.pg.mrp.srp.avg_token_workload", (total_token_holdtime / token_count));
545 icmap_set_uint32(
"runtime.totem.pg.mrp.srp.avg_backlog_calc", (total_backlog_calc / token_count));
551 corosync_totem_stats_updater,
552 &corosync_stats_timer_handle);
555 static void totem_dynamic_notify(
557 const char *key_name,
566 int add_new_member = 0;
567 int remove_old_member = 0;
570 res = sscanf(key_name,
"nodelist.node.%u.ring%u%s", &member_no, &ring_no, tmp_str);
574 if (strcmp(tmp_str,
"_addr") != 0) {
583 remove_old_member = 1;
589 remove_old_member = 1;
592 if (remove_old_member) {
594 "removing dynamic member %s for ring %u", (
char *)old_val.
data, ring_no);
600 if (add_new_member) {
602 "adding dynamic member %s for ring %u", (
char *)new_val.
data, ring_no);
609 static void corosync_totem_dynamic_init (
void)
615 totem_dynamic_notify,
620 static void corosync_totem_stats_init (
void)
628 corosync_totem_stats_updater,
629 &corosync_stats_timer_handle);
633 static void deliver_fn (
636 unsigned int msg_len,
637 int endian_conversion_required)
639 const struct qb_ipc_request_header *
header;
645 if (endian_conversion_required) {
668 if (endian_conversion_required) {
669 assert(
corosync_service[service]->exec_engine[fn_id].exec_endian_convert_fn != NULL);
679 const struct iovec *iovec,
680 unsigned int iov_len,
683 const struct qb_ipc_request_header *req = iovec->iov_base;
687 service = req->id >> 16;
688 fn_id = req->id & 0xffff;
697 static qb_loop_timer_handle recheck_the_q_level_timer;
713 unsigned int service,
716 void *sending_allowed_private_data)
720 struct iovec reserve_iovec;
721 struct qb_ipc_request_header *header = (
struct qb_ipc_request_header *)msg;
724 reserve_iovec.iov_base = (
char *)header;
725 reserve_iovec.iov_len = header->size;
728 corosync_group_handle,
734 sending_allowed = QB_FALSE;
740 sending_allowed = QB_TRUE;
742 sending_allowed = QB_TRUE;
749 return -EHOSTUNREACH;
752 return (sending_allowed);
770 assert (source != NULL);
781 assert ((source != NULL) && (conn != NULL));
794 static void timer_function_scheduler_timeout (
void *data)
797 unsigned long long tv_current;
798 unsigned long long tv_diff;
800 tv_current = qb_util_nano_current_get ();
802 if (timeout_data->
tv_prev == 0) {
806 timeout_data->
tv_prev = tv_current;
810 tv_diff = tv_current - timeout_data->
tv_prev;
811 timeout_data->
tv_prev = tv_current;
815 "(threshold is %0.4f ms). Consider token timeout increase.",
816 (
float)tv_diff / QB_TIME_NS_IN_MSEC, (
float)timeout_data->
max_tv_diff / QB_TIME_NS_IN_MSEC);
823 qb_loop_timer_add (corosync_poll_handle,
827 timer_function_scheduler_timeout,
832 static void corosync_setscheduler (
void)
834 #if defined(HAVE_PTHREAD_SETSCHEDPARAM) && defined(HAVE_SCHED_GET_PRIORITY_MAX) && defined(HAVE_SCHED_SETSCHEDULER)
837 sched_priority = sched_get_priority_max (SCHED_RR);
838 if (sched_priority != -1) {
843 "Could not set SCHED_RR at priority %d",
847 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
848 qb_log_thread_priority_set (SCHED_OTHER, 0);
855 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
856 res = qb_log_thread_priority_set (SCHED_RR, sched_priority);
862 "Could not set logsys thread priority."
863 " Can't continue because of priority inversions.");
869 "Could not get maximum scheduler priority");
874 "The Platform is missing process priority setting features. Leaving at default.");
879 _logsys_log_printf(
int level,
int subsys,
880 const char *function_name,
881 const char *file_name,
887 _logsys_log_printf(
int level,
int subsys,
888 const
char *function_name,
889 const
char *file_name,
891 const
char *format, ...)
895 va_start(ap, format);
896 qb_log_from_external_source_va(function_name, file_name,
897 format, level, file_line,
902 static void fplay_key_change_notify_fn (
904 const char *key_name,
909 if (strcmp(key_name,
"runtime.blackbox.dump_flight_data") == 0) {
910 fprintf(stderr,
"Writetofile\n");
911 corosync_blackbox_write_to_file ();
913 if (strcmp(key_name,
"runtime.blackbox.dump_state") == 0) {
914 fprintf(stderr,
"statefump\n");
919 static void corosync_fplay_control_init (
void)
928 fplay_key_change_notify_fn,
932 fplay_key_change_notify_fn,
943 static void set_icmap_ro_keys_flag (
void)
966 static void main_service_ready (
void)
979 corosync_totem_stats_init ();
980 corosync_fplay_control_init ();
981 corosync_totem_dynamic_init ();
983 corosync_sync_callbacks_retrieve,
984 corosync_sync_completed);
987 static enum e_corosync_done corosync_flock (
const char *lockfile, pid_t pid)
997 lf = open (lockfile, O_WRONLY | O_CREAT, 0640);
1004 lock.l_type = F_WRLCK;
1006 lock.l_whence = SEEK_SET;
1008 if (fcntl (lf, F_SETLK, &
lock) == -1) {
1028 if (ftruncate (lf, 0) == -1) {
1032 goto error_close_unlink;
1035 memset (pid_s, 0,
sizeof (pid_s));
1036 snprintf (pid_s,
sizeof (pid_s) - 1,
"%u\n", pid);
1039 if (write (lf, pid_s, strlen (pid_s)) != strlen (pid_s)) {
1040 if (errno == EINTR) {
1044 "Error was %s", strerror (errno));
1046 goto error_close_unlink;
1050 if ((fd_flag = fcntl (lf, F_GETFD, 0)) == -1) {
1052 "Error was %s", strerror (errno));
1054 goto error_close_unlink;
1056 fd_flag |= FD_CLOEXEC;
1057 if (fcntl (lf, F_SETFD, fd_flag) == -1) {
1059 "Error was %s", strerror (errno));
1061 goto error_close_unlink;
1074 int main (
int argc,
char **argv,
char **envp)
1076 const char *error_string;
1079 int background, setprio;
1080 struct stat stat_out;
1081 char corosync_lib_dir[PATH_MAX];
1083 uint64_t totem_config_warnings;
1091 while ((ch = getopt (argc, argv,
"fprv")) != EOF) {
1103 printf (
"Corosync Cluster Engine, version '%s'\n",
VERSION);
1104 printf (
"Copyright (c) 2006-2009 Red Hat, Inc.\n");
1105 return EXIT_SUCCESS;
1111 " -f : Start application in foreground.\n"\
1112 " -p : Does nothing. \n"\
1113 " -r : Set round robin realtime scheduling \n"\
1114 " -v : Display version and SVN revision of Corosync and exit.\n");
1115 return EXIT_FAILURE;
1125 corosync_setscheduler ();
1128 corosync_mlockall ();
1133 (void)signal (SIGSEGV, sigsegv_handler);
1134 (void)signal (SIGABRT, sigabrt_handler);
1135 #if MSG_NOSIGNAL != 0
1136 (void)signal (SIGPIPE, SIG_IGN);
1143 set_icmap_ro_keys_flag();
1166 fprintf(stderr,
"%s", error_string);
1177 sprintf (corosync_lib_dir,
"%s/lib/corosync",
LOCALSTATEDIR);
1178 res = stat (corosync_lib_dir, &stat_out);
1179 if ((res == -1) || (res == 0 && !S_ISDIR(stat_out.st_mode))) {
1202 if (totem_config_warnings != 0) {
1235 corosync_tty_detach ();
1238 corosync_poll_handle = qb_loop_create ();
1240 memset(&scheduler_pause_timeout_data, 0,
sizeof(scheduler_pause_timeout_data));
1242 timer_function_scheduler_timeout (&scheduler_pause_timeout_data);
1244 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_LOW,
1245 SIGUSR2, NULL, sig_diag_handler, NULL);
1246 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1247 SIGINT, NULL, sig_exit_handler, NULL);
1248 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1249 SIGQUIT, NULL, sig_exit_handler, NULL);
1250 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1251 SIGTERM, NULL, sig_exit_handler, NULL);
1258 if ((flock_err = corosync_flock (corosync_lock_file, getpid ())) !=
COROSYNC_DONE_EXIT) {
1273 corosync_poll_handle,
1277 main_service_ready);
1280 &corosync_group_handle,
1285 corosync_group_handle,
1306 qb_loop_run (corosync_poll_handle);
1316 qb_loop_destroy (corosync_poll_handle);
1325 unlink (corosync_lock_file);
1329 return EXIT_SUCCESS;
int32_t cs_ipcs_q_level_get(void)
unsigned int corosync_service_defaults_link_and_init(struct corosync_api_v1 *corosync_api)
Load all of the default services.
void(*) in log_level_security)
int corosync_quorum_is_quorate(void)
void totempg_finalize(void)
Totem Single Ring Protocol.
uint64_t memb_commit_token_rx
#define LOGSYS_LEVEL_INFO
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
#define LOGSYS_LEVEL_TRACE
void totempg_trans_ack(void)
struct totem_config * totem_config
void sync_start(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
void(* timer_delete)(corosync_timer_handle_t timer_handle)
qb_loop_timer_handle handle
totemsrp_token_stats_t token[TOTEM_TOKEN_STATS_MAX]
Totem Single Ring Protocol.
int(* sync_process)(void)
void(* exec_dump_fn)(void)
int sync_init(int(*sync_callbacks_retrieve)(int service_id, struct sync_callbacks *callbacks), void(*synchronization_completed)(void))
int totemip_parse(struct totem_ip_address *totemip, const char *addr, int family)
int coroparse_configparse(icmap_map_t config_map, const char **error_string)
int(* timer_add_duration)(unsigned long long nanoseconds_in_future, void *data, void(*timer_nf)(void *data), corosync_timer_handle_t *handle)
qb_loop_t * cs_poll_handle_get(void)
void corosync_recheck_the_q_level(void *data)
#define corosync_exit_error(err)
void(* exec_endian_convert_fn)(void *msg)
int totempg_groups_joined_release(int msg_count)
struct message_header header
uint64_t memb_merge_detect_rx
void schedwrk_init(void(*serialize_lock_fn)(void), void(*serialize_unlock_fn)(void))
cs_error_t icmap_set_string(const char *key_name, const char *value)
int cs_poll_dispatch_add(qb_loop_t *handle, int fd, int events, void *data, int(*dispatch_fn)(int fd, int revents, void *data))
int totempg_groups_initialize(void **instance, void(*deliver_fn)(unsigned int nodeid, const void *msg, unsigned int msg_len, int endian_conversion_required), void(*confchg_fn)(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id))
Initialize a groups instance.
uint64_t gather_token_lost
icmap_map_t icmap_get_global_map(void)
unsigned long long max_tv_diff
struct corosync_service_engine * corosync_service[SERVICES_COUNT_MAX]
int _logsys_subsys_create(const char *subsys, const char *filename)
uint64_t memb_commit_token_tx
#define LOGSYS_PERROR(err_num, level, fmt, args...)
cs_error_t icmap_inc(const char *key_name)
#define log_printf(level, format, args...)
#define MAX_NO_CONT_SENDMSG_FAILURES
void(* exec_handler_fn)(const void *msg, unsigned int nodeid)
#define TOTEM_TOKEN_STATS_MAX
void cs_ipcs_sync_state_changed(int32_t sync_in_process)
int corosync_sending_allowed(unsigned int service, unsigned int id, const void *msg, void *sending_allowed_private_data)
int main_mcast(const struct iovec *iovec, unsigned int iov_len, unsigned int guarantee)
void cs_ipc_allow_connections(int32_t allow)
void corosync_service_unlink_all(struct corosync_api_v1 *api, void(*unlink_all_complete)(void))
Unlink and exit all corosync services.
uint64_t operational_entered
#define ICMAP_TRACK_DELETE
void totempg_service_ready_register(void(*totem_service_ready)(void))
void(* sync_activate)(void)
#define ICMAP_KEYNAME_MAXLEN
uint64_t operational_token_lost
#define MAX_NO_CONT_GATHER
Maximum number of continuous gather states.
uint64_t consensus_timeouts
int totem_config_keyread(struct totem_config *totem_config, const char **error_string)
uint64_t recovery_token_lost
#define LOGSYS_LEVEL_WARNING
#define ICMAP_TRACK_MODIFY
struct corosync_exec_handler * exec_engine
unsigned int totempg_my_nodeid_get(void)
cs_error_t icmap_set_uint32(const char *key_name, uint32_t value)
uint64_t commit_token_lost
uint64_t token_hold_cancel_rx
int logsys_thread_start(void)
#define LOGSYS_DECLARE_SUBSYS(subsys)
int totem_config_validate(struct totem_config *totem_config, const char **error_string)
#define LOGSYS_MODE_OUTPUT_SYSLOG
int totempg_groups_joined_reserve(void *instance, const struct iovec *iovec, unsigned int iov_len)
#define LOGSYS_LEVEL_ERROR
void sync_save_transitional(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
int totempg_groups_mcast_joined(void *instance, const struct iovec *iovec, unsigned int iov_len, int guarantee)
const char * service_stats_tx[SERVICES_COUNT_MAX][SERVICE_HANDLER_MAXIMUM_COUNT]
cs_error_t icmap_get(const char *key_name, void *value, size_t *value_len, icmap_value_types_t *type)
int corosync_log_config_read(cmap_handle_t cmap_h, const char *default_logfile, const char **error_string)
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
#define LOGSYS_LEVEL_DEBUG
void(* sync_activate)(void)
int totempg_member_remove(const struct totem_ip_address *member, int ring_no)
void(* confchg_fn)(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id)
#define TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED
#define TOTEM_CONFIG_WARNING_MEMBERS_IGNORED
uint32_t continuous_gather
cs_error_t icmap_set_uint64(const char *key_name, uint64_t value)
cs_error_t icmap_fast_inc(const char *key_name)
#define MILLI_2_NANO_SECONDS
void message_source_set(mar_message_source_t *source, void *conn)
int(* sync_process)(void)
void logsys_config_apply(void)
uint64_t token_hold_cancel_tx
unsigned int token_timeout
void totempg_check_q_level(void *instance)
const char * service_stats_rx[SERVICES_COUNT_MAX][SERVICE_HANDLER_MAXIMUM_COUNT]
struct corosync_api_v1 * apidef_get(void)
void corosync_sending_allowed_release(void *sending_allowed_private_data)
void cs_ipcs_stats_update(void)
uint64_t memb_merge_detect_tx
#define SERVICES_COUNT_MAX
unsigned long long tv_prev
void corosync_state_dump(void)
qb_loop_timer_handle corosync_timer_handle_t
int cs_poll_dispatch_delete(qb_loop_t *handle, int fd)
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
int message_source_is_local(const mar_message_source_t *source)
struct totem_logging_configuration totem_logging_configuration
int totempg_groups_join(void *instance, const struct totempg_group *groups, size_t group_cnt)
#define LOGSYS_LEVEL_NOTICE
cs_error_t icmap_set_uint8(const char *key_name, uint8_t value)
uint64_t recovery_entered
cs_error_t icmap_set_ro_access(const char *key_name, int prefix, int ro_access)
int totempg_initialize(qb_loop_t *poll_handle, struct totem_config *totem_config)
Initialize the totem process groups abstraction.
void corosync_shutdown_request(void)
struct sched_param global_sched_param
uint32_t continuous_sendmsg_failures
struct memb_ring_id ring_id
const char *(* totem_ifaces_print)(unsigned int nodeid)
int main(int argc, char **argv, char **envp)
cs_error_t icmap_init(void)
#define TOTEM_CONFIG_WARNING_TOTEM_NODEID_IGNORED
LOGSYS_DECLARE_SYSTEM("corosync", LOGSYS_MODE_OUTPUT_STDERR|LOGSYS_MODE_OUTPUT_SYSLOG, LOG_DAEMON, LOG_INFO)
#define LOGSYS_MODE_OUTPUT_STDERR
void *(* totem_get_stats)(void)
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
int totem_config_read(struct totem_config *totem_config, const char **error_string, uint64_t *warnings)
#define ICMAP_TRACK_PREFIX
int totempg_member_add(const struct totem_ip_address *member, int ring_no)