76 #include <sys/types.h>
81 #include <sys/socket.h>
84 #include <sys/resource.h>
86 #include <netinet/in.h>
87 #include <arpa/inet.h>
96 #include <semaphore.h>
98 #include <qb/qbdefs.h>
100 #include <qb/qbloop.h>
101 #include <qb/qbutil.h>
102 #include <qb/qbipcs.h>
124 #ifdef HAVE_SMALL_MEMORY_FOOTPRINT
125 #define IPC_LOGSYS_SIZE 1024*64
127 #define IPC_LOGSYS_SIZE 8192*128
137 #define SERVER_BACKLOG 5
139 static int sched_priority = 0;
141 static unsigned int service_count = 32;
147 static int sync_in_process = 1;
149 static qb_loop_t *corosync_poll_handle;
155 static const char *corosync_lock_file =
LOCALSTATEDIR"/run/corosync.pid";
157 static int ip_version = AF_INET;
161 return (corosync_poll_handle);
169 int (*dispatch_fn) (
int fd,
173 return qb_loop_poll_add(handle, QB_LOOP_MED, fd, events, data,
179 return qb_loop_poll_del(handle, fd);
193 static void corosync_blackbox_write_to_file (
void)
195 char fname[PATH_MAX];
196 char time_str[PATH_MAX];
197 struct tm cur_time_tm;
200 cur_time_t = time(NULL);
201 localtime_r(&cur_time_t, &cur_time_tm);
203 strftime(time_str, PATH_MAX,
"%Y-%m-%dT%H:%M:%S", &cur_time_tm);
204 snprintf(fname, PATH_MAX,
"%s/fdata-%s-%lld",
207 (
long long int)getpid());
209 qb_log_blackbox_write_to_file(fname);
215 static void unlink_all_completed (
void)
218 qb_loop_stop (corosync_poll_handle);
227 static int32_t sig_diag_handler (
int num,
void *data)
233 static int32_t sig_exit_handler (
int num,
void *data)
240 static void sigsegv_handler (
int num)
242 (void)signal (SIGSEGV, SIG_DFL);
243 corosync_blackbox_write_to_file ();
248 static void sigabrt_handler (
int num)
250 (void)signal (SIGABRT, SIG_DFL);
251 corosync_blackbox_write_to_file ();
256 #define LOCALHOST_IP inet_addr("127.0.0.1")
258 static void *corosync_group_handle;
265 static void serialize_lock (
void)
269 static void serialize_unlock (
void)
273 static void corosync_sync_completed (
void)
276 "Completed service synchronization, ready to provide service.");
287 static int corosync_sync_callbacks_retrieve (
295 if (callbacks == NULL) {
310 static void member_object_joined (
unsigned int nodeid)
317 "runtime.totem.pg.mrp.srp.members.%u.ip", nodeid);
319 "runtime.totem.pg.mrp.srp.members.%u.join_count", nodeid);
321 "runtime.totem.pg.mrp.srp.members.%u.status", nodeid);
336 static void member_object_left (
unsigned int nodeid)
341 "runtime.totem.pg.mrp.srp.members.%u.status", nodeid);
348 static void confchg_fn (
350 const unsigned int *member_list,
size_t member_list_entries,
351 const unsigned int *left_list,
size_t left_list_entries,
352 const unsigned int *joined_list,
size_t joined_list_entries,
356 int abort_activate = 0;
358 if (sync_in_process == 1) {
363 memcpy (&corosync_ring_id, ring_id,
sizeof (
struct memb_ring_id));
365 for (i = 0; i < left_list_entries; i++) {
366 member_object_left (left_list[i]);
368 for (i = 0; i < joined_list_entries; i++) {
369 member_object_joined (joined_list[i]);
374 for (i = 0; i < service_count; i++) {
377 member_list, member_list_entries,
378 left_list, left_list_entries,
379 joined_list, joined_list_entries, ring_id);
383 if (abort_activate) {
390 sync_start (member_list, member_list_entries, ring_id);
394 static void priv_drop (
void)
399 static void corosync_tty_detach (
void)
427 devnull = open(
"/dev/null", O_RDWR);
432 if (dup2(devnull, 0) < 0 || dup2(devnull, 1) < 0
433 || dup2(devnull, 2) < 0) {
438 static void corosync_mlockall (
void)
441 struct rlimit rlimit;
443 rlimit.rlim_cur = RLIM_INFINITY;
444 rlimit.rlim_max = RLIM_INFINITY;
446 #ifndef RLIMIT_MEMLOCK
447 #define RLIMIT_MEMLOCK RLIMIT_VMEM
450 setrlimit (RLIMIT_MEMLOCK, &rlimit);
452 res = mlockall (MCL_CURRENT | MCL_FUTURE);
455 "Could not lock memory of service to avoid page faults");
460 static void corosync_totem_stats_updater (
void *data)
463 uint32_t total_mtt_rx_token;
464 uint32_t total_backlog_calc;
465 uint32_t total_token_holdtime;
501 icmap_set_uint8(
"runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure",
507 "Totem is unable to form a cluster because of an "
508 "operating system or network fault. The most common "
509 "cause of this message is that the local firewall is "
510 "configured improperly.");
511 icmap_set_uint8(
"runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure", 1);
513 icmap_set_uint8(
"runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure", 0);
520 total_mtt_rx_token = 0;
521 total_token_holdtime = 0;
522 total_backlog_calc = 0;
543 icmap_set_uint32(
"runtime.totem.pg.mrp.srp.mtt_rx_token", (total_mtt_rx_token / token_count));
544 icmap_set_uint32(
"runtime.totem.pg.mrp.srp.avg_token_workload", (total_token_holdtime / token_count));
545 icmap_set_uint32(
"runtime.totem.pg.mrp.srp.avg_backlog_calc", (total_backlog_calc / token_count));
551 corosync_totem_stats_updater,
552 &corosync_stats_timer_handle);
555 static void totem_dynamic_notify(
557 const char *key_name,
566 int add_new_member = 0;
567 int remove_old_member = 0;
570 res = sscanf(key_name,
"nodelist.node.%u.ring%u%s", &member_no, &ring_no, tmp_str);
574 if (strcmp(tmp_str,
"_addr") != 0) {
583 remove_old_member = 1;
589 remove_old_member = 1;
592 if (remove_old_member) {
594 "removing dynamic member %s for ring %u", (
char *)old_val.
data, ring_no);
600 if (add_new_member) {
602 "adding dynamic member %s for ring %u", (
char *)new_val.
data, ring_no);
609 static void corosync_totem_dynamic_init (
void)
615 totem_dynamic_notify,
620 static void corosync_totem_stats_init (
void)
628 corosync_totem_stats_updater,
629 &corosync_stats_timer_handle);
633 static void deliver_fn (
636 unsigned int msg_len,
637 int endian_conversion_required)
639 const struct qb_ipc_request_header *
header;
645 if (endian_conversion_required) {
668 if (endian_conversion_required) {
669 assert(
corosync_service[service]->exec_engine[fn_id].exec_endian_convert_fn != NULL);
679 const struct iovec *iovec,
680 unsigned int iov_len,
683 const struct qb_ipc_request_header *req = iovec->iov_base;
687 service = req->id >> 16;
688 fn_id = req->id & 0xffff;
697 static qb_loop_timer_handle recheck_the_q_level_timer;
713 unsigned int service,
716 void *sending_allowed_private_data)
720 struct iovec reserve_iovec;
721 struct qb_ipc_request_header *header = (
struct qb_ipc_request_header *)msg;
724 reserve_iovec.iov_base = (
char *)header;
725 reserve_iovec.iov_len = header->size;
728 corosync_group_handle,
734 sending_allowed = QB_FALSE;
740 sending_allowed = QB_TRUE;
742 sending_allowed = QB_TRUE;
749 return -EHOSTUNREACH;
752 return (sending_allowed);
770 assert (source != NULL);
781 assert ((source != NULL) && (conn != NULL));
794 static void timer_function_scheduler_timeout (
void *data)
797 unsigned long long tv_current;
798 unsigned long long tv_diff;
800 tv_current = qb_util_nano_current_get ();
802 if (timeout_data->
tv_prev == 0) {
806 timeout_data->
tv_prev = tv_current;
810 tv_diff = tv_current - timeout_data->
tv_prev;
811 timeout_data->
tv_prev = tv_current;
815 "(threshold is %0.4f ms). Consider token timeout increase.",
816 (
float)tv_diff / QB_TIME_NS_IN_MSEC, (
float)timeout_data->
max_tv_diff / QB_TIME_NS_IN_MSEC);
823 qb_loop_timer_add (corosync_poll_handle,
827 timer_function_scheduler_timeout,
832 static void corosync_setscheduler (
void)
834 #if defined(HAVE_PTHREAD_SETSCHEDPARAM) && defined(HAVE_SCHED_GET_PRIORITY_MAX) && defined(HAVE_SCHED_SETSCHEDULER)
837 sched_priority = sched_get_priority_max (SCHED_RR);
838 if (sched_priority != -1) {
843 "Could not set SCHED_RR at priority %d",
847 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
848 qb_log_thread_priority_set (SCHED_OTHER, 0);
855 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
856 res = qb_log_thread_priority_set (SCHED_RR, sched_priority);
862 "Could not set logsys thread priority."
863 " Can't continue because of priority inversions.");
869 "Could not get maximum scheduler priority");
874 "The Platform is missing process priority setting features. Leaving at default.");
879 _logsys_log_printf(
int level,
int subsys,
880 const char *function_name,
881 const char *file_name,
887 _logsys_log_printf(
int level,
int subsys,
888 const
char *function_name,
889 const
char *file_name,
891 const
char *format, ...)
895 va_start(ap, format);
896 qb_log_from_external_source_va(function_name, file_name,
897 format, level, file_line,
902 static void fplay_key_change_notify_fn (
904 const char *key_name,
909 if (strcmp(key_name,
"runtime.blackbox.dump_flight_data") == 0) {
910 fprintf(stderr,
"Writetofile\n");
911 corosync_blackbox_write_to_file ();
913 if (strcmp(key_name,
"runtime.blackbox.dump_state") == 0) {
914 fprintf(stderr,
"statefump\n");
919 static void corosync_fplay_control_init (
void)
928 fplay_key_change_notify_fn,
932 fplay_key_change_notify_fn,
943 static void set_icmap_ro_keys_flag (
void)
966 static void main_service_ready (
void)
979 corosync_totem_stats_init ();
980 corosync_fplay_control_init ();
981 corosync_totem_dynamic_init ();
983 corosync_sync_callbacks_retrieve,
984 corosync_sync_completed);
987 static enum e_corosync_done corosync_flock (
const char *lockfile, pid_t pid)
997 lf = open (lockfile, O_WRONLY | O_CREAT, 0640);
1004 lock.l_type = F_WRLCK;
1006 lock.l_whence = SEEK_SET;
1008 if (fcntl (lf, F_SETLK, &
lock) == -1) {
1028 if (ftruncate (lf, 0) == -1) {
1032 goto error_close_unlink;
1035 memset (pid_s, 0,
sizeof (pid_s));
1036 snprintf (pid_s,
sizeof (pid_s) - 1,
"%u\n", pid);
1039 if (write (lf, pid_s, strlen (pid_s)) != strlen (pid_s)) {
1040 if (errno == EINTR) {
1044 "Error was %s", strerror (errno));
1046 goto error_close_unlink;
1050 if ((fd_flag = fcntl (lf, F_GETFD, 0)) == -1) {
1052 "Error was %s", strerror (errno));
1054 goto error_close_unlink;
1056 fd_flag |= FD_CLOEXEC;
1057 if (fcntl (lf, F_SETFD, fd_flag) == -1) {
1059 "Error was %s", strerror (errno));
1061 goto error_close_unlink;
1074 int main (
int argc,
char **argv,
char **envp)
1076 const char *error_string;
1079 int background, setprio;
1080 struct stat stat_out;
1081 char corosync_lib_dir[PATH_MAX];
1083 uint64_t totem_config_warnings;
1091 while ((ch = getopt (argc, argv,
"fprv")) != EOF) {
1103 printf (
"Corosync Cluster Engine, version '%s'\n",
VERSION);
1104 printf (
"Copyright (c) 2006-2009 Red Hat, Inc.\n");
1105 return EXIT_SUCCESS;
1111 " -f : Start application in foreground.\n"\
1112 " -p : Does nothing. \n"\
1113 " -r : Set round robin realtime scheduling \n"\
1114 " -v : Display version and SVN revision of Corosync and exit.\n");
1115 return EXIT_FAILURE;
1125 corosync_setscheduler ();
1128 corosync_mlockall ();
1133 (void)signal (SIGSEGV, sigsegv_handler);
1134 (void)signal (SIGABRT, sigabrt_handler);
1135 #if MSG_NOSIGNAL != 0
1136 (void)signal (SIGPIPE, SIG_IGN);
1143 set_icmap_ro_keys_flag();
1166 fprintf(stderr,
"%s", error_string);
1177 sprintf (corosync_lib_dir,
"%s/lib/corosync",
LOCALSTATEDIR);
1178 res = stat (corosync_lib_dir, &stat_out);
1179 if ((res == -1) || (res == 0 && !S_ISDIR(stat_out.st_mode))) {
1202 if (totem_config_warnings != 0) {
1235 corosync_tty_detach ();
1238 corosync_poll_handle = qb_loop_create ();
1240 memset(&scheduler_pause_timeout_data, 0,
sizeof(scheduler_pause_timeout_data));
1242 timer_function_scheduler_timeout (&scheduler_pause_timeout_data);
1244 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_LOW,
1245 SIGUSR2, NULL, sig_diag_handler, NULL);
1246 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1247 SIGINT, NULL, sig_exit_handler, NULL);
1248 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1249 SIGQUIT, NULL, sig_exit_handler, NULL);
1250 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1251 SIGTERM, NULL, sig_exit_handler, NULL);
1258 if ((flock_err = corosync_flock (corosync_lock_file, getpid ())) !=
COROSYNC_DONE_EXIT) {
1273 corosync_poll_handle,
1277 main_service_ready);
1280 &corosync_group_handle,
1285 corosync_group_handle,
1306 qb_loop_run (corosync_poll_handle);
1316 qb_loop_destroy (corosync_poll_handle);
1325 unlink (corosync_lock_file);
1329 return EXIT_SUCCESS;