corosync  2.3.2
totemrrp.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2005 MontaVista Software, Inc.
3  * Copyright (c) 2006-2012 Red Hat, Inc.
4  *
5  * All rights reserved.
6  *
7  * Author: Steven Dake (sdake@redhat.com)
8  *
9  * This software licensed under BSD license, the text of which follows:
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are met:
13  *
14  * - Redistributions of source code must retain the above copyright notice,
15  * this list of conditions and the following disclaimer.
16  * - Redistributions in binary form must reproduce the above copyright notice,
17  * this list of conditions and the following disclaimer in the documentation
18  * and/or other materials provided with the distribution.
19  * - Neither the name of the MontaVista Software, Inc. nor the names of its
20  * contributors may be used to endorse or promote products derived from this
21  * software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 #include <config.h>
37 
38 #include <assert.h>
39 #include <pthread.h>
40 #include <sys/mman.h>
41 #include <sys/types.h>
42 #include <sys/stat.h>
43 #include <sys/socket.h>
44 #include <netdb.h>
45 #include <sys/un.h>
46 #include <sys/ioctl.h>
47 #include <sys/param.h>
48 #include <netinet/in.h>
49 #include <arpa/inet.h>
50 #include <unistd.h>
51 #include <fcntl.h>
52 #include <stdlib.h>
53 #include <stdio.h>
54 #include <errno.h>
55 #include <sched.h>
56 #include <time.h>
57 #include <sys/time.h>
58 #include <sys/poll.h>
59 #include <limits.h>
60 
61 #include <corosync/sq.h>
62 #include <corosync/list.h>
63 #include <corosync/swab.h>
64 #include <qb/qbdefs.h>
65 #include <qb/qbloop.h>
66 #define LOGSYS_UTILS_ONLY 1
67 #include <corosync/logsys.h>
68 
69 #include "totemnet.h"
70 #include "totemrrp.h"
71 
72 void rrp_deliver_fn (
73  void *context,
74  const void *msg,
75  unsigned int msg_len);
76 
78  void *context,
79  const struct totem_ip_address *iface_addr);
80 
81 struct totemrrp_instance;
84  unsigned int *faulty;
85  unsigned int *token_recv_count;
86  unsigned int *mcast_recv_count;
87  unsigned char token[15000];
88  unsigned int token_len;
89  qb_loop_timer_handle timer_expired_token;
90  qb_loop_timer_handle timer_problem_decrementer;
92  unsigned int token_xmit_iface;
93  unsigned int msg_xmit_iface;
94 };
95 
98  unsigned int *faulty;
99  unsigned int *last_token_recv;
100  unsigned int *counter_problems;
101  unsigned char token[15000];
102  unsigned int token_len;
103  unsigned int last_token_seq;
104  qb_loop_timer_handle timer_expired_token;
105  qb_loop_timer_handle timer_problem_decrementer;
107 };
108 
109 struct rrp_algo {
110  const char *name;
111 
112  void * (*initialize) (
113  struct totemrrp_instance *rrp_instance,
114  int interface_count);
115 
116  void (*mcast_recv) (
117  struct totemrrp_instance *instance,
118  unsigned int iface_no,
119  void *context,
120  const void *msg,
121  unsigned int msg_len);
122 
124  struct totemrrp_instance *instance,
125  const void *msg,
126  unsigned int msg_len);
127 
129  struct totemrrp_instance *instance,
130  const void *msg,
131  unsigned int msg_len);
132 
133  void (*token_recv) (
134  struct totemrrp_instance *instance,
135  unsigned int iface_no,
136  void *context,
137  const void *msg,
138  unsigned int msg_len,
139  unsigned int token_seqid);
140 
141  void (*token_send) (
142  struct totemrrp_instance *instance,
143  const void *msg,
144  unsigned int msg_len);
145 
146  void (*recv_flush) (
147  struct totemrrp_instance *instance);
148 
149  void (*send_flush) (
150  struct totemrrp_instance *instance);
151 
152  void (*iface_check) (
153  struct totemrrp_instance *instance);
154 
156  struct totemrrp_instance *instance,
157  unsigned int processor_count);
158 
160  struct totemrrp_instance *instance,
161  struct totem_ip_address *token_target,
162  unsigned int iface_no);
163 
164  void (*ring_reenable) (
165  struct totemrrp_instance *instance,
166  unsigned int iface_no);
167 
169  struct totemrrp_instance *instance);
170 
171  int (*member_add) (
172  struct totemrrp_instance *instance,
173  const struct totem_ip_address *member,
174  unsigned int iface_no);
175 
176  int (*member_remove) (
177  struct totemrrp_instance *instance,
178  const struct totem_ip_address *member,
179  unsigned int iface_no);
180 
181 };
182 
183 #define STATUS_STR_LEN 512
185  qb_loop_t *poll_handle;
186 
188 
190 
191  void *context;
192 
194 
196  void *context,
197  const void *msg,
198  unsigned int msg_len);
199 
201  void *context,
202  const struct totem_ip_address *iface_addr,
203  unsigned int iface_no);
204 
206  const void *msg,
207  unsigned int *seqid,
208  unsigned int *token_is);
209 
211  void *context);
212 
213  unsigned int (*totemrrp_msgs_missing) (void);
214 
215  /*
216  * Function and data used to log messages
217  */
219 
221 
223 
225 
227 
229 
231  int level,
232  int subsys,
233  const char *function,
234  const char *file,
235  int line,
236  const char *format, ...)__attribute__((format(printf, 6, 7)));
237 
238  void **net_handles;
239 
241 
243 
245 
247 
249 
251 
253 
255 };
256 
257 static void stats_set_interface_faulty(struct totemrrp_instance *rrp_instance,
258  unsigned int iface_no, int is_faulty);
259 
260 /*
261  * None Replication Forward Declerations
262  */
263 static void none_mcast_recv (
264  struct totemrrp_instance *instance,
265  unsigned int iface_no,
266  void *context,
267  const void *msg,
268  unsigned int msg_len);
269 
270 static void none_mcast_noflush_send (
271  struct totemrrp_instance *instance,
272  const void *msg,
273  unsigned int msg_len);
274 
275 static void none_mcast_flush_send (
276  struct totemrrp_instance *instance,
277  const void *msg,
278  unsigned int msg_len);
279 
280 static void none_token_recv (
281  struct totemrrp_instance *instance,
282  unsigned int iface_no,
283  void *context,
284  const void *msg,
285  unsigned int msg_len,
286  unsigned int token_seqid);
287 
288 static void none_token_send (
289  struct totemrrp_instance *instance,
290  const void *msg,
291  unsigned int msg_len);
292 
293 static void none_recv_flush (
294  struct totemrrp_instance *instance);
295 
296 static void none_send_flush (
297  struct totemrrp_instance *instance);
298 
299 static void none_iface_check (
300  struct totemrrp_instance *instance);
301 
302 static void none_processor_count_set (
303  struct totemrrp_instance *instance,
304  unsigned int processor_count_set);
305 
306 static void none_token_target_set (
307  struct totemrrp_instance *instance,
308  struct totem_ip_address *token_target,
309  unsigned int iface_no);
310 
311 static void none_ring_reenable (
312  struct totemrrp_instance *instance,
313  unsigned int iface_no);
314 
315 static int none_mcast_recv_empty (
316  struct totemrrp_instance *instance);
317 
318 static int none_member_add (
319  struct totemrrp_instance *instance,
320  const struct totem_ip_address *member,
321  unsigned int iface_no);
322 
323 static int none_member_remove (
324  struct totemrrp_instance *instance,
325  const struct totem_ip_address *member,
326  unsigned int iface_no);
327 /*
328  * Passive Replication Forward Declerations
329  */
330 static void *passive_instance_initialize (
331  struct totemrrp_instance *rrp_instance,
332  int interface_count);
333 
334 static void passive_mcast_recv (
335  struct totemrrp_instance *instance,
336  unsigned int iface_no,
337  void *context,
338  const void *msg,
339  unsigned int msg_len);
340 
341 static void passive_mcast_noflush_send (
342  struct totemrrp_instance *instance,
343  const void *msg,
344  unsigned int msg_len);
345 
346 static void passive_mcast_flush_send (
347  struct totemrrp_instance *instance,
348  const void *msg,
349  unsigned int msg_len);
350 
351 static void passive_monitor (
352  struct totemrrp_instance *rrp_instance,
353  unsigned int iface_no,
354  int is_token_recv_count);
355 
356 static void passive_token_recv (
357  struct totemrrp_instance *instance,
358  unsigned int iface_no,
359  void *context,
360  const void *msg,
361  unsigned int msg_len,
362  unsigned int token_seqid);
363 
364 static void passive_token_send (
365  struct totemrrp_instance *instance,
366  const void *msg,
367  unsigned int msg_len);
368 
369 static void passive_recv_flush (
370  struct totemrrp_instance *instance);
371 
372 static void passive_send_flush (
373  struct totemrrp_instance *instance);
374 
375 static void passive_iface_check (
376  struct totemrrp_instance *instance);
377 
378 static void passive_processor_count_set (
379  struct totemrrp_instance *instance,
380  unsigned int processor_count_set);
381 
382 static void passive_token_target_set (
383  struct totemrrp_instance *instance,
384  struct totem_ip_address *token_target,
385  unsigned int iface_no);
386 
387 static void passive_ring_reenable (
388  struct totemrrp_instance *instance,
389  unsigned int iface_no);
390 
391 static int passive_mcast_recv_empty (
392  struct totemrrp_instance *instance);
393 
394 static int passive_member_add (
395  struct totemrrp_instance *instance,
396  const struct totem_ip_address *member,
397  unsigned int iface_no);
398 
399 static int passive_member_remove (
400  struct totemrrp_instance *instance,
401  const struct totem_ip_address *member,
402  unsigned int iface_no);
403 /*
404  * Active Replication Forward Definitions
405  */
406 static void *active_instance_initialize (
407  struct totemrrp_instance *rrp_instance,
408  int interface_count);
409 
410 static void active_mcast_recv (
411  struct totemrrp_instance *instance,
412  unsigned int iface_no,
413  void *context,
414  const void *msg,
415  unsigned int msg_len);
416 
417 static void active_mcast_noflush_send (
418  struct totemrrp_instance *instance,
419  const void *msg,
420  unsigned int msg_len);
421 
422 static void active_mcast_flush_send (
423  struct totemrrp_instance *instance,
424  const void *msg,
425  unsigned int msg_len);
426 
427 static void active_token_recv (
428  struct totemrrp_instance *instance,
429  unsigned int iface_no,
430  void *context,
431  const void *msg,
432  unsigned int msg_len,
433  unsigned int token_seqid);
434 
435 static void active_token_send (
436  struct totemrrp_instance *instance,
437  const void *msg,
438  unsigned int msg_len);
439 
440 static void active_recv_flush (
441  struct totemrrp_instance *instance);
442 
443 static void active_send_flush (
444  struct totemrrp_instance *instance);
445 
446 static void active_iface_check (
447  struct totemrrp_instance *instance);
448 
449 static void active_processor_count_set (
450  struct totemrrp_instance *instance,
451  unsigned int processor_count_set);
452 
453 static void active_token_target_set (
454  struct totemrrp_instance *instance,
455  struct totem_ip_address *token_target,
456  unsigned int iface_no);
457 
458 static void active_ring_reenable (
459  struct totemrrp_instance *instance,
460  unsigned int iface_no);
461 
462 static int active_mcast_recv_empty (
463  struct totemrrp_instance *instance);
464 
465 static int active_member_add (
466  struct totemrrp_instance *instance,
467  const struct totem_ip_address *member,
468  unsigned int iface_no);
469 
470 static int active_member_remove (
471  struct totemrrp_instance *instance,
472  const struct totem_ip_address *member,
473  unsigned int iface_no);
474 
475 static void active_timer_expired_token_start (
477 
478 static void active_timer_expired_token_cancel (
480 
481 static void active_timer_problem_decrementer_start (
483 
484 static void active_timer_problem_decrementer_cancel (
486 
487 /*
488  * 0-5 reserved for totemsrp.c
489  */
490 #define MESSAGE_TYPE_RING_TEST_ACTIVE 6
491 #define MESSAGE_TYPE_RING_TEST_ACTIVATE 7
492 
493 #define ENDIAN_LOCAL 0xff22
494 
495 /*
496  * Rollover handling:
497  *
498  * ARR_SEQNO_START_TOKEN is the starting sequence number of last seen sequence
499  * for a token for active redundand ring. This should remain zero, unless testing
500  * overflow in which case 07fffff00 or 0xffffff00 are good starting values.
501  * It should be same as on defined in totemsrp.c
502  */
503 
504 #define ARR_SEQNO_START_TOKEN 0x0
505 
506 /*
507  * These can be used ot test different rollover points
508  * #define ARR_SEQNO_START_MSG 0xfffffe00
509  */
510 
511 /*
512  * Threshold value when recv_count for passive rrp should be adjusted.
513  * Set this value to some smaller for testing of adjusting proper
514  * functionality. Also keep in mind that this value must be smaller
515  * then rrp_problem_count_threshold
516  */
517 #define PASSIVE_RECV_COUNT_THRESHOLD (INT_MAX / 2)
518 
520  char type;
522  unsigned short endian_detector;
525 } __attribute__((packed));
529  void *context;
530  int iface_no;
531 };
532 
533 struct rrp_algo none_algo = {
534  .name = "none",
535  .initialize = NULL,
536  .mcast_recv = none_mcast_recv,
537  .mcast_noflush_send = none_mcast_noflush_send,
538  .mcast_flush_send = none_mcast_flush_send,
539  .token_recv = none_token_recv,
540  .token_send = none_token_send,
541  .recv_flush = none_recv_flush,
542  .send_flush = none_send_flush,
543  .iface_check = none_iface_check,
544  .processor_count_set = none_processor_count_set,
545  .token_target_set = none_token_target_set,
546  .ring_reenable = none_ring_reenable,
547  .mcast_recv_empty = none_mcast_recv_empty,
548  .member_add = none_member_add,
549  .member_remove = none_member_remove
550 };
551 
553  .name = "passive",
554  .initialize = passive_instance_initialize,
555  .mcast_recv = passive_mcast_recv,
556  .mcast_noflush_send = passive_mcast_noflush_send,
557  .mcast_flush_send = passive_mcast_flush_send,
558  .token_recv = passive_token_recv,
559  .token_send = passive_token_send,
560  .recv_flush = passive_recv_flush,
561  .send_flush = passive_send_flush,
562  .iface_check = passive_iface_check,
563  .processor_count_set = passive_processor_count_set,
564  .token_target_set = passive_token_target_set,
565  .ring_reenable = passive_ring_reenable,
566  .mcast_recv_empty = passive_mcast_recv_empty,
567  .member_add = passive_member_add,
568  .member_remove = passive_member_remove
569 };
570 
572  .name = "active",
573  .initialize = active_instance_initialize,
574  .mcast_recv = active_mcast_recv,
575  .mcast_noflush_send = active_mcast_noflush_send,
576  .mcast_flush_send = active_mcast_flush_send,
577  .token_recv = active_token_recv,
578  .token_send = active_token_send,
579  .recv_flush = active_recv_flush,
580  .send_flush = active_send_flush,
581  .iface_check = active_iface_check,
582  .processor_count_set = active_processor_count_set,
583  .token_target_set = active_token_target_set,
584  .ring_reenable = active_ring_reenable,
585  .mcast_recv_empty = active_mcast_recv_empty,
586  .member_add = active_member_add,
587  .member_remove = active_member_remove
588 };
589 
590 struct rrp_algo *rrp_algos[] = {
591  &none_algo,
592  &passive_algo,
593  &active_algo
594 };
595 
596 #define RRP_ALGOS_COUNT 3
597 
598 #define log_printf(level, format, args...) \
599 do { \
600  rrp_instance->totemrrp_log_printf ( \
601  level, rrp_instance->totemrrp_subsys_id, \
602  __FUNCTION__, __FILE__, __LINE__, \
603  format, ##args); \
604 } while (0);
605 
606 static void stats_set_interface_faulty(struct totemrrp_instance *rrp_instance,
607  unsigned int iface_no, int is_faulty)
608 {
609  rrp_instance->stats.faulty[iface_no] = (is_faulty ? 1 : 0);
610 }
611 
612 static void test_active_msg_endian_convert(const struct message_header *in, struct message_header *out)
613 {
614  out->type = in->type;
615  out->encapsulated = in->encapsulated;
617  out->ring_number = swab32 (in->ring_number);
619 }
620 
621 static void timer_function_test_ring_timeout (void *context)
622 {
623  struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
624  struct totemrrp_instance *rrp_instance = deliver_fn_context->instance;
625  unsigned int *faulty = NULL;
626  int iface_no = deliver_fn_context->iface_no;
627  struct message_header msg = {
629  .endian_detector = ENDIAN_LOCAL,
630  };
631 
632  if (strcmp(rrp_instance->totem_config->rrp_mode, "active") == 0)
633  faulty = ((struct active_instance *)(rrp_instance->rrp_algo_instance))->faulty;
634  if (strcmp(rrp_instance->totem_config->rrp_mode, "passive") == 0)
635  faulty = ((struct passive_instance *)(rrp_instance->rrp_algo_instance))->faulty;
636 
637  assert (faulty != NULL);
638 
639  if (faulty[iface_no] == 1) {
640  msg.ring_number = iface_no;
641  msg.nodeid_activator = rrp_instance->my_nodeid;
643  rrp_instance->net_handles[iface_no],
644  &msg, sizeof (struct message_header));
645  qb_loop_timer_add (rrp_instance->poll_handle,
646  QB_LOOP_MED,
647  rrp_instance->totem_config->rrp_autorecovery_check_timeout*QB_TIME_NS_IN_MSEC,
648  (void *)deliver_fn_context,
649  timer_function_test_ring_timeout,
650  &rrp_instance->timer_active_test_ring_timeout[iface_no]);
651  }
652 }
653 
654 /*
655  * None Replication Implementation
656  */
657 
658 static void none_mcast_recv (
659  struct totemrrp_instance *rrp_instance,
660  unsigned int iface_no,
661  void *context,
662  const void *msg,
663  unsigned int msg_len)
664 {
665  rrp_instance->totemrrp_deliver_fn (
666  context,
667  msg,
668  msg_len);
669 }
670 
671 static void none_mcast_flush_send (
672  struct totemrrp_instance *instance,
673  const void *msg,
674  unsigned int msg_len)
675 {
676  totemnet_mcast_flush_send (instance->net_handles[0], msg, msg_len);
677 }
678 
679 static void none_mcast_noflush_send (
680  struct totemrrp_instance *instance,
681  const void *msg,
682  unsigned int msg_len)
683 {
684  totemnet_mcast_noflush_send (instance->net_handles[0], msg, msg_len);
685 }
686 
687 static void none_token_recv (
688  struct totemrrp_instance *rrp_instance,
689  unsigned int iface_no,
690  void *context,
691  const void *msg,
692  unsigned int msg_len,
693  unsigned int token_seq)
694 {
695  rrp_instance->totemrrp_deliver_fn (
696  context,
697  msg,
698  msg_len);
699 }
700 
701 static void none_token_send (
702  struct totemrrp_instance *instance,
703  const void *msg,
704  unsigned int msg_len)
705 {
707  instance->net_handles[0],
708  msg, msg_len);
709 }
710 
711 static void none_recv_flush (struct totemrrp_instance *instance)
712 {
713  totemnet_recv_flush (instance->net_handles[0]);
714 }
715 
716 static void none_send_flush (struct totemrrp_instance *instance)
717 {
718  totemnet_send_flush (instance->net_handles[0]);
719 }
720 
721 static void none_iface_check (struct totemrrp_instance *instance)
722 {
723  totemnet_iface_check (instance->net_handles[0]);
724 }
725 
726 static void none_processor_count_set (
727  struct totemrrp_instance *instance,
728  unsigned int processor_count)
729 {
731  processor_count);
732 }
733 
734 static void none_token_target_set (
735  struct totemrrp_instance *instance,
736  struct totem_ip_address *token_target,
737  unsigned int iface_no)
738 {
739  totemnet_token_target_set (instance->net_handles[0], token_target);
740 }
741 
742 static void none_ring_reenable (
743  struct totemrrp_instance *instance,
744  unsigned int iface_no)
745 {
746  /*
747  * No operation
748  */
749 }
750 
751 static int none_mcast_recv_empty (
752  struct totemrrp_instance *instance)
753 {
754  int res;
755 
756  res = totemnet_recv_mcast_empty (instance->net_handles[0]);
757 
758  return (res);
759 }
760 
761 static int none_member_add (
762  struct totemrrp_instance *instance,
763  const struct totem_ip_address *member,
764  unsigned int iface_no)
765 {
766  int res;
767  res = totemnet_member_add (instance->net_handles[0], member);
768  return (res);
769 }
770 
771 static int none_member_remove (
772  struct totemrrp_instance *instance,
773  const struct totem_ip_address *member,
774  unsigned int iface_no)
775 {
776  int res;
777  res = totemnet_member_remove (instance->net_handles[0], member);
778  return (res);
779 }
780 
781 
782 /*
783  * Passive Replication Implementation
784  */
785 void *passive_instance_initialize (
786  struct totemrrp_instance *rrp_instance,
787  int interface_count)
788 {
789  struct passive_instance *instance;
790  int i;
791 
792  instance = malloc (sizeof (struct passive_instance));
793  if (instance == 0) {
794  goto error_exit;
795  }
796  memset (instance, 0, sizeof (struct passive_instance));
797 
798  instance->faulty = malloc (sizeof (int) * interface_count);
799  if (instance->faulty == 0) {
800  free (instance);
801  instance = 0;
802  goto error_exit;
803  }
804  memset (instance->faulty, 0, sizeof (int) * interface_count);
805 
806  for (i = 0; i < interface_count; i++) {
807  stats_set_interface_faulty (rrp_instance, i, 0);
808  }
809 
810  instance->token_recv_count = malloc (sizeof (int) * interface_count);
811  if (instance->token_recv_count == 0) {
812  free (instance->faulty);
813  free (instance);
814  instance = 0;
815  goto error_exit;
816  }
817  memset (instance->token_recv_count, 0, sizeof (int) * interface_count);
818 
819  instance->mcast_recv_count = malloc (sizeof (int) * interface_count);
820  if (instance->mcast_recv_count == 0) {
821  free (instance->token_recv_count);
822  free (instance->faulty);
823  free (instance);
824  instance = 0;
825  goto error_exit;
826  }
827  memset (instance->mcast_recv_count, 0, sizeof (int) * interface_count);
828 
829 error_exit:
830  return ((void *)instance);
831 }
832 
833 static void timer_function_passive_token_expired (void *context)
834 {
835  struct passive_instance *passive_instance = (struct passive_instance *)context;
836  struct totemrrp_instance *rrp_instance = passive_instance->rrp_instance;
837 
838  rrp_instance->totemrrp_deliver_fn (
839  passive_instance->totemrrp_context,
840  passive_instance->token,
841  passive_instance->token_len);
842 }
843 
844 /* TODO
845 static void timer_function_passive_problem_decrementer (void *context)
846 {
847 // struct passive_instance *passive_instance = (struct passive_instance *)context;
848 // struct totemrrp_instance *rrp_instance = passive_instance->rrp_instance;
849 
850 }
851 */
852 
853 
854 static void passive_timer_expired_token_start (
855  struct passive_instance *passive_instance)
856 {
857  qb_loop_timer_add (
858  passive_instance->rrp_instance->poll_handle,
859  QB_LOOP_MED,
860  passive_instance->rrp_instance->totem_config->rrp_token_expired_timeout*QB_TIME_NS_IN_MSEC,
861  (void *)passive_instance,
862  timer_function_passive_token_expired,
863  &passive_instance->timer_expired_token);
864 }
865 
866 static void passive_timer_expired_token_cancel (
867  struct passive_instance *passive_instance)
868 {
869  qb_loop_timer_del (
870  passive_instance->rrp_instance->poll_handle,
871  passive_instance->timer_expired_token);
872 }
873 
874 /*
875 static void passive_timer_problem_decrementer_start (
876  struct passive_instance *passive_instance)
877 {
878  qb_loop_timer_add (
879  QB_LOOP_MED,
880  passive_instance->rrp_instance->poll_handle,
881  passive_instance->rrp_instance->totem_config->rrp_problem_count_timeout*QB_TIME_NS_IN_MSEC,
882  (void *)passive_instance,
883  timer_function_passive_problem_decrementer,
884  &passive_instance->timer_problem_decrementer);
885 }
886 
887 static void passive_timer_problem_decrementer_cancel (
888  struct passive_instance *passive_instance)
889 {
890  qb_loop_timer_del (
891  passive_instance->rrp_instance->poll_handle,
892  passive_instance->timer_problem_decrementer);
893 }
894 */
895 
896 /*
897  * Monitor function implementation from rrp paper.
898  * rrp_instance is passive rrp instance, iface_no is interface with received messgae/token and
899  * is_token_recv_count is boolean variable which donates if message is token (>1) or regular
900  * message (= 0)
901  */
902 static void passive_monitor (
903  struct totemrrp_instance *rrp_instance,
904  unsigned int iface_no,
905  int is_token_recv_count)
906 {
907  struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
908  unsigned int *recv_count;
909  unsigned int max;
910  unsigned int i;
911  unsigned int min_all, min_active;
912  unsigned int threshold;
913 
914  /*
915  * Monitor for failures
916  */
917  if (is_token_recv_count) {
918  recv_count = passive_instance->token_recv_count;
919  threshold = rrp_instance->totem_config->rrp_problem_count_threshold;
920  } else {
921  recv_count = passive_instance->mcast_recv_count;
922  threshold = rrp_instance->totem_config->rrp_problem_count_mcast_threshold;
923  }
924 
925  recv_count[iface_no] += 1;
926 
927  max = 0;
928  for (i = 0; i < rrp_instance->interface_count; i++) {
929  if (max < recv_count[i]) {
930  max = recv_count[i];
931  }
932  }
933 
934  /*
935  * Max is larger then threshold -> start adjusting process
936  */
938  min_all = min_active = recv_count[iface_no];
939 
940  for (i = 0; i < rrp_instance->interface_count; i++) {
941  if (recv_count[i] < min_all) {
942  min_all = recv_count[i];
943  }
944 
945  if (passive_instance->faulty[i] == 0 &&
946  recv_count[i] < min_active) {
947  min_active = recv_count[i];
948  }
949  }
950 
951  if (min_all > 0) {
952  /*
953  * There is one or more faulty device with recv_count > 0
954  */
955  for (i = 0; i < rrp_instance->interface_count; i++) {
956  recv_count[i] -= min_all;
957  }
958  } else {
959  /*
960  * No faulty device with recv_count > 0, adjust only active
961  * devices
962  */
963  for (i = 0; i < rrp_instance->interface_count; i++) {
964  if (passive_instance->faulty[i] == 0) {
965  recv_count[i] -= min_active;
966  }
967  }
968  }
969 
970  /*
971  * Find again max
972  */
973  max = 0;
974 
975  for (i = 0; i < rrp_instance->interface_count; i++) {
976  if (max < recv_count[i]) {
977  max = recv_count[i];
978  }
979  }
980  }
981 
982  for (i = 0; i < rrp_instance->interface_count; i++) {
983  if ((passive_instance->faulty[i] == 0) &&
984  (max - recv_count[i] > threshold)) {
985  passive_instance->faulty[i] = 1;
986 
987  qb_loop_timer_add (rrp_instance->poll_handle,
988  QB_LOOP_MED,
989  rrp_instance->totem_config->rrp_autorecovery_check_timeout*QB_TIME_NS_IN_MSEC,
990  rrp_instance->deliver_fn_context[i],
991  timer_function_test_ring_timeout,
992  &rrp_instance->timer_active_test_ring_timeout[i]);
993 
994  stats_set_interface_faulty (rrp_instance, i, passive_instance->faulty[i]);
995 
996  snprintf (rrp_instance->status[i], STATUS_STR_LEN,
997  "Marking ringid %u interface %s FAULTY",
998  i,
999  totemnet_iface_print (rrp_instance->net_handles[i]));
1000  log_printf (
1001  rrp_instance->totemrrp_log_level_error,
1002  "%s",
1003  rrp_instance->status[i]);
1004  }
1005  }
1006 }
1007 
1008 static void passive_mcast_recv (
1009  struct totemrrp_instance *rrp_instance,
1010  unsigned int iface_no,
1011  void *context,
1012  const void *msg,
1013  unsigned int msg_len)
1014 {
1015  struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
1016 
1017  rrp_instance->totemrrp_deliver_fn (
1018  context,
1019  msg,
1020  msg_len);
1021 
1022  if (rrp_instance->totemrrp_msgs_missing() == 0 &&
1023  passive_instance->timer_expired_token) {
1024  /*
1025  * Delivers the last token
1026  */
1027  rrp_instance->totemrrp_deliver_fn (
1028  passive_instance->totemrrp_context,
1029  passive_instance->token,
1030  passive_instance->token_len);
1031  passive_timer_expired_token_cancel (passive_instance);
1032  }
1033 
1034  passive_monitor (rrp_instance, iface_no, 0);
1035 }
1036 
1037 static void passive_mcast_flush_send (
1038  struct totemrrp_instance *instance,
1039  const void *msg,
1040  unsigned int msg_len)
1041 {
1042  struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
1043  int i = 0;
1044 
1045  do {
1046  passive_instance->msg_xmit_iface = (passive_instance->msg_xmit_iface + 1) % instance->interface_count;
1047  i++;
1048  } while ((i <= instance->interface_count) && (passive_instance->faulty[passive_instance->msg_xmit_iface] == 1));
1049 
1050  if (i <= instance->interface_count) {
1051  totemnet_mcast_flush_send (instance->net_handles[passive_instance->msg_xmit_iface], msg, msg_len);
1052  }
1053 }
1054 
1055 static void passive_mcast_noflush_send (
1056  struct totemrrp_instance *instance,
1057  const void *msg,
1058  unsigned int msg_len)
1059 {
1060  struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
1061  int i = 0;
1062 
1063  do {
1064  passive_instance->msg_xmit_iface = (passive_instance->msg_xmit_iface + 1) % instance->interface_count;
1065  i++;
1066  } while ((i <= instance->interface_count) && (passive_instance->faulty[passive_instance->msg_xmit_iface] == 1));
1067 
1068  if (i <= instance->interface_count) {
1069  totemnet_mcast_noflush_send (instance->net_handles[passive_instance->msg_xmit_iface], msg, msg_len);
1070  }
1071 }
1072 
1073 static void passive_token_recv (
1074  struct totemrrp_instance *rrp_instance,
1075  unsigned int iface_no,
1076  void *context,
1077  const void *msg,
1078  unsigned int msg_len,
1079  unsigned int token_seq)
1080 {
1081  struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
1082 
1083  passive_instance->totemrrp_context = context; // this should be in totemrrp_instance ? TODO
1084 
1085  if (rrp_instance->totemrrp_msgs_missing() == 0) {
1086  rrp_instance->totemrrp_deliver_fn (
1087  context,
1088  msg,
1089  msg_len);
1090  } else {
1091  memcpy (passive_instance->token, msg, msg_len);
1092  passive_timer_expired_token_start (passive_instance);
1093 
1094  }
1095 
1096  passive_monitor (rrp_instance, iface_no, 1);
1097 }
1098 
1099 static void passive_token_send (
1100  struct totemrrp_instance *instance,
1101  const void *msg,
1102  unsigned int msg_len)
1103 {
1104  struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
1105  int i = 0;
1106 
1107  do {
1108  passive_instance->token_xmit_iface = (passive_instance->token_xmit_iface + 1) % instance->interface_count;
1109  i++;
1110  } while ((i <= instance->interface_count) && (passive_instance->faulty[passive_instance->token_xmit_iface] == 1));
1111 
1112  if (i <= instance->interface_count) {
1114  instance->net_handles[passive_instance->token_xmit_iface],
1115  msg, msg_len);
1116  }
1117 
1118 }
1119 
1120 static void passive_recv_flush (struct totemrrp_instance *instance)
1121 {
1122  struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1123  unsigned int i;
1124 
1125  for (i = 0; i < instance->interface_count; i++) {
1126  if (rrp_algo_instance->faulty[i] == 0) {
1127 
1128  totemnet_recv_flush (instance->net_handles[i]);
1129  }
1130  }
1131 }
1132 
1133 static void passive_send_flush (struct totemrrp_instance *instance)
1134 {
1135  struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1136  unsigned int i;
1137 
1138  for (i = 0; i < instance->interface_count; i++) {
1139  if (rrp_algo_instance->faulty[i] == 0) {
1140 
1141  totemnet_send_flush (instance->net_handles[i]);
1142  }
1143  }
1144 }
1145 
1146 static void passive_iface_check (struct totemrrp_instance *instance)
1147 {
1148  struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1149  unsigned int i;
1150 
1151  for (i = 0; i < instance->interface_count; i++) {
1152  if (rrp_algo_instance->faulty[i] == 0) {
1153 
1154  totemnet_iface_check (instance->net_handles[i]);
1155  }
1156  }
1157 }
1158 
1159 static void passive_processor_count_set (
1160  struct totemrrp_instance *instance,
1161  unsigned int processor_count)
1162 {
1163  struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1164  unsigned int i;
1165 
1166  for (i = 0; i < instance->interface_count; i++) {
1167  if (rrp_algo_instance->faulty[i] == 0) {
1168 
1170  processor_count);
1171  }
1172  }
1173 }
1174 
1175 static void passive_token_target_set (
1176  struct totemrrp_instance *instance,
1177  struct totem_ip_address *token_target,
1178  unsigned int iface_no)
1179 {
1180  totemnet_token_target_set (instance->net_handles[iface_no], token_target);
1181 }
1182 
1183 static int passive_mcast_recv_empty (
1184  struct totemrrp_instance *instance)
1185 {
1186  int res;
1187  int msgs_emptied = 0;
1188  int i;
1189 
1190  for (i = 0; i < instance->interface_count; i++) {
1191  res = totemnet_recv_mcast_empty (instance->net_handles[i]);
1192  if (res == -1) {
1193  return (-1);
1194  }
1195  if (res == 1) {
1196  msgs_emptied = 1;
1197  }
1198  }
1199 
1200  return (msgs_emptied);
1201 }
1202 
1203 static int passive_member_add (
1204  struct totemrrp_instance *instance,
1205  const struct totem_ip_address *member,
1206  unsigned int iface_no)
1207 {
1208  int res;
1209  res = totemnet_member_add (instance->net_handles[iface_no], member);
1210  return (res);
1211 }
1212 
1213 static int passive_member_remove (
1214  struct totemrrp_instance *instance,
1215  const struct totem_ip_address *member,
1216  unsigned int iface_no)
1217 {
1218  int res;
1219  res = totemnet_member_remove (instance->net_handles[iface_no], member);
1220  return (res);
1221 }
1222 
1223 
1224 static void passive_ring_reenable (
1225  struct totemrrp_instance *instance,
1226  unsigned int iface_no)
1227 {
1228  struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
1229  int i;
1230 
1231  memset (rrp_algo_instance->mcast_recv_count, 0, sizeof (unsigned int) *
1232  instance->interface_count);
1233  memset (rrp_algo_instance->token_recv_count, 0, sizeof (unsigned int) *
1234  instance->interface_count);
1235 
1236  if (iface_no == instance->interface_count) {
1237  memset (rrp_algo_instance->faulty, 0, sizeof (unsigned int) *
1238  instance->interface_count);
1239  for (i = 0; i < instance->interface_count; i++) {
1240  stats_set_interface_faulty (instance, i, 0);
1241  }
1242  } else {
1243  rrp_algo_instance->faulty[iface_no] = 0;
1244  stats_set_interface_faulty (instance, iface_no, 0);
1245  }
1246 }
1247 
1248 /*
1249  * Active Replication Implementation
1250  */
1251 void *active_instance_initialize (
1252  struct totemrrp_instance *rrp_instance,
1253  int interface_count)
1254 {
1255  struct active_instance *instance;
1256  int i;
1257 
1258  instance = malloc (sizeof (struct active_instance));
1259  if (instance == 0) {
1260  goto error_exit;
1261  }
1262  memset (instance, 0, sizeof (struct active_instance));
1263 
1264  instance->faulty = malloc (sizeof (int) * interface_count);
1265  if (instance->faulty == 0) {
1266  free (instance);
1267  instance = 0;
1268  goto error_exit;
1269  }
1270  memset (instance->faulty, 0, sizeof (unsigned int) * interface_count);
1271 
1272  for (i = 0; i < interface_count; i++) {
1273  stats_set_interface_faulty (rrp_instance, i, 0);
1274  }
1275 
1276  instance->last_token_recv = malloc (sizeof (int) * interface_count);
1277  if (instance->last_token_recv == 0) {
1278  free (instance->faulty);
1279  free (instance);
1280  instance = 0;
1281  goto error_exit;
1282  }
1283  memset (instance->last_token_recv, 0, sizeof (unsigned int) * interface_count);
1284 
1285  instance->counter_problems = malloc (sizeof (int) * interface_count);
1286  if (instance->counter_problems == 0) {
1287  free (instance->last_token_recv);
1288  free (instance->faulty);
1289  free (instance);
1290  instance = 0;
1291  goto error_exit;
1292  }
1293  memset (instance->counter_problems, 0, sizeof (unsigned int) * interface_count);
1294 
1295  instance->timer_expired_token = 0;
1296 
1297  instance->timer_problem_decrementer = 0;
1298 
1299  instance->rrp_instance = rrp_instance;
1300 
1301  instance->last_token_seq = ARR_SEQNO_START_TOKEN - 1;
1302 
1303 error_exit:
1304  return ((void *)instance);
1305 }
1306 static void timer_function_active_problem_decrementer (void *context)
1307 {
1308  struct active_instance *active_instance = (struct active_instance *)context;
1309  struct totemrrp_instance *rrp_instance = active_instance->rrp_instance;
1310  unsigned int problem_found = 0;
1311  unsigned int i;
1312 
1313  for (i = 0; i < rrp_instance->interface_count; i++) {
1314  if (active_instance->counter_problems[i] > 0) {
1315  problem_found = 1;
1316  active_instance->counter_problems[i] -= 1;
1317  if (active_instance->counter_problems[i] == 0) {
1318  snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1319  "ring %d active with no faults", i);
1320  } else {
1321  snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1322  "Decrementing problem counter for iface %s to [%d of %d]",
1323  totemnet_iface_print (rrp_instance->net_handles[i]),
1324  active_instance->counter_problems[i],
1325  rrp_instance->totem_config->rrp_problem_count_threshold);
1326  }
1327  log_printf (
1328  rrp_instance->totemrrp_log_level_warning,
1329  "%s",
1330  rrp_instance->status[i]);
1331  }
1332  }
1333  if (problem_found) {
1334  active_timer_problem_decrementer_start (active_instance);
1335  } else {
1336  active_instance->timer_problem_decrementer = 0;
1337  }
1338 }
1339 
1340 static void timer_function_active_token_expired (void *context)
1341 {
1342  struct active_instance *active_instance = (struct active_instance *)context;
1343  struct totemrrp_instance *rrp_instance = active_instance->rrp_instance;
1344  unsigned int i;
1345 
1346  for (i = 0; i < rrp_instance->interface_count; i++) {
1347  if (active_instance->last_token_recv[i] == 0) {
1348  active_instance->counter_problems[i] += 1;
1349 
1350  if (active_instance->timer_problem_decrementer == 0) {
1351  active_timer_problem_decrementer_start (active_instance);
1352  }
1353  snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1354  "Incrementing problem counter for seqid %d iface %s to [%d of %d]",
1355  active_instance->last_token_seq,
1356  totemnet_iface_print (rrp_instance->net_handles[i]),
1357  active_instance->counter_problems[i],
1358  rrp_instance->totem_config->rrp_problem_count_threshold);
1359  log_printf (
1360  rrp_instance->totemrrp_log_level_warning,
1361  "%s",
1362  rrp_instance->status[i]);
1363  }
1364  }
1365  for (i = 0; i < rrp_instance->interface_count; i++) {
1366  if (active_instance->counter_problems[i] >= rrp_instance->totem_config->rrp_problem_count_threshold &&
1367  active_instance->faulty[i] == 0) {
1368  active_instance->faulty[i] = 1;
1369 
1370  qb_loop_timer_add (rrp_instance->poll_handle,
1371  QB_LOOP_MED,
1372  rrp_instance->totem_config->rrp_autorecovery_check_timeout*QB_TIME_NS_IN_MSEC,
1373  rrp_instance->deliver_fn_context[i],
1374  timer_function_test_ring_timeout,
1375  &rrp_instance->timer_active_test_ring_timeout[i]);
1376 
1377  stats_set_interface_faulty (rrp_instance, i, active_instance->faulty[i]);
1378 
1379  snprintf (rrp_instance->status[i], STATUS_STR_LEN,
1380  "Marking seqid %d ringid %u interface %s FAULTY",
1381  active_instance->last_token_seq,
1382  i,
1383  totemnet_iface_print (rrp_instance->net_handles[i]));
1384  log_printf (
1385  rrp_instance->totemrrp_log_level_error,
1386  "%s",
1387  rrp_instance->status[i]);
1388  active_timer_problem_decrementer_cancel (active_instance);
1389  }
1390  }
1391 
1392  rrp_instance->totemrrp_deliver_fn (
1393  active_instance->totemrrp_context,
1394  active_instance->token,
1395  active_instance->token_len);
1396 }
1397 
1398 static void active_timer_expired_token_start (
1399  struct active_instance *active_instance)
1400 {
1401  qb_loop_timer_add (
1402  active_instance->rrp_instance->poll_handle,
1403  QB_LOOP_MED,
1404  active_instance->rrp_instance->totem_config->rrp_token_expired_timeout*QB_TIME_NS_IN_MSEC,
1405  (void *)active_instance,
1406  timer_function_active_token_expired,
1407  &active_instance->timer_expired_token);
1408 }
1409 
1410 static void active_timer_expired_token_cancel (
1411  struct active_instance *active_instance)
1412 {
1413  qb_loop_timer_del (
1414  active_instance->rrp_instance->poll_handle,
1415  active_instance->timer_expired_token);
1416 }
1417 
1418 static void active_timer_problem_decrementer_start (
1419  struct active_instance *active_instance)
1420 {
1421  qb_loop_timer_add (
1422  active_instance->rrp_instance->poll_handle,
1423  QB_LOOP_MED,
1424  active_instance->rrp_instance->totem_config->rrp_problem_count_timeout*QB_TIME_NS_IN_MSEC,
1425  (void *)active_instance,
1426  timer_function_active_problem_decrementer,
1427  &active_instance->timer_problem_decrementer);
1428 }
1429 
1430 static void active_timer_problem_decrementer_cancel (
1431  struct active_instance *active_instance)
1432 {
1433  qb_loop_timer_del (
1434  active_instance->rrp_instance->poll_handle,
1435  active_instance->timer_problem_decrementer);
1436 }
1437 
1438 
1439 /*
1440  * active replication
1441  */
1442 static void active_mcast_recv (
1443  struct totemrrp_instance *instance,
1444  unsigned int iface_no,
1445  void *context,
1446  const void *msg,
1447  unsigned int msg_len)
1448 {
1449  instance->totemrrp_deliver_fn (
1450  context,
1451  msg,
1452  msg_len);
1453 }
1454 
1455 static void active_mcast_flush_send (
1456  struct totemrrp_instance *instance,
1457  const void *msg,
1458  unsigned int msg_len)
1459 {
1460  int i;
1461  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1462 
1463  for (i = 0; i < instance->interface_count; i++) {
1464  if (rrp_algo_instance->faulty[i] == 0) {
1465  totemnet_mcast_flush_send (instance->net_handles[i], msg, msg_len);
1466  }
1467  }
1468 }
1469 
1470 static void active_mcast_noflush_send (
1471  struct totemrrp_instance *instance,
1472  const void *msg,
1473  unsigned int msg_len)
1474 {
1475  int i;
1476  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1477 
1478  for (i = 0; i < instance->interface_count; i++) {
1479  if (rrp_algo_instance->faulty[i] == 0) {
1480  totemnet_mcast_noflush_send (instance->net_handles[i], msg, msg_len);
1481  }
1482  }
1483 }
1484 
1485 static void active_token_recv (
1486  struct totemrrp_instance *rrp_instance,
1487  unsigned int iface_no,
1488  void *context,
1489  const void *msg,
1490  unsigned int msg_len,
1491  unsigned int token_seq)
1492 {
1493  int i;
1494  struct active_instance *active_instance = (struct active_instance *)rrp_instance->rrp_algo_instance;
1495 
1496  active_instance->totemrrp_context = context;
1497  if (sq_lt_compare (active_instance->last_token_seq, token_seq)) {
1498  memcpy (active_instance->token, msg, msg_len);
1499  active_instance->token_len = msg_len;
1500  for (i = 0; i < rrp_instance->interface_count; i++) {
1501  active_instance->last_token_recv[i] = 0;
1502  }
1503 
1504  active_instance->last_token_recv[iface_no] = 1;
1505  active_timer_expired_token_start (active_instance);
1506  }
1507 
1508  /*
1509  * This doesn't follow spec because the spec assumes we will know
1510  * when token resets occur.
1511  */
1512  active_instance->last_token_seq = token_seq;
1513 
1514  if (token_seq == active_instance->last_token_seq) {
1515  active_instance->last_token_recv[iface_no] = 1;
1516  for (i = 0; i < rrp_instance->interface_count; i++) {
1517  if ((active_instance->last_token_recv[i] == 0) &&
1518  active_instance->faulty[i] == 0) {
1519  return; /* don't deliver token */
1520  }
1521  }
1522  active_timer_expired_token_cancel (active_instance);
1523 
1524  rrp_instance->totemrrp_deliver_fn (
1525  context,
1526  msg,
1527  msg_len);
1528  }
1529 }
1530 
1531 static void active_token_send (
1532  struct totemrrp_instance *instance,
1533  const void *msg,
1534  unsigned int msg_len)
1535 {
1536  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1537  int i;
1538 
1539  for (i = 0; i < instance->interface_count; i++) {
1540  if (rrp_algo_instance->faulty[i] == 0) {
1542  instance->net_handles[i],
1543  msg, msg_len);
1544 
1545  }
1546  }
1547 }
1548 
1549 static void active_recv_flush (struct totemrrp_instance *instance)
1550 {
1551  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1552  unsigned int i;
1553 
1554  for (i = 0; i < instance->interface_count; i++) {
1555  if (rrp_algo_instance->faulty[i] == 0) {
1556 
1557  totemnet_recv_flush (instance->net_handles[i]);
1558  }
1559  }
1560 }
1561 
1562 static void active_send_flush (struct totemrrp_instance *instance)
1563 {
1564  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1565  unsigned int i;
1566 
1567  for (i = 0; i < instance->interface_count; i++) {
1568  if (rrp_algo_instance->faulty[i] == 0) {
1569 
1570  totemnet_send_flush (instance->net_handles[i]);
1571  }
1572  }
1573 }
1574 
1575 static int active_member_add (
1576  struct totemrrp_instance *instance,
1577  const struct totem_ip_address *member,
1578  unsigned int iface_no)
1579 {
1580  int res;
1581  res = totemnet_member_add (instance->net_handles[iface_no], member);
1582  return (res);
1583 }
1584 
1585 static int active_member_remove (
1586  struct totemrrp_instance *instance,
1587  const struct totem_ip_address *member,
1588  unsigned int iface_no)
1589 {
1590  int res;
1591  res = totemnet_member_remove (instance->net_handles[iface_no], member);
1592  return (res);
1593 }
1594 
1595 static void active_iface_check (struct totemrrp_instance *instance)
1596 {
1597  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1598  unsigned int i;
1599 
1600  for (i = 0; i < instance->interface_count; i++) {
1601  if (rrp_algo_instance->faulty[i] == 0) {
1602 
1603  totemnet_iface_check (instance->net_handles[i]);
1604  }
1605  }
1606 }
1607 
1608 static void active_processor_count_set (
1609  struct totemrrp_instance *instance,
1610  unsigned int processor_count)
1611 {
1612  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1613  unsigned int i;
1614 
1615  for (i = 0; i < instance->interface_count; i++) {
1616  if (rrp_algo_instance->faulty[i] == 0) {
1617 
1619  processor_count);
1620  }
1621  }
1622 }
1623 
1624 static void active_token_target_set (
1625  struct totemrrp_instance *instance,
1626  struct totem_ip_address *token_target,
1627  unsigned int iface_no)
1628 {
1629  totemnet_token_target_set (instance->net_handles[iface_no], token_target);
1630 }
1631 
1632 static int active_mcast_recv_empty (
1633  struct totemrrp_instance *instance)
1634 {
1635  int res;
1636  int msgs_emptied = 0;
1637  int i;
1638 
1639  for (i = 0; i < instance->interface_count; i++) {
1640  res = totemnet_recv_mcast_empty (instance->net_handles[i]);
1641  if (res == -1) {
1642  return (-1);
1643  }
1644  if (res == 1) {
1645  msgs_emptied = 1;
1646  }
1647  }
1648 
1649  return (msgs_emptied);
1650 }
1651 
1652 static void active_ring_reenable (
1653  struct totemrrp_instance *instance,
1654  unsigned int iface_no)
1655 {
1656  struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1657  int i;
1658 
1659  if (iface_no == instance->interface_count) {
1660  memset (rrp_algo_instance->last_token_recv, 0, sizeof (unsigned int) *
1661  instance->interface_count);
1662  memset (rrp_algo_instance->faulty, 0, sizeof (unsigned int) *
1663  instance->interface_count);
1664  memset (rrp_algo_instance->counter_problems, 0, sizeof (unsigned int) *
1665  instance->interface_count);
1666 
1667  for (i = 0; i < instance->interface_count; i++) {
1668  stats_set_interface_faulty (instance, i, 0);
1669  }
1670  } else {
1671  rrp_algo_instance->last_token_recv[iface_no] = 0;
1672  rrp_algo_instance->faulty[iface_no] = 0;
1673  rrp_algo_instance->counter_problems[iface_no] = 0;
1674 
1675  stats_set_interface_faulty (instance, iface_no, 0);
1676  }
1677 }
1678 
1679 static void totemrrp_instance_initialize (struct totemrrp_instance *instance)
1680 {
1681  memset (instance, 0, sizeof (struct totemrrp_instance));
1682 }
1683 
1684 static int totemrrp_algorithm_set (
1685  struct totem_config *totem_config,
1686  struct totemrrp_instance *instance)
1687 {
1688  unsigned int res = -1;
1689  unsigned int i;
1690 
1691  for (i = 0; i < RRP_ALGOS_COUNT; i++) {
1692  if (strcmp (totem_config->rrp_mode, rrp_algos[i]->name) == 0) {
1693  instance->rrp_algo = rrp_algos[i];
1694  if (rrp_algos[i]->initialize) {
1695  instance->rrp_algo_instance = rrp_algos[i]->initialize (
1696  instance,
1697  totem_config->interface_count);
1698  }
1699  res = 0;
1700  break;
1701  }
1702  }
1703  for (i = 0; i < totem_config->interface_count; i++) {
1704  instance->status[i] = malloc (STATUS_STR_LEN+1);
1705  snprintf (instance->status[i], STATUS_STR_LEN,
1706  "ring %d active with no faults", i);
1707  }
1708  return (res);
1709 }
1710 
1712  void *context,
1713  const void *msg,
1714  unsigned int msg_len)
1715 {
1716  unsigned int token_seqid;
1717  unsigned int token_is;
1718 
1719  struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
1720  struct totemrrp_instance *rrp_instance = deliver_fn_context->instance;
1721  const struct message_header *hdr = msg;
1722  struct message_header tmp_msg, activate_msg;
1723 
1724  memset(&tmp_msg, 0, sizeof(struct message_header));
1725  memset(&activate_msg, 0, sizeof(struct message_header));
1726 
1727  rrp_instance->totemrrp_token_seqid_get (
1728  msg,
1729  &token_seqid,
1730  &token_is);
1731 
1732  if (hdr->type == MESSAGE_TYPE_RING_TEST_ACTIVE) {
1733  log_printf (
1734  rrp_instance->totemrrp_log_level_debug,
1735  "received message requesting test of ring now active");
1736 
1737  if (hdr->endian_detector != ENDIAN_LOCAL) {
1738  test_active_msg_endian_convert(hdr, &tmp_msg);
1739  hdr = &tmp_msg;
1740  }
1741 
1742  if (hdr->nodeid_activator == rrp_instance->my_nodeid) {
1743  /*
1744  * Send an activate message
1745  */
1746  activate_msg.type = MESSAGE_TYPE_RING_TEST_ACTIVATE;
1747  activate_msg.endian_detector = ENDIAN_LOCAL;
1748  activate_msg.ring_number = hdr->ring_number;
1749  activate_msg.nodeid_activator = rrp_instance->my_nodeid;
1751  rrp_instance->net_handles[deliver_fn_context->iface_no],
1752  &activate_msg, sizeof (struct message_header));
1753  } else {
1754  /*
1755  * Send a ring test message
1756  */
1758  rrp_instance->net_handles[deliver_fn_context->iface_no],
1759  msg, msg_len);
1760  }
1761  } else
1762  if (hdr->type == MESSAGE_TYPE_RING_TEST_ACTIVATE) {
1763  log_printf (
1764  rrp_instance->totemrrp_log_level_notice,
1765  "Automatically recovered ring %d", hdr->ring_number);
1766 
1767  if (hdr->endian_detector != ENDIAN_LOCAL) {
1768  test_active_msg_endian_convert(hdr, &tmp_msg);
1769  hdr = &tmp_msg;
1770  }
1771 
1772  totemrrp_ring_reenable (rrp_instance, deliver_fn_context->iface_no);
1773  if (hdr->nodeid_activator != rrp_instance->my_nodeid) {
1775  rrp_instance->net_handles[deliver_fn_context->iface_no],
1776  msg, msg_len);
1777  }
1778  } else
1779  if (token_is) {
1780  /*
1781  * Deliver to the token receiver for this rrp algorithm
1782  */
1783  rrp_instance->rrp_algo->token_recv (
1784  rrp_instance,
1785  deliver_fn_context->iface_no,
1786  deliver_fn_context->context,
1787  msg,
1788  msg_len,
1789  token_seqid);
1790  } else {
1791  /*
1792  * Deliver to the mcast receiver for this rrp algorithm
1793  */
1794  rrp_instance->rrp_algo->mcast_recv (
1795  rrp_instance,
1796  deliver_fn_context->iface_no,
1797  deliver_fn_context->context,
1798  msg,
1799  msg_len);
1800  }
1801 }
1802 
1804  void *context,
1805  const struct totem_ip_address *iface_addr)
1806 {
1807  struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
1808 
1809  deliver_fn_context->instance->my_nodeid = iface_addr->nodeid;
1810  deliver_fn_context->instance->totemrrp_iface_change_fn (
1811  deliver_fn_context->context,
1812  iface_addr,
1813  deliver_fn_context->iface_no);
1814 }
1815 
1817  void *rrp_context)
1818 {
1819  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
1820  int i;
1821 
1822  for (i = 0; i < instance->interface_count; i++) {
1823  totemnet_finalize (instance->net_handles[i]);
1824  }
1825  free (instance->net_handles);
1826  free (instance);
1827  return (0);
1828 }
1829 
1830 static void rrp_target_set_completed (void *context)
1831 {
1832  struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
1833 
1834  deliver_fn_context->instance->totemrrp_target_set_completed (deliver_fn_context->context);
1835 }
1836 
1837 /*
1838  * Totem Redundant Ring interface
1839  * depends on poll abstraction, POSIX, IPV4
1840  */
1841 
1842 /*
1843  * Create an instance
1844  */
1846  qb_loop_t *poll_handle,
1847  void **rrp_context,
1848  struct totem_config *totem_config,
1849  totemsrp_stats_t *stats,
1850  void *context,
1851 
1852  void (*deliver_fn) (
1853  void *context,
1854  const void *msg,
1855  unsigned int msg_len),
1856 
1857  void (*iface_change_fn) (
1858  void *context,
1859  const struct totem_ip_address *iface_addr,
1860  unsigned int iface_no),
1861 
1862  void (*token_seqid_get) (
1863  const void *msg,
1864  unsigned int *seqid,
1865  unsigned int *token_is),
1866 
1867  unsigned int (*msgs_missing) (void),
1868 
1869  void (*target_set_completed) (void *context))
1870 {
1871  struct totemrrp_instance *instance;
1872  unsigned int res;
1873  int i;
1874 
1875  instance = malloc (sizeof (struct totemrrp_instance));
1876  if (instance == 0) {
1877  return (-1);
1878  }
1879 
1880  totemrrp_instance_initialize (instance);
1881 
1882  instance->totem_config = totem_config;
1883  stats->rrp = &instance->stats;
1884  instance->stats.interface_count = totem_config->interface_count;
1885  instance->stats.faulty = calloc(instance->stats.interface_count, sizeof(uint8_t));
1886 
1887  res = totemrrp_algorithm_set (
1888  instance->totem_config,
1889  instance);
1890  if (res == -1) {
1891  goto error_destroy;
1892  }
1893 
1894  /*
1895  * Configure logging
1896  */
1904 
1905  instance->interfaces = totem_config->interfaces;
1906 
1907  instance->poll_handle = poll_handle;
1908 
1909  instance->totemrrp_deliver_fn = deliver_fn;
1910 
1911  instance->totemrrp_iface_change_fn = iface_change_fn;
1912 
1913  instance->totemrrp_token_seqid_get = token_seqid_get;
1914 
1915  instance->totemrrp_target_set_completed = target_set_completed;
1916 
1917  instance->totemrrp_msgs_missing = msgs_missing;
1918 
1919  instance->interface_count = totem_config->interface_count;
1920 
1921  instance->net_handles = malloc (sizeof (void *) * totem_config->interface_count);
1922 
1923  instance->context = context;
1924 
1925  instance->poll_handle = poll_handle;
1926 
1927 
1928  for (i = 0; i < totem_config->interface_count; i++) {
1929  struct deliver_fn_context *deliver_fn_context;
1930 
1931  deliver_fn_context = malloc (sizeof (struct deliver_fn_context));
1932  assert (deliver_fn_context);
1933  deliver_fn_context->instance = instance;
1934  deliver_fn_context->context = context;
1935  deliver_fn_context->iface_no = i;
1936  instance->deliver_fn_context[i] = (void *)deliver_fn_context;
1937 
1939  poll_handle,
1940  &instance->net_handles[i],
1941  totem_config,
1942  stats,
1943  i,
1944  (void *)deliver_fn_context,
1947  rrp_target_set_completed);
1948 
1949  totemnet_net_mtu_adjust (instance->net_handles[i], totem_config);
1950  }
1951 
1952  *rrp_context = instance;
1953  return (0);
1954 
1955 error_destroy:
1956  free (instance);
1957  return (res);
1958 }
1959 
1960 void *totemrrp_buffer_alloc (void *rrp_context)
1961 {
1962  struct totemrrp_instance *instance = rrp_context;
1963  assert (instance != NULL);
1964  return totemnet_buffer_alloc (instance->net_handles[0]);
1965 }
1966 
1967 void totemrrp_buffer_release (void *rrp_context, void *ptr)
1968 {
1969  struct totemrrp_instance *instance = rrp_context;
1970  assert (instance != NULL);
1971  totemnet_buffer_release (instance->net_handles[0], ptr);
1972 }
1973 
1975  void *rrp_context,
1976  unsigned int processor_count)
1977 {
1978  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
1979  instance->rrp_algo->processor_count_set (instance, processor_count);
1980 
1981  instance->processor_count = processor_count;
1982 
1983  return (0);
1984 }
1985 
1987  void *rrp_context,
1988  struct totem_ip_address *addr,
1989  unsigned int iface_no)
1990 {
1991  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
1992  instance->rrp_algo->token_target_set (instance, addr, iface_no);
1993 
1994  return (0);
1995 }
1996 int totemrrp_recv_flush (void *rrp_context)
1997 {
1998  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
1999 
2000  instance->rrp_algo->recv_flush (instance);
2001 
2002  return (0);
2003 }
2004 
2005 int totemrrp_send_flush (void *rrp_context)
2006 {
2007  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2008  instance->rrp_algo->send_flush (instance);
2009 
2010  return (0);
2011 }
2012 
2014  void *rrp_context,
2015  const void *msg,
2016  unsigned int msg_len)
2017 {
2018  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2019  instance->rrp_algo->token_send (instance, msg, msg_len);
2020 
2021  return (0);
2022 }
2023 
2025  void *rrp_context,
2026  const void *msg,
2027  unsigned int msg_len)
2028 {
2029  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2030  int res = 0;
2031 
2032 // TODO this needs to return the result
2033  instance->rrp_algo->mcast_flush_send (instance, msg, msg_len);
2034 
2035  return (res);
2036 }
2037 
2039  void *rrp_context,
2040  const void *msg,
2041  unsigned int msg_len)
2042 {
2043  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2044  /*
2045  * merge detects go out through mcast_flush_send so it is safe to
2046  * flush these messages if we are only one processor. This avoids
2047  * an encryption/hmac and decryption/hmac
2048  */
2049  if (instance->processor_count > 1) {
2050 
2051 // TODO this needs to return the result
2052  instance->rrp_algo->mcast_noflush_send (instance, msg, msg_len);
2053  }
2054 
2055  return (0);
2056 }
2057 
2058 int totemrrp_iface_check (void *rrp_context)
2059 {
2060  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2061 
2062  instance->rrp_algo->iface_check (instance);
2063 
2064  return (0);
2065 }
2066 
2068  void *rrp_context,
2069  char ***status,
2070  unsigned int *iface_count)
2071 {
2072  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2073  *status = instance->status;
2074 
2075  if (iface_count) {
2076  *iface_count = instance->interface_count;
2077  }
2078 
2079  return (0);
2080 }
2081 
2083  void *rrp_context,
2084  const char *cipher_type,
2085  const char *hash_type)
2086 {
2087  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2088  int res;
2089 
2090  res = totemnet_crypto_set(instance->net_handles[0], cipher_type, hash_type);
2091 
2092  return (res);
2093 }
2094 
2095 
2096 /*
2097  * iface_no indicates the interface number [0, ..., interface_count-1] of the
2098  * specific ring which will be reenabled. We specify iface_no == interface_count
2099  * means reenabling all the rings.
2100  */
2102  void *rrp_context,
2103  unsigned int iface_no)
2104 {
2105  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2106  int res = 0;
2107  unsigned int i;
2108 
2109  instance->rrp_algo->ring_reenable (instance, iface_no);
2110 
2111  if (iface_no == instance->interface_count) {
2112  for (i = 0; i < instance->interface_count; i++) {
2113  snprintf (instance->status[i], STATUS_STR_LEN,
2114  "ring %d active with no faults", i);
2115  }
2116  } else {
2117  snprintf (instance->status[iface_no], STATUS_STR_LEN,
2118  "ring %d active with no faults", iface_no);
2119  }
2120 
2121  return (res);
2122 }
2123 
2125  void *rrp_context)
2126 {
2127  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2128  int res;
2129 
2130  res = instance->rrp_algo->mcast_recv_empty (instance);
2131 
2132  return (res);
2133 }
2134 
2136  void *rrp_context,
2137  const struct totem_ip_address *member,
2138  int iface_no)
2139 {
2140  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2141  int res;
2142 
2143  res = instance->rrp_algo->member_add (instance, member, iface_no);
2144 
2145  return (res);
2146 }
2147 
2149  void *rrp_context,
2150  const struct totem_ip_address *member,
2151  int iface_no)
2152 {
2153  struct totemrrp_instance *instance = (struct totemrrp_instance *)rrp_context;
2154  int res;
2155 
2156  res = instance->rrp_algo->member_remove (instance, member, iface_no);
2157 
2158  return (res);
2159 }